diff --git a/hcl/hcl.go b/hcl/hcl.go index d1ddd36..56d9095 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -5,12 +5,13 @@ package hcl import ( - "bytes" "fmt" + "maps" "sort" "strings" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" ) @@ -27,25 +28,199 @@ func blockToMap(blocks []*hclwrite.Block) map[string]*hclwrite.Block { return blockMap } -func setAttrs(sourceBlock *hclwrite.Block, targetBlock *hclwrite.Block) { - attributes := sourceBlock.Body().Attributes() +// mergeTokens only merges tokens if they are a map, otherwise defaults to the aTokens +// only merges the top level keys of the map, if it exists the value is overridden +func (m *Merger) mergeTokens(aTokens hclwrite.Tokens, bTokens hclwrite.Tokens) (hclwrite.Tokens, error) { + if aTokens[0].Type != hclsyntax.TokenOBrace || aTokens[len(aTokens)-1].Type != hclsyntax.TokenCBrace { + return bTokens, nil + } + + if bTokens[0].Type != hclsyntax.TokenOBrace || bTokens[len(bTokens)-1].Type != hclsyntax.TokenCBrace { + return bTokens, nil + } + + aMap, err := objectForTokensMap(aTokens) + if err != nil { + return nil, fmt.Errorf("failed to deserialize tokens: %w", err) + } + + bMap, err := objectForTokensMap(bTokens) + if err != nil { + return nil, fmt.Errorf("failed to deserialize tokens: %w", err) + } + + outMap := make(map[string]hclwrite.ObjectAttrTokens) + // this merges the top layer of the map, where nested maps are overwritten + maps.Copy(outMap, aMap) + maps.Copy(outMap, bMap) - // sort the attributes to ensure consistent ordering - keys := make([]string, 0, len(attributes)) - for key := range attributes { + var values []hclwrite.ObjectAttrTokens + + // sort the keys to ensure consistent ordering + keys := make([]string, 0, len(outMap)) + for key := range outMap { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { - targetBlock.Body().SetAttributeRaw(key, attributes[key].Expr().BuildTokens(nil)) + values = append(values, outMap[key]) } + + return hclwrite.TokensForObject(values), nil } -func merge(aFile *hclwrite.File, bFile *hclwrite.File) *hclwrite.File { +// mergeAttrs merges two blocks' attributes together. Attributes are composed of hclTokens +// and are identified by their key, e.g. +// key = value +// or key = { ... } +func (m *Merger) mergeAttrs(aAttr map[string]*hclwrite.Attribute, bAttr map[string]*hclwrite.Attribute) map[string]hclwrite.Tokens { + outAttr := make(map[string]hclwrite.Tokens) + + for key, aValue := range aAttr { + bValue, found := bAttr[key] + + aAttrTokens := aValue.Expr().BuildTokens(nil) + + if found && m.options.MergeMapKeys { + bAttrTokens := bValue.Expr().BuildTokens(nil) + // attempt to merge the value, which are a list of attributes broken up into hclTokens + mergedTokens, err := m.mergeTokens(aAttrTokens, bAttrTokens) + if err != nil { + // if there was an error merging the tokens, default to the bAttrTokens + outAttr[key] = bAttrTokens + continue + } + + outAttr[key] = mergedTokens + } else if found { + // if the key is found in both attributes, default to the bAttrTokens + bAttrTokens := bValue.Expr().BuildTokens(nil) + outAttr[key] = bAttrTokens + } else { + outAttr[key] = aAttrTokens + } + } + + // add any attributes that are in bAttr but not in aAttr + for key, bValue := range bAttr { + _, found := aAttr[key] + + if !found { + bAttrTokens := bValue.Expr().BuildTokens(nil) + outAttr[key] = bAttrTokens + } + } + + return outAttr +} + +// objectForTokensMap is the inverse of hclwrite.TokensForObject, only merges the top level keys, but not the values of the keys. +// if a value exists, it is overridden +func objectForTokensMap(tokens hclwrite.Tokens) (map[string]hclwrite.ObjectAttrTokens, error) { + if len(tokens) < 2 || tokens[0].Type != hclsyntax.TokenOBrace || tokens[len(tokens)-1].Type != hclsyntax.TokenCBrace { + return nil, fmt.Errorf("tokens are not a valid object") + } + + result := make(map[string]hclwrite.ObjectAttrTokens) + var currentKey string // used for the result map + var currentKeyTokens hclwrite.Tokens // used for the ObjectAttrTokens key + var currentValueTokens hclwrite.Tokens // used for the ObjectAttrTokens value + var inValue bool // flag to determine if we are in the value part of the tokens when parsing + + i := 1 // start after the opening brace + for i < len(tokens)-1 { // skip the closing brace + token := tokens[i] + + switch token.Type { + case hclsyntax.TokenIdent, hclsyntax.TokenQuotedLit: + if inValue { + // set the value if in the value + currentValueTokens = append(currentValueTokens, token) + } else { + // set the key + currentKey = string(token.Bytes) + currentKeyTokens = append(currentKeyTokens, token) + } + + case hclsyntax.TokenEqual: + // flag that we are in the value part of the tokens + inValue = true + + case hclsyntax.TokenOBrace, hclsyntax.TokenOBrack: + // find the closing token for the look ahead + cToken := hclsyntax.TokenCBrace + if token.Type == hclsyntax.TokenOBrack { + cToken = hclsyntax.TokenCBrack + } + + // look ahead to find the end index of map/array + unclosedTokens := 1 + endIndex := -1 + for j := i + 1; j < len(tokens); j++ { + if tokens[j].Type == token.Type { + unclosedTokens++ + } else if tokens[j].Type == cToken { + unclosedTokens-- + } + if unclosedTokens == 0 { + endIndex = j + break + } + } + + if endIndex == -1 { + return nil, fmt.Errorf("failed to find closing token") + } + + // include the tokens for the map/array in the current value + currentValueTokens = append(currentValueTokens, tokens[i:endIndex+1]...) + i = endIndex + + case hclsyntax.TokenNewline, hclsyntax.TokenComma: + // if at the end of the value, add the key and value to the result map + if inValue { + result[currentKey] = hclwrite.ObjectAttrTokens{ + Name: currentKeyTokens, + Value: currentValueTokens, + } + + // reset the current key and value tokens to parse the next attribute + currentKey = "" + currentKeyTokens = hclwrite.Tokens{} + currentValueTokens = hclwrite.Tokens{} + inValue = false + } + + default: + if inValue { + // add tokens to the value until we hit the end of the value (comma or newline) + currentValueTokens = append(currentValueTokens, token) + } else { + // add tokens to the key until we hit the end of the key (equal sign) + currentKeyTokens = append(currentKeyTokens, token) + } + } + + i++ + } + + // add the last attribute found to the result map + if len(currentKeyTokens) > 0 && len(currentValueTokens) > 0 { + result[currentKey] = hclwrite.ObjectAttrTokens{ + Name: currentKeyTokens, + Value: currentValueTokens, + } + } + + return result, nil +} + +// mergeFiles merges two HCL files together +func (m *Merger) mergeFiles(aFile *hclwrite.File, bFile *hclwrite.File) *hclwrite.File { out := hclwrite.NewFile() - outBlocks := mergeBlocks(aFile.Body().Blocks(), bFile.Body().Blocks()) + outBlocks := m.mergeBlocks(aFile.Body().Blocks(), bFile.Body().Blocks()) lastIndex := len(outBlocks) - 1 @@ -62,7 +237,10 @@ func merge(aFile *hclwrite.File, bFile *hclwrite.File) *hclwrite.File { return out } -func mergeBlocks(aBlocks []*hclwrite.Block, bBlocks []*hclwrite.Block) []*hclwrite.Block { +// mergeBlocks merges two blocks together, a block is identified by its type and labels, e.g. +// type "label" { ... } +// or type { ... } +func (m *Merger) mergeBlocks(aBlocks []*hclwrite.Block, bBlocks []*hclwrite.Block) []*hclwrite.Block { outBlocks := make([]*hclwrite.Block, 0) aBlockMap := blockToMap(aBlocks) bBlockMap := blockToMap(bBlocks) @@ -76,14 +254,24 @@ func mergeBlocks(aBlocks []*hclwrite.Block, bBlocks []*hclwrite.Block) []*hclwri // override outBlock with the new block to merge the two blocks into outBlock = hclwrite.NewBlock(aBlock.Type(), aBlock.Labels()) - // set block attributes of the new block - setAttrs(aBlock, outBlock) - setAttrs(bBlock, outBlock) + // merge block attributes + outAttributes := m.mergeAttrs(aBlock.Body().Attributes(), bBlock.Body().Attributes()) + // sort the keys to ensure consistent ordering + keys := make([]string, 0, len(outAttributes)) + for key := range outAttributes { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + outBlock.Body().SetAttributeRaw(key, outAttributes[key]) + } // recursively merge nested blocks aNestedBlocks := aBlock.Body().Blocks() bNestedBlocks := bBlock.Body().Blocks() - outNestedBlocks := mergeBlocks(aNestedBlocks, bNestedBlocks) + outNestedBlocks := m.mergeBlocks(aNestedBlocks, bNestedBlocks) for _, nestedBlock := range outNestedBlocks { outBlock.Body().AppendNewline() @@ -116,7 +304,37 @@ func parseBytes(bytes []byte) (*hclwrite.File, error) { return sourceHclFile, nil } -func Merge(a string, b string) (string, error) { +// MergeOptions are the options for merging two HCL strings +type MergeOptions struct { + // MergeMapKeys merges the keys of maps together, note this does not merge the values of the keys. If + // unset, the keys of the second map will override the keys of the first map. + MergeMapKeys bool +} + +type merger interface { + Merge(a string, b string) (string, error) +} + +var _ merger = &Merger{} + +// NewMerger creates a new Merger with the provided options +func NewMerger(options *MergeOptions) *Merger { + if options == nil { + options = &MergeOptions{} + } + + return &Merger{ + options: options, + } +} + +// Merger is the struct that merges two HCL strings together +type Merger struct { + options *MergeOptions +} + +// Merge merges two HCL strings together +func (m *Merger) Merge(a string, b string) (string, error) { aBytes := []byte(a) bBytes := []byte(b) @@ -131,15 +349,9 @@ func Merge(a string, b string) (string, error) { return "", err } - // merge the blocks from the HCL files - out := merge(aFile, bFile) - - // write file to buffer - var buf bytes.Buffer - _, err = out.WriteTo(&buf) - if err != nil { - return "", fmt.Errorf("error writing HCL to file: %w", err) - } + // merge the blocks and attributes from the HCL files + outFile := m.mergeFiles(aFile, bFile) + outFileFormatted := hclwrite.Format(outFile.Bytes()) - return buf.String(), nil + return string(outFileFormatted), nil } diff --git a/hcl/hcl_test.go b/hcl/hcl_test.go index f2a27ae..2d19d8f 100644 --- a/hcl/hcl_test.go +++ b/hcl/hcl_test.go @@ -15,8 +15,9 @@ func TestMerge(t *testing.T) { t.Parallel() type input struct { - a string - b string + a string + b string + options *hcl.MergeOptions } tests := []struct { @@ -66,15 +67,15 @@ variable "b" { type = string description = "Variable A" override = true - b = "b" + b = "b" }`, }, want: `variable "a" { a = "a" + b = "b" description = "Variable A" override = true type = string - b = "b" } `, @@ -85,7 +86,7 @@ variable "b" { input: input{ a: `monitor "a" { description = "Monitor A" - + threshold { critical = 90 warning = 80 @@ -93,7 +94,7 @@ variable "b" { }`, b: `monitor "a" { description = "Monitor A" - + threshold { critical = 100 recovery = 10 @@ -105,21 +106,140 @@ variable "b" { threshold { critical = 100 - warning = 80 recovery = 10 + warning = 80 } } `, wantErr: nil, }, + { + name: "merge nested duplicate", + input: input{ + a: `module "b" { + + c = { + "foo" = { + value = 1 + } + } + } + `, + b: `module "b" { + + c = { + "bar" = { + value = 2 + } + } + } + `, + options: &hcl.MergeOptions{ + MergeMapKeys: true, + }, + }, + want: `module "b" { + c = { + "bar" = { + value = 2 + } + "foo" = { + value = 1 + } + } +} + +`, + }, + { + name: "merge complicated nested duplicate", + input: input{ + a: `module "b" { + test_map = { + string_key = "string_value" + "int_key" = 42 + var_key = var.value + float_key = 3.14 + bool_key = true + list_key = ["item1", "item2", 3, true] + nested_key = { + nested_string = "nested_value" + deep_nested = { + deep_key = "deep_value" + } + } + empty_map_key = {} + } +}`, + b: `module "b" { + test_map = { + string_key_2 = "string_value" + int_key_2 = 43 + float_key = 3.14 + bool_key = true + null_key = null + list_key = ["item1", "item2", 3, true, "new"] + nested_key = { + nested_string = "nested_value" + nested_int = 100 + deep_nested = { + deep_key = "deep_value" + new = "new" + } + } + empty_map_key = {} + "quoted.key" = "quoted_value" + mixed_key = { + inner_string = "inner_value" + inner_list = [1, 2, 3] + inner_map = { key = "value" } + } + } +}`, + options: &hcl.MergeOptions{ + MergeMapKeys: true, + }, + }, + want: `module "b" { + test_map = { + bool_key = true + empty_map_key = {} + float_key = 3.14 + "int_key" = 42 + int_key_2 = 43 + list_key = ["item1", "item2", 3, true, "new"] + mixed_key = { + inner_string = "inner_value" + inner_list = [1, 2, 3] + inner_map = { key = "value" } + } + nested_key = { + nested_string = "nested_value" + nested_int = 100 + deep_nested = { + deep_key = "deep_value" + new = "new" + } + } + null_key = null + "quoted.key" = "quoted_value" + string_key = "string_value" + string_key_2 = "string_value" + var_key = var.value + } +} + +`, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() - got, err := hcl.Merge(tc.input.a, tc.input.b) + merger := hcl.NewMerger(tc.input.options) + got, err := merger.Merge(tc.input.a, tc.input.b) assert.Equal(t, tc.want, got) if tc.wantErr != nil { diff --git a/main.go b/main.go index 3b12a0e..da24975 100644 --- a/main.go +++ b/main.go @@ -45,13 +45,37 @@ func main() { c := make(chan struct{}, 0) registerFn("merge", func(this js.Value, args []js.Value) (interface{}, error) { - if len(args) < 2 { - return nil, fmt.Errorf("Not enough arguments, expected (2)") + argCount := len(args) + if argCount < 2 || argCount > 3 { + return nil, fmt.Errorf("Invalid number of arguments, expected (2 - 3)") + } + + if args[0].Type() != js.TypeString { + return nil, fmt.Errorf("Invalid first argument type, expected string") + } + + if args[1].Type() != js.TypeString { + return nil, fmt.Errorf("Invalid second argument type, expected string") + } + + options := &hcl.MergeOptions{} + + if argCount == 3 { + arg2Type := args[2].Type() + if arg2Type != js.TypeObject && arg2Type != js.TypeUndefined { + return nil, fmt.Errorf("Invalid third argument type, expected optional object") + } + + if arg2Type == js.TypeObject { + options.MergeMapKeys = args[2].Get("mergeMapKeys").Bool() + } } aHclString := args[0].String() bHclString := args[1].String() - return hcl.Merge(aHclString, bHclString) + + hclmerger := hcl.NewMerger(options) + return hclmerger.Merge(aHclString, bHclString) }) <-c diff --git a/src/bridge.ts b/src/bridge.ts index ffb44c8..d40eadb 100644 --- a/src/bridge.ts +++ b/src/bridge.ts @@ -10,7 +10,7 @@ import { join } from "path"; import { gunzipSync } from "zlib"; interface GoBridge { - merge: (a: string, b: string) => Promise; + merge: (a: string, b: string, options?: object) => Promise; } declare const WebAssembly: any; diff --git a/src/hcl.test.ts b/src/hcl.test.ts index fcf61d1..047d57d 100644 --- a/src/hcl.test.ts +++ b/src/hcl.test.ts @@ -52,4 +52,74 @@ variable "b" { const actual = await merge(a, b); expect(actual).toBe(expected); }); + + it("should merge nested map keys", async () => { + const a = `variable "a" { + foo = "bar" + map1 = { + "key1" = { + numval = 1 + numval2 = 3 + varval = local.myvar + nested_map = { + "nested_num" = 100 + "nested_string" = "baz" + } + }, + } + map2 = { + "key2" = { + foo = "bar" + }, + } +} +`; + const b = `variable "a" { + bar = "baz" + map1 = { + "key1" = { + numval = 9 + } + } + map3 = { + "key3" = { + numval = 9 + nested_map = { + "nested_num" = 100 + "nested_string" = "baz" + } + }, + } +} +`; + + const expected = `variable "a" { + bar = "baz" + foo = "bar" + map1 = { + "key1" = { + numval = 9 + } + } + map2 = { + "key2" = { + foo = "bar" + }, + } + map3 = { + "key3" = { + numval = 9 + nested_map = { + "nested_num" = 100 + "nested_string" = "baz" + } + }, + } +} + +`; + const options = { mergeMapKeys: true }; + const actual = await merge(a, b, options); + expect(actual).toBe(expected); + }); }); diff --git a/src/hcl.ts b/src/hcl.ts index 4442060..32311e5 100644 --- a/src/hcl.ts +++ b/src/hcl.ts @@ -4,6 +4,18 @@ */ import { wasm } from "./bridge"; -export async function merge(a: string, b: string): Promise { - return await wasm.merge(a, b); +// MergeOptions is the options for the merge function +export class MergeOptions { + // mergeMapKeys merges map keys for hcl block attributes that are a map. + // note: this will not merge the values of the keys. + public mergeMapKeys: boolean = false; +} + +// merge merges two HCL strings +export async function merge( + a: string, + b: string, + options?: MergeOptions, +): Promise { + return await wasm.merge(a, b, options); }