From 58d00da716a6dc79a610a9f3de44eae0227e42a1 Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Mon, 16 Sep 2024 08:46:57 +0000 Subject: [PATCH 1/9] Nested blocks should not be merged by overrides See https://developer.hashicorp.com/terraform/language/files/override#merging-behavior - Within a top-level block, any nested blocks within an override block replace all blocks of the same type in the original block. Any block types that do not appear in the override block remain from the original block. - The contents of nested configuration blocks are not merged. --- terraform/module.go | 26 ++++++++++++- terraform/module_test.go | 84 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index e9f90eac4..c523ea4dc 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -152,10 +152,18 @@ func (m *Module) PartialContent(schema *hclext.BodySchema, ctx *Evaluator) (*hcl return content, diags } -// overrideBlocks changes the attributes in the passed primary blocks by override blocks recursively. +// overrideBlocks overrides the primary blocks passed with override blocks, +// following Terraform's merge behavior. +// https://developer.hashicorp.com/terraform/language/files/override#merging-behavior +// +// Note that this function returns the overwritten primary blocks +// but has side effects on the primary blocks. func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { dict := map[string]*hclext.Block{} for _, primary := range primaries { + // A top-level block in an override file merges with a block in a normal configuration file + // that has the same block header. + // The block header is the block type and any quoted labels that follow it. key := fmt.Sprintf("%s[%s]", primary.Type, strings.Join(primary.Labels, ",")) dict[key] = primary } @@ -163,10 +171,24 @@ func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { for _, override := range overrides { key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) if primary, exists := dict[key]; exists { + // Within a top-level block, an attribute argument within an override block + // replaces any argument of the same name in the original block. for name, attr := range override.Body.Attributes { primary.Body.Attributes[name] = attr } - primary.Body.Blocks = overrideBlocks(primary.Body.Blocks, override.Body.Blocks) + + // Within a top-level block, any nested blocks within an override block replace + // all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + for _, overrideBlock := range override.Body.Blocks { + overriddenBlocks := hclext.Blocks{} + for _, primaryBlock := range primary.Body.Blocks { + if primaryBlock.Type != overrideBlock.Type { + overriddenBlocks = append(overriddenBlocks, primaryBlock) + } + } + primary.Body.Blocks = append(overriddenBlocks, overrideBlock) + } } } diff --git a/terraform/module_test.go b/terraform/module_test.go index 20f472e42..50cb5bfcc 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -603,6 +603,7 @@ func Test_overrideBlocks(t *testing.T) { Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ "foo": &hclext.Attribute{Name: "bar"}, + "baz": &hclext.Attribute{Name: "baz"}, }, }, }, @@ -614,6 +615,7 @@ func Test_overrideBlocks(t *testing.T) { Attributes: hclext.Attributes{ "foo": &hclext.Attribute{Name: "bar"}, "bar": &hclext.Attribute{Name: "bar"}, + "baz": &hclext.Attribute{Name: "baz"}, }, }, }, @@ -651,6 +653,7 @@ func Test_overrideBlocks(t *testing.T) { Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ "baz": &hclext.Attribute{Name: "qux"}, + "bar": &hclext.Attribute{Name: "bar"}, }, }, }, @@ -668,11 +671,92 @@ func Test_overrideBlocks(t *testing.T) { Type: "nested", Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ + // The contents of nested configuration blocks are not merged. "baz": &hclext.Attribute{Name: "qux"}, + "bar": &hclext.Attribute{Name: "bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override multiple nested blocks", + Primaries: hclext.Blocks{ + { + Type: "resource", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "foo"}, + }, + }, + }, + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "bar": &hclext.Attribute{Name: "bar"}, + }, + }, + }, + { + Type: "other_nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "qux": &hclext.Attribute{Name: "qux"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "resource", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "baz": &hclext.Attribute{Name: "baz"}, + }, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "resource", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + // Any block types that do not appear in the override block remain from the original block. + { + Type: "other_nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ "qux": &hclext.Attribute{Name: "qux"}, }, }, }, + // override block replace all blocks of the same type in the original block. + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "baz": &hclext.Attribute{Name: "baz"}, + }, + }, + }, }, }, }, From 4c28333775f3b0bfae5522dc87933117e4072a20 Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Mon, 16 Sep 2024 10:05:39 +0000 Subject: [PATCH 2/9] Override lexicographical order See https://developer.hashicorp.com/terraform/language/files/override#merging-behavior If more than one override file defines the same top-level block, the overriding effect is compounded, with later blocks taking precedence over earlier blocks. Overrides are processed in order first by filename (in lexicographical order) and then by position in each file. Regarding the position in each file, no additional considerations are necessary since hclext.Blocks are already sorted. --- terraform/module.go | 14 +++++- terraform/module_test.go | 96 +++++++++++++++++++++++++++++----------- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index c523ea4dc..9a32e03d5 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -2,6 +2,9 @@ package terraform import ( "fmt" + "maps" + "slices" + "sort" "strings" "github.com/hashicorp/hcl/v2" @@ -138,8 +141,15 @@ func (m *Module) PartialContent(schema *hclext.BodySchema, ctx *Evaluator) (*hcl } content.Blocks = append(content.Blocks, c.Blocks...) } - for _, f := range m.overrides { - expanded, d := ctx.ExpandBlock(f.Body, schema) + + // If more than one override file defines the same top-level block, the overriding effect is compounded, + // with later blocks taking precedence over earlier blocks. + // Overrides are processed in order first by filename (in lexicographical order) + // and then by position in each file. + overrideFilenames := slices.Collect(maps.Keys(m.overrides)) + sort.Strings(overrideFilenames) + for _, filename := range overrideFilenames { + expanded, d := ctx.ExpandBlock(m.overrides[filename].Body, schema) diags = diags.Extend(d) c, d := hclext.PartialContent(expanded, schema) diags = diags.Extend(d) diff --git a/terraform/module_test.go b/terraform/module_test.go index 50cb5bfcc..f532abf87 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -236,19 +236,19 @@ resource "aws_instance" "bar" { Type: "resource", Labels: []string{"aws_instance", "foo"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main1.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main1.tf", Start: hcl.Pos{Line: 3}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main1.tf"}, + DefRange: hcl.Range{Filename: "main1.tf", Start: hcl.Pos{Line: 2}}, }, { Type: "resource", Labels: []string{"aws_instance", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main2.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main2.tf", Start: hcl.Pos{Line: 3}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main2.tf"}, + DefRange: hcl.Range{Filename: "main2.tf", Start: hcl.Pos{Line: 2}}, }, }, }, @@ -281,10 +281,53 @@ resource "aws_instance" "foo" { Type: "resource", Labels: []string{"aws_instance", "foo"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main_override.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main_override.tf", Start: hcl.Pos{Line: 3}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 2}}, + }, + }, + }, + }, + { + name: "overrides by multiple files/blocks", + files: map[string]string{ + "main.tf": ` +resource "aws_instance" "foo" { + instance_type = "t2.micro" +}`, + "main1_override.tf": ` +resource "aws_instance" "foo" { + instance_type = "m5.2xlarge" +}`, + "main2_override.tf": ` +resource "aws_instance" "foo" { + instance_type = "m5.4xlarge" +} +resource "aws_instance" "foo" { + instance_type = "m5.8xlarge" +}`, + }, + schema: &hclext.BodySchema{ + Attributes: []hclext.AttributeSchema{{Name: "foo"}}, + Blocks: []hclext.BlockSchema{ + { + Type: "resource", + LabelNames: []string{"type", "name"}, + Body: &hclext.BodySchema{Attributes: []hclext.AttributeSchema{{Name: "instance_type"}}}, + }, + }, + }, + want: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"aws_instance", "foo"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main2_override.tf", Start: hcl.Pos{Line: 6}}}}, + Blocks: hclext.Blocks{}, + }, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 2}}, }, }, }, @@ -312,12 +355,12 @@ locals { Type: "locals", Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ - "foo": &hclext.Attribute{Name: "foo", Range: hcl.Range{Filename: "main.tf"}}, - "bar": &hclext.Attribute{Name: "bar", Range: hcl.Range{Filename: "main.tf"}}, + "foo": &hclext.Attribute{Name: "foo", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 3}}}, + "bar": &hclext.Attribute{Name: "bar", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 4}}}, }, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 2}}, }, }, }, @@ -351,19 +394,19 @@ resource "aws_instance" "bar" { Type: "resource", Labels: []string{"aws_instance", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 6}}, }, { Type: "resource", Labels: []string{"aws_instance", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 6}}, }, }, }, @@ -397,19 +440,19 @@ module "bar" { Type: "module", Labels: []string{"bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 6}}, }, { Type: "module", Labels: []string{"bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"instance_type": &hclext.Attribute{Name: "instance_type", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 6}}, }, }, }, @@ -461,14 +504,14 @@ resource "aws_instance" "bar" { { Type: "ebs_block_device", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 4}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 3}}, }, }, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 2}}, }, { Type: "resource", @@ -479,22 +522,22 @@ resource "aws_instance" "bar" { { Type: "ebs_block_device", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 11}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}, }, { Type: "ebs_block_device", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf"}}}, + Attributes: hclext.Attributes{"volume_size": &hclext.Attribute{Name: "volume_size", Range: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 11}}}}, Blocks: hclext.Blocks{}, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 8}}, }, }, }, - DefRange: hcl.Range{Filename: "main.tf"}, + DefRange: hcl.Range{Filename: "main.tf", Start: hcl.Pos{Line: 7}}, }, }, }, @@ -539,7 +582,8 @@ resource "aws_instance" "bar" { opts := cmp.Options{ cmpopts.IgnoreFields(hclext.Block{}, "TypeRange", "LabelRanges"), cmpopts.IgnoreFields(hclext.Attribute{}, "Expr", "NameRange"), - cmpopts.IgnoreFields(hcl.Range{}, "Start", "End"), + cmpopts.IgnoreFields(hcl.Range{}, "End"), + cmpopts.IgnoreFields(hcl.Pos{}, "Column", "Byte"), cmpopts.SortSlices(func(i, j *hclext.Block) bool { return i.DefRange.String() < j.DefRange.String() }), From 30020bca55694385c130bc605a5fe18661fa2311 Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Mon, 23 Sep 2024 16:19:45 +0000 Subject: [PATCH 3/9] Override "terraform" blocks See https://developer.hashicorp.com/terraform/language/files/override#merging-terraform-blocks The settings within terraform blocks are considered individually when merging. If the required_providers argument is set, its value is merged on an element-by-element basis, which allows an override block to adjust the constraint for a single provider without affecting the constraints for other providers. In both the required_version and required_providers settings, each override constraint entirely replaces the constraints for the same component in the original block. If both the base block and the override block both set required_version then the constraints in the base block are entirely ignored. The presence of a block defining a backend (either cloud or backend) in an override file always takes precedence over a block defining a backend in the original configuration. That is, if a cloud block is set within the original configuration and a backend block is set in the override file, Terraform will use the backend block specified in the override file upon merging. Similarly, if a backend block is set within the original configuration and a cloud block is set in the override file, Terraform will use the cloud block specified in the override file upon merging. --- terraform/module.go | 132 +++++++++-- terraform/module_test.go | 477 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 587 insertions(+), 22 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index 9a32e03d5..bf6b1e8a1 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -169,40 +169,128 @@ func (m *Module) PartialContent(schema *hclext.BodySchema, ctx *Evaluator) (*hcl // Note that this function returns the overwritten primary blocks // but has side effects on the primary blocks. func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { - dict := map[string]*hclext.Block{} + dict := map[string]hclext.Blocks{} for _, primary := range primaries { - // A top-level block in an override file merges with a block in a normal configuration file - // that has the same block header. - // The block header is the block type and any quoted labels that follow it. - key := fmt.Sprintf("%s[%s]", primary.Type, strings.Join(primary.Labels, ",")) - dict[key] = primary + switch primary.Type { + case "terraform": + // The "terraform" blocks are allowed to be declared multiple times. + dict[primary.Type] = append(dict[primary.Type], primary) + + default: + // A top-level block in an override file merges with a block in a normal configuration file + // that has the same block header. + // The block header is the block type and any quoted labels that follow it. + key := fmt.Sprintf("%s[%s]", primary.Type, strings.Join(primary.Labels, ",")) + dict[key] = hclext.Blocks{primary} + } } + newPrimaries := hclext.Blocks{} for _, override := range overrides { - key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) - if primary, exists := dict[key]; exists { - // Within a top-level block, an attribute argument within an override block - // replaces any argument of the same name in the original block. - for name, attr := range override.Body.Attributes { - primary.Body.Attributes[name] = attr + switch override.Type { + case "terraform": + // Any required_providers that were not used for overrides will be added, + // so we will track whether they were used for overrides or not. + overrideRequiredProviders := override.Body.Blocks.ByType()["required_providers"] + + for _, primary := range dict[override.Type] { + // In both the required_version and required_providers settings, + // each override constraint entirely replaces the constraints for + // the same component in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } + + for _, overrideInnerBlock := range override.Body.Blocks { + switch overrideInnerBlock.Type { + case "required_providers": + // If the required_providers argument is set, its value is merged on an element-by-element basis + for _, primaryInnerBlock := range primary.Body.Blocks { + if primaryInnerBlock.Type == "required_providers" { + for name, attr := range overrideInnerBlock.Body.Attributes { + if _, exists := primaryInnerBlock.Body.Attributes[name]; exists { + primaryInnerBlock.Body.Attributes[name] = attr + // Remove the required provider that was used to override. + for _, requiredProvider := range overrideRequiredProviders { + delete(requiredProvider.Body.Attributes, name) + } + } + } + } + } + + case "cloud", "backend": + // The presence of a block defining a backend (either cloud or backend) in an override file + // always takes precedence over a block defining a backend in the original configuration. + newInnerBlocks := hclext.Blocks{} + for _, primaryInnerBlock := range primary.Body.Blocks { + if primaryInnerBlock.Type != "cloud" && primaryInnerBlock.Type != "backend" { + newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) + } + } + primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) + + default: + newInnerBlocks := hclext.Blocks{} + for _, primaryInnerBlock := range primary.Body.Blocks { + if primaryInnerBlock.Type != overrideInnerBlock.Type { + newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) + } + } + primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) + } + } + } + + // Any remaining required providers that aren't overridden will be added as a new block. + newRequiredProviders := hclext.Blocks{} + for _, requiredProvider := range overrideRequiredProviders { + if len(requiredProvider.Body.Attributes) > 0 { + newRequiredProviders = append(newRequiredProviders, requiredProvider) + } } + if len(newRequiredProviders) > 0 { + newPrimaries = append(newPrimaries, &hclext.Block{ + Type: override.Type, + Labels: override.Labels, + Body: &hclext.BodyContent{ + Blocks: newRequiredProviders, + }, + DefRange: override.DefRange, + TypeRange: override.TypeRange, + LabelRanges: override.LabelRanges, + }) + } + + default: + key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) + if primaries, exists := dict[key]; exists { + // The general rule, duplicated blocks are not allowed. + primary := primaries[0] + + // Within a top-level block, an attribute argument within an override block + // replaces any argument of the same name in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } - // Within a top-level block, any nested blocks within an override block replace - // all blocks of the same type in the original block. - // Any block types that do not appear in the override block remain from the original block. - for _, overrideBlock := range override.Body.Blocks { - overriddenBlocks := hclext.Blocks{} - for _, primaryBlock := range primary.Body.Blocks { - if primaryBlock.Type != overrideBlock.Type { - overriddenBlocks = append(overriddenBlocks, primaryBlock) + // Within a top-level block, any nested blocks within an override block replace + // all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + for _, overrideInnerBlock := range override.Body.Blocks { + newInnerBlocks := hclext.Blocks{} + for _, primaryInnerBlock := range primary.Body.Blocks { + if primaryInnerBlock.Type != overrideInnerBlock.Type { + newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) + } } + primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) } - primary.Body.Blocks = append(overriddenBlocks, overrideBlock) } } } - return primaries + return append(primaries, newPrimaries...) } var moduleSchema = &hclext.BodySchema{ diff --git a/terraform/module_test.go b/terraform/module_test.go index f532abf87..10f5ea3ed 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -806,6 +806,483 @@ func Test_overrideBlocks(t *testing.T) { }, }, }, + { + Name: "no override multiple required_version", + Primaries: hclext.Blocks{ + // The "terraform" blocks are allowed to be declared multiple times. + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version1"}}, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version2"}}, + }, + }, + }, + Overrides: hclext.Blocks{}, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version1"}}, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version2"}}, + }, + }, + }, + }, + { + Name: "override multiple required_version", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version1"}}, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version2"}}, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version3"}}, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version4"}}, + }, + }, + }, + Want: hclext.Blocks{ + // In both the required_version and required_providers settings, + // each override constraint entirely replaces the constraints for the same component in the original block. + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version4"}}, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version4"}}, + }, + }, + }, + }, + { + Name: "no override required_providers", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{}, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + // If the required_providers argument is set, its value is merged on an element-by-element basis + Name: "override required_providers", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "assert": &hclext.Attribute{Name: "assert"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google": &hclext.Attribute{Name: "google2"}, + "time": &hclext.Attribute{Name: "time"}, + }, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "google": &hclext.Attribute{Name: "google2"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + // Blocks not present in the primaries are added + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "assert": &hclext.Attribute{Name: "assert"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "time": &hclext.Attribute{Name: "time"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override backend", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"local"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"path": &hclext.Attribute{Name: "path"}}, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"remote"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"host": &hclext.Attribute{Name: "host"}}, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"remote"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"host": &hclext.Attribute{Name: "host"}}, + }, + }, + }, + }, + }, + }, + }, + { + // The presence of a block defining a backend (either cloud or backend) in an override file + // always takes precedence over a block defining a backend in the original configuration + Name: "override backend by cloud", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"local"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"path": &hclext.Attribute{Name: "path"}}, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "cloud", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"organization": &hclext.Attribute{Name: "organization"}}, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "cloud", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"organization": &hclext.Attribute{Name: "organization"}}, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override cloud by backend", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "cloud", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"organization": &hclext.Attribute{Name: "organization"}}, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"remote"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"host": &hclext.Attribute{Name: "host"}}, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "backend", + Labels: []string{"remote"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"host": &hclext.Attribute{Name: "host"}}, + }, + }, + }, + }, + }, + }, + }, } for _, test := range tests { From c6214bd0a6d21ef38a9fed44816adb700cb1d7e0 Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sat, 5 Oct 2024 09:41:13 +0000 Subject: [PATCH 4/9] Override "locals" blocks See also https://developer.hashicorp.com/terraform/language/files/override#merging-locals-blocks Each locals block defines a number of named values. Overrides are applied on a value-by-value basis, ignoring which locals block they are defined in. --- terraform/module.go | 36 ++++++++++++++++++++++-- terraform/module_test.go | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index bf6b1e8a1..b154c9295 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -172,8 +172,8 @@ func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { dict := map[string]hclext.Blocks{} for _, primary := range primaries { switch primary.Type { - case "terraform": - // The "terraform" blocks are allowed to be declared multiple times. + case "locals", "terraform": + // The "locals", "terraform" blocks are allowed to be declared multiple times. dict[primary.Type] = append(dict[primary.Type], primary) default: @@ -188,6 +188,38 @@ func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { newPrimaries := hclext.Blocks{} for _, override := range overrides { switch override.Type { + case "locals": + // Tracks locals ​​that were not used to override. + remainLocals := hclext.Attributes{} + for name, attr := range override.Body.Attributes { + remainLocals[name] = attr + } + + // Each locals block defines a number of named values. + // Overrides are applied on a value-by-value basis, ignoring which locals block they are defined in. + for _, primary := range dict[override.Type] { + for name, attr := range override.Body.Attributes { + if _, exists := primary.Body.Attributes[name]; exists { + primary.Body.Attributes[name] = attr + delete(remainLocals, name) + } + } + } + + // Any remaining locals that aren't overridden will be added as a new block. + if len(remainLocals) > 0 { + newPrimaries = append(newPrimaries, &hclext.Block{ + Type: override.Type, + Labels: override.Labels, + Body: &hclext.BodyContent{ + Attributes: remainLocals, + }, + DefRange: override.DefRange, + TypeRange: override.TypeRange, + LabelRanges: override.LabelRanges, + }) + } + case "terraform": // Any required_providers that were not used for overrides will be added, // so we will track whether they were used for overrides or not. diff --git a/terraform/module_test.go b/terraform/module_test.go index 10f5ea3ed..42ca638a7 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -806,6 +806,65 @@ func Test_overrideBlocks(t *testing.T) { }, }, }, + { + Name: "override locals", + Primaries: hclext.Blocks{ + // The "locals" blocks are allowed to be declared multiple times. + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar"}}, + }, + }, + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz"}, "qux": &hclext.Attribute{Name: "qux"}}, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "foo2": &hclext.Attribute{Name: "foo2"}}, + }, + }, + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"bar": &hclext.Attribute{Name: "bar2"}, "qux2": &hclext.Attribute{Name: "qux2"}}, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar2"}}, + }, + }, + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "qux": &hclext.Attribute{Name: "qux"}}, + }, + }, + // Locals not present in the primaries are added + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo2": &hclext.Attribute{Name: "foo2"}}, + }, + }, + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"qux2": &hclext.Attribute{Name: "qux2"}}, + }, + }, + }, + }, { Name: "no override multiple required_version", Primaries: hclext.Blocks{ From b64e65a45503ae5799a73eb6fc4f7a67135b75fe Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sat, 5 Oct 2024 10:22:05 +0000 Subject: [PATCH 5/9] Override "resource" blocks See https://developer.hashicorp.com/terraform/language/files/override#merging-resource-and-data-blocks Within a resource block, the contents of any lifecycle nested block are merged on an argument-by-argument basis. For example, if an override block sets only the create_before_destroy argument then any ignore_changes argument in the original block will be preserved. If an overriding resource block contains one or more provisioner blocks then any provisioner blocks in the original block are ignored. If an overriding resource block contains a connection block then it completely overrides any connection block present in the original block. --- terraform/module.go | 33 ++++++++++++++++ terraform/module_test.go | 85 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/terraform/module.go b/terraform/module.go index b154c9295..911d57da7 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -188,6 +188,39 @@ func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { newPrimaries := hclext.Blocks{} for _, override := range overrides { switch override.Type { + case "resource": + key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) + if primaries, exists := dict[key]; exists { + // Duplicated blocks are not allowed. + primary := primaries[0] + + // Within a top-level block, an attribute argument within an override block + // replaces any argument of the same name in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } + + // Within a top-level block, any nested blocks within an override block replace + // all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + for _, overrideInnerBlock := range override.Body.Blocks { + newInnerBlocks := hclext.Blocks{} + for _, primaryInnerBlock := range primary.Body.Blocks { + if primaryInnerBlock.Type != overrideInnerBlock.Type { + newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) + } else if overrideInnerBlock.Type == "lifecycle" { + // Within a resource block, the contents of any lifecycle nested block are merged on an argument-by-argument basis. + for name, attr := range overrideInnerBlock.Body.Attributes { + primaryInnerBlock.Body.Attributes[name] = attr + } + // Can't override nested blocks + newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) + } + } + primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) + } + } + case "locals": // Tracks locals ​​that were not used to override. remainLocals := hclext.Attributes{} diff --git a/terraform/module_test.go b/terraform/module_test.go index 42ca638a7..f06143676 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -806,6 +806,91 @@ func Test_overrideBlocks(t *testing.T) { }, }, }, + { + Name: "override lifecycle/provisioner/connection", + Primaries: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"random_id", "server"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}}, + Blocks: hclext.Blocks{ + { + Type: "provisioner", + Labels: []string{"local-exec"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"command": &hclext.Attribute{Name: "command"}}, + }, + }, + { + Type: "provisioner", + Labels: []string{"file"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"content": &hclext.Attribute{Name: "content"}}, + }, + }, + { + Type: "connection", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"type": &hclext.Attribute{Name: "type"}}, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"random_id", "server"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"ignore_changes": &hclext.Attribute{Name: "ignore_changes"}, "create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}}, + Blocks: hclext.Blocks{ + { + Type: "provisioner", + Labels: []string{"remote-exec"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"inline": &hclext.Attribute{Name: "inline"}}, + }, + }, + { + Type: "connection", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"user": &hclext.Attribute{Name: "user"}}, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"random_id", "server"}, + Body: &hclext.BodyContent{ + // the contents of any lifecycle nested block are merged on an argument-by-argument basis. + Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}, "ignore_changes": &hclext.Attribute{Name: "ignore_changes"}}, + // If an overriding resource block contains one or more provisioner blocks then any provisioner blocks in the original block are ignored. + // If an overriding resource block contains a connection block then it completely overrides any connection block present in the original block. + Blocks: hclext.Blocks{ + { + Type: "provisioner", + Labels: []string{"remote-exec"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"inline": &hclext.Attribute{Name: "inline"}}, + }, + }, + { + Type: "connection", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"user": &hclext.Attribute{Name: "user"}}, + }, + }, + }, + }, + }, + }, + }, { Name: "override locals", Primaries: hclext.Blocks{ From c690111082a4aa38ff7aacc4523bfd66792f1f7c Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sat, 5 Oct 2024 15:56:35 +0000 Subject: [PATCH 6/9] refactor --- terraform/module.go | 376 +++++++++++++++++++++++---------------- terraform/module_test.go | 323 +++++++++++++++++---------------- 2 files changed, 395 insertions(+), 304 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index 911d57da7..27557b85f 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -162,200 +162,268 @@ func (m *Module) PartialContent(schema *hclext.BodySchema, ctx *Evaluator) (*hcl return content, diags } +// blockAddr returns an identifier of the given block (e.g. "resource.aws_instance.main"). +// This is used for overrides, which use the block type and labels as identifiers. +func blockAddr(b *hclext.Block) string { + if len(b.Labels) > 0 { + return fmt.Sprintf("%s.%s", b.Type, strings.Join(b.Labels, ".")) + } + return b.Type +} + // overrideBlocks overrides the primary blocks passed with override blocks, // following Terraform's merge behavior. // https://developer.hashicorp.com/terraform/language/files/override#merging-behavior // // Note that this function returns the overwritten primary blocks -// but has side effects on the primary blocks. +// but has side effects on the primary blocks and the overrides blocks. func overrideBlocks(primaries, overrides hclext.Blocks) hclext.Blocks { - dict := map[string]hclext.Blocks{} + overridesByAddr := map[string]hclext.Blocks{} for _, primary := range primaries { - switch primary.Type { - case "locals", "terraform": - // The "locals", "terraform" blocks are allowed to be declared multiple times. - dict[primary.Type] = append(dict[primary.Type], primary) - - default: - // A top-level block in an override file merges with a block in a normal configuration file - // that has the same block header. - // The block header is the block type and any quoted labels that follow it. - key := fmt.Sprintf("%s[%s]", primary.Type, strings.Join(primary.Labels, ",")) - dict[key] = hclext.Blocks{primary} - } + addr := blockAddr(primary) + overridesByAddr[addr] = append(overridesByAddr[addr], primary) } + // The block containing elements that cannot be overridden will be added as new primaries. + // e.g. Local values ​not present in the primaries. + // + // Intuitively, if there is always only one corresponding block, + // it is hard to imagine a case where it cannot be overwritten, + // but since "locals" and "terraform" blocks can be declared multiple times, + // please note that the block to be overwritten cannot be uniquely determined. newPrimaries := hclext.Blocks{} + for _, override := range overrides { + addr := blockAddr(override) + switch override.Type { case "resource": - key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) - if primaries, exists := dict[key]; exists { - // Duplicated blocks are not allowed. - primary := primaries[0] + if primaries, exists := overridesByAddr[addr]; exists { + // Duplicate resource blocks are not allowed. + overrideResourceBlock(primaries[0], override) + } - // Within a top-level block, an attribute argument within an override block - // replaces any argument of the same name in the original block. - for name, attr := range override.Body.Attributes { - primary.Body.Attributes[name] = attr - } + // The "data" block is the same as generic block except for "depends_on". + // The "depends_on" arguments should not be merged, but Terraform will throw an error about it, + // so we won't take that into consideration here. - // Within a top-level block, any nested blocks within an override block replace - // all blocks of the same type in the original block. - // Any block types that do not appear in the override block remain from the original block. - for _, overrideInnerBlock := range override.Body.Blocks { - newInnerBlocks := hclext.Blocks{} - for _, primaryInnerBlock := range primary.Body.Blocks { - if primaryInnerBlock.Type != overrideInnerBlock.Type { - newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) - } else if overrideInnerBlock.Type == "lifecycle" { - // Within a resource block, the contents of any lifecycle nested block are merged on an argument-by-argument basis. - for name, attr := range overrideInnerBlock.Body.Attributes { - primaryInnerBlock.Body.Attributes[name] = attr - } - // Can't override nested blocks - newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) - } - } - primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) - } - } + // The "variable" block is the same as generic block except for "type" and "default". + // Conversion of default values ​​is done during evaluation and is not considered here. + + // The "output" block is the same as generic block except for "depends_on". case "locals": - // Tracks locals ​​that were not used to override. - remainLocals := hclext.Attributes{} - for name, attr := range override.Body.Attributes { - remainLocals[name] = attr + remain := overrideLocalBlocks(overridesByAddr[addr], override) + if remain != nil { + newPrimaries = append(newPrimaries, remain) } - // Each locals block defines a number of named values. - // Overrides are applied on a value-by-value basis, ignoring which locals block they are defined in. - for _, primary := range dict[override.Type] { - for name, attr := range override.Body.Attributes { - if _, exists := primary.Body.Attributes[name]; exists { - primary.Body.Attributes[name] = attr - delete(remainLocals, name) - } - } + case "terraform": + remain := overrideTerraformBlocks(overridesByAddr[addr], override) + if remain != nil { + newPrimaries = append(newPrimaries, remain) } - // Any remaining locals that aren't overridden will be added as a new block. - if len(remainLocals) > 0 { - newPrimaries = append(newPrimaries, &hclext.Block{ - Type: override.Type, - Labels: override.Labels, - Body: &hclext.BodyContent{ - Attributes: remainLocals, - }, - DefRange: override.DefRange, - TypeRange: override.TypeRange, - LabelRanges: override.LabelRanges, - }) + default: + if primaries, exists := overridesByAddr[addr]; exists { + // The general rule, duplicated blocks are not allowed. + overrideGenericBlock(primaries[0], override) } + } + } - case "terraform": - // Any required_providers that were not used for overrides will be added, - // so we will track whether they were used for overrides or not. - overrideRequiredProviders := override.Body.Blocks.ByType()["required_providers"] - - for _, primary := range dict[override.Type] { - // In both the required_version and required_providers settings, - // each override constraint entirely replaces the constraints for - // the same component in the original block. + return append(primaries, newPrimaries...) +} + +// overrideResourceBlock overrides "resource" block +// https://developer.hashicorp.com/terraform/language/files/override#merging-resource-and-data-blocks +// +// The "depends_on" arguments should not be merged, but Terraform will throw an error about it, +// so we won't take that into consideration here. +// +// This function modifies the given primary directly. +func overrideResourceBlock(primary, override *hclext.Block) { + // An attribute argument within an override block + // replaces any argument of the same name in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } + + // Exit early if blocks are empty. + if len(primary.Body.Blocks) == 0 && len(override.Body.Blocks) == 0 { + return + } + overridesByType := override.Body.Blocks.ByType() + + // Any nested blocks within an override block replace all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + primary.Body.Blocks = filterBlocks(primary.Body.Blocks, func(p *hclext.Block) bool { + overrides, exists := overridesByType[p.Type] + if !exists { + return true + } + + if p.Type == "lifecycle" { + // Contents of any lifecycle nested block are merged on an argument-by-argument basis. + // Can't override nested blocks like precondition/postcondition. + for _, override := range overrides { for name, attr := range override.Body.Attributes { - primary.Body.Attributes[name] = attr + p.Body.Attributes[name] = attr } + } + return true + } - for _, overrideInnerBlock := range override.Body.Blocks { - switch overrideInnerBlock.Type { - case "required_providers": - // If the required_providers argument is set, its value is merged on an element-by-element basis - for _, primaryInnerBlock := range primary.Body.Blocks { - if primaryInnerBlock.Type == "required_providers" { - for name, attr := range overrideInnerBlock.Body.Attributes { - if _, exists := primaryInnerBlock.Body.Attributes[name]; exists { - primaryInnerBlock.Body.Attributes[name] = attr - // Remove the required provider that was used to override. - for _, requiredProvider := range overrideRequiredProviders { - delete(requiredProvider.Body.Attributes, name) - } - } - } - } - } + return false + }) + primary.Body.Blocks = append( + primary.Body.Blocks, + filterBlocks(override.Body.Blocks, func(b *hclext.Block) bool { return b.Type != "lifecycle" })..., + ) +} - case "cloud", "backend": - // The presence of a block defining a backend (either cloud or backend) in an override file - // always takes precedence over a block defining a backend in the original configuration. - newInnerBlocks := hclext.Blocks{} - for _, primaryInnerBlock := range primary.Body.Blocks { - if primaryInnerBlock.Type != "cloud" && primaryInnerBlock.Type != "backend" { - newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) - } - } - primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) +// overrideLocalBlocks overrides "local" blocks +// https://developer.hashicorp.com/terraform/language/files/override#merging-locals-blocks +// +// This function modifies the given primaries directly. +// If the given override contains elements that cannot be overridden, (e.g. new local values) +// it is returned to the caller with only those elements remaining. +// This operation modifies the given override directly. +func overrideLocalBlocks(primaries hclext.Blocks, override *hclext.Block) *hclext.Block { + // Tracks locals ​​that were not used to override. + remains := hclext.Attributes{} + for name, attr := range override.Body.Attributes { + remains[name] = attr + } + + // Overrides are applied on a value-by-value basis, ignoring which locals block they are defined in. + for _, primary := range primaries { + for name, attr := range override.Body.Attributes { + if _, exists := primary.Body.Attributes[name]; exists { + primary.Body.Attributes[name] = attr + delete(remains, name) + } + } + } + + // Any remaining locals that aren't overridden will be added as a new block. + if len(remains) > 0 { + override.Body.Attributes = remains + return override + } + return nil +} - default: - newInnerBlocks := hclext.Blocks{} - for _, primaryInnerBlock := range primary.Body.Blocks { - if primaryInnerBlock.Type != overrideInnerBlock.Type { - newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) +// overrideTerraformBlocks overrides "terraform" blocks +// https://developer.hashicorp.com/terraform/language/files/override#merging-terraform-blocks +// +// This function modifies the given primaries directly. +// If the given override contains elements that cannot be overridden, (e.g. new required providers) +// it is returned to the caller with only those elements remaining. +// This operation modifies the given override directly. +func overrideTerraformBlocks(primaries hclext.Blocks, override *hclext.Block) *hclext.Block { + // Tracks required providers ​​that were not used to override. + remainRequiredProviders := override.Body.Blocks.ByType()["required_providers"] + + for _, primary := range primaries { + // An attribute argument within an override block + // replaces any argument of the same name in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } + + // Exit early if blocks are empty. + if len(primary.Body.Blocks) == 0 && len(override.Body.Blocks) == 0 { + continue + } + overridesByType := override.Body.Blocks.ByType() + + // Any nested blocks within an override block replace all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + primary.Body.Blocks = filterBlocks(primary.Body.Blocks, func(p *hclext.Block) bool { + switch p.Type { + case "required_providers": + // If the required_providers argument is set, its value is merged on an element-by-element basis + for _, override := range overridesByType[p.Type] { + for name, attr := range override.Body.Attributes { + if _, exists := p.Body.Attributes[name]; exists { + p.Body.Attributes[name] = attr + for _, remain := range remainRequiredProviders { + delete(remain.Body.Attributes, name) } } - primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) } } - } + return true - // Any remaining required providers that aren't overridden will be added as a new block. - newRequiredProviders := hclext.Blocks{} - for _, requiredProvider := range overrideRequiredProviders { - if len(requiredProvider.Body.Attributes) > 0 { - newRequiredProviders = append(newRequiredProviders, requiredProvider) + case "cloud", "backend": + // The presence of a block defining a backend (either cloud or backend) in an override file + // always takes precedence over a block defining a backend in the original configuration. + if _, exists := overridesByType["cloud"]; exists { + return false } + if _, exists := overridesByType["backend"]; exists { + return false + } + return true + + default: + _, exists := overridesByType[p.Type] + return !exists } - if len(newRequiredProviders) > 0 { - newPrimaries = append(newPrimaries, &hclext.Block{ - Type: override.Type, - Labels: override.Labels, - Body: &hclext.BodyContent{ - Blocks: newRequiredProviders, - }, - DefRange: override.DefRange, - TypeRange: override.TypeRange, - LabelRanges: override.LabelRanges, - }) - } + }) + primary.Body.Blocks = append( + primary.Body.Blocks, + filterBlocks(override.Body.Blocks, func(b *hclext.Block) bool { return b.Type != "required_providers" })..., + ) + } - default: - key := fmt.Sprintf("%s[%s]", override.Type, strings.Join(override.Labels, ",")) - if primaries, exists := dict[key]; exists { - // The general rule, duplicated blocks are not allowed. - primary := primaries[0] + // Any remaining required providers that aren't overridden will be added as a new block. + remainRequiredProviders = filterBlocks(remainRequiredProviders, func(b *hclext.Block) bool { + return len(b.Body.Attributes) > 0 + }) + if len(remainRequiredProviders) > 0 { + override.Body.Blocks = remainRequiredProviders + return override + } + return nil +} - // Within a top-level block, an attribute argument within an override block - // replaces any argument of the same name in the original block. - for name, attr := range override.Body.Attributes { - primary.Body.Attributes[name] = attr - } +// overrideGenericBlock overrides generic blocks. +// https://developer.hashicorp.com/terraform/language/files/override#merging-behavior +// +// Except for a few special blocks, most blocks are overridden by this rule. +// This function modifies the given primary directly. +func overrideGenericBlock(primary, override *hclext.Block) { + // An attribute argument within an override block + // replaces any argument of the same name in the original block. + for name, attr := range override.Body.Attributes { + primary.Body.Attributes[name] = attr + } - // Within a top-level block, any nested blocks within an override block replace - // all blocks of the same type in the original block. - // Any block types that do not appear in the override block remain from the original block. - for _, overrideInnerBlock := range override.Body.Blocks { - newInnerBlocks := hclext.Blocks{} - for _, primaryInnerBlock := range primary.Body.Blocks { - if primaryInnerBlock.Type != overrideInnerBlock.Type { - newInnerBlocks = append(newInnerBlocks, primaryInnerBlock) - } - } - primary.Body.Blocks = append(newInnerBlocks, overrideInnerBlock) - } - } - } + // Exit early if blocks are empty. + if len(primary.Body.Blocks) == 0 && len(override.Body.Blocks) == 0 { + return } + overridesByType := override.Body.Blocks.ByType() + + // Any nested blocks within an override block replace all blocks of the same type in the original block. + // Any block types that do not appear in the override block remain from the original block. + primary.Body.Blocks = filterBlocks(primary.Body.Blocks, func(p *hclext.Block) bool { + _, exists := overridesByType[p.Type] + return !exists + }) + primary.Body.Blocks = append(primary.Body.Blocks, override.Body.Blocks...) +} - return append(primaries, newPrimaries...) +func filterBlocks(in hclext.Blocks, fn func(*hclext.Block) bool) hclext.Blocks { + out := hclext.Blocks{} + for _, block := range in { + if fn(block) { + out = append(out, block) + } + } + return out } var moduleSchema = &hclext.BodySchema{ diff --git a/terraform/module_test.go b/terraform/module_test.go index f06143676..4a8bda5a8 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -612,7 +612,8 @@ func Test_overrideBlocks(t *testing.T) { Name: "no override", Primaries: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}}, }, @@ -621,7 +622,38 @@ func Test_overrideBlocks(t *testing.T) { Overrides: hclext.Blocks{}, Want: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}}, + }, + }, + }, + }, + { + Name: "no override because resources are difference", + Primaries: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"foo", "bar"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}}, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"baz", "qux"}, + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo2"}}, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}}, }, @@ -632,7 +664,8 @@ func Test_overrideBlocks(t *testing.T) { Name: "override", Primaries: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ "foo": &hclext.Attribute{Name: "foo"}, @@ -643,7 +676,8 @@ func Test_overrideBlocks(t *testing.T) { }, Overrides: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ "foo": &hclext.Attribute{Name: "bar"}, @@ -654,7 +688,8 @@ func Test_overrideBlocks(t *testing.T) { }, Want: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{ "foo": &hclext.Attribute{Name: "bar"}, @@ -669,7 +704,8 @@ func Test_overrideBlocks(t *testing.T) { Name: "override nested blocks", Primaries: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}}, Blocks: hclext.Blocks{ @@ -688,7 +724,8 @@ func Test_overrideBlocks(t *testing.T) { }, Overrides: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "bar"}}, Blocks: hclext.Blocks{ @@ -707,7 +744,8 @@ func Test_overrideBlocks(t *testing.T) { }, Want: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "bar"}}, Blocks: hclext.Blocks{ @@ -730,7 +768,8 @@ func Test_overrideBlocks(t *testing.T) { Name: "override multiple nested blocks", Primaries: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Blocks: hclext.Blocks{ { @@ -763,7 +802,8 @@ func Test_overrideBlocks(t *testing.T) { }, Overrides: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Blocks: hclext.Blocks{ { @@ -780,7 +820,8 @@ func Test_overrideBlocks(t *testing.T) { }, Want: hclext.Blocks{ { - Type: "resource", + Type: "resource", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ Blocks: hclext.Blocks{ // Any block types that do not appear in the override block remain from the original block. @@ -811,10 +852,15 @@ func Test_overrideBlocks(t *testing.T) { Primaries: hclext.Blocks{ { Type: "resource", - Labels: []string{"random_id", "server"}, + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}}, Blocks: hclext.Blocks{ + { + Type: "lifecycle", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}}, + }, + }, { Type: "provisioner", Labels: []string{"local-exec"}, @@ -842,10 +888,15 @@ func Test_overrideBlocks(t *testing.T) { Overrides: hclext.Blocks{ { Type: "resource", - Labels: []string{"random_id", "server"}, + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"ignore_changes": &hclext.Attribute{Name: "ignore_changes"}, "create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}}, Blocks: hclext.Blocks{ + { + Type: "lifecycle", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"ignore_changes": &hclext.Attribute{Name: "ignore_changes"}, "create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}}, + }, + }, { Type: "provisioner", Labels: []string{"remote-exec"}, @@ -866,13 +917,17 @@ func Test_overrideBlocks(t *testing.T) { Want: hclext.Blocks{ { Type: "resource", - Labels: []string{"random_id", "server"}, + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - // the contents of any lifecycle nested block are merged on an argument-by-argument basis. - Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}, "ignore_changes": &hclext.Attribute{Name: "ignore_changes"}}, - // If an overriding resource block contains one or more provisioner blocks then any provisioner blocks in the original block are ignored. - // If an overriding resource block contains a connection block then it completely overrides any connection block present in the original block. Blocks: hclext.Blocks{ + // the contents of any lifecycle nested block are merged on an argument-by-argument basis. + { + Type: "lifecycle", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"create_before_destroy": &hclext.Attribute{Name: "create_before_destroy2"}, "prevent_destroy": &hclext.Attribute{Name: "prevent_destroy"}, "ignore_changes": &hclext.Attribute{Name: "ignore_changes"}}, + }, + }, + // If an overriding resource block contains one or more provisioner blocks then any provisioner blocks in the original block are ignored. { Type: "provisioner", Labels: []string{"remote-exec"}, @@ -880,6 +935,7 @@ func Test_overrideBlocks(t *testing.T) { Attributes: hclext.Attributes{"inline": &hclext.Attribute{Name: "inline"}}, }, }, + // If an overriding resource block contains a connection block then it completely overrides any connection block present in the original block. { Type: "connection", Body: &hclext.BodyContent{ @@ -892,93 +948,147 @@ func Test_overrideBlocks(t *testing.T) { }, }, { - Name: "override locals", + Name: "override data sources", Primaries: hclext.Blocks{ - // The "locals" blocks are allowed to be declared multiple times. - { - Type: "locals", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar"}}, - }, - }, { - Type: "locals", + Type: "data", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz"}, "qux": &hclext.Attribute{Name: "qux"}}, + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "foo"}, + "bar": &hclext.Attribute{Name: "bar"}, + }, + Blocks: hclext.Blocks{ + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "foo"}, + }, + }, + }, + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "bar": &hclext.Attribute{Name: "bar"}, + }, + }, + }, + { + Type: "other_nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "qux": &hclext.Attribute{Name: "qux"}, + }, + }, + }, + }, }, }, }, Overrides: hclext.Blocks{ { - Type: "locals", + Type: "data", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "foo2": &hclext.Attribute{Name: "foo2"}}, + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "bar"}, + "baz": &hclext.Attribute{Name: "baz"}, + }, + Blocks: hclext.Blocks{ + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "baz": &hclext.Attribute{Name: "baz"}, + }, + }, + }, + }, }, }, + }, + Want: hclext.Blocks{ { - Type: "locals", + Type: "data", + Labels: []string{"foo", "bar"}, Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"bar": &hclext.Attribute{Name: "bar2"}, "qux2": &hclext.Attribute{Name: "qux2"}}, + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "bar"}, + "bar": &hclext.Attribute{Name: "bar"}, + "baz": &hclext.Attribute{Name: "baz"}, + }, + Blocks: hclext.Blocks{ + { + Type: "other_nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "qux": &hclext.Attribute{Name: "qux"}, + }, + }, + }, + { + Type: "nested", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "baz": &hclext.Attribute{Name: "baz"}, + }, + }, + }, + }, }, }, }, - Want: hclext.Blocks{ + }, + { + Name: "override locals", + Primaries: hclext.Blocks{ { Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar2"}}, + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar"}}, }, }, { Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "qux": &hclext.Attribute{Name: "qux"}}, + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz"}, "qux": &hclext.Attribute{Name: "qux"}}, }, }, - // Locals not present in the primaries are added + }, + Overrides: hclext.Blocks{ { Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"foo2": &hclext.Attribute{Name: "foo2"}}, + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "foo2": &hclext.Attribute{Name: "foo2"}}, }, }, { Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"qux2": &hclext.Attribute{Name: "qux2"}}, + Attributes: hclext.Attributes{"bar": &hclext.Attribute{Name: "bar2"}}, }, }, }, - }, - { - Name: "no override multiple required_version", - Primaries: hclext.Blocks{ - // The "terraform" blocks are allowed to be declared multiple times. + Want: hclext.Blocks{ { - Type: "terraform", + Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version1"}}, + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar2"}}, }, }, { - Type: "terraform", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version2"}}, - }, - }, - }, - Overrides: hclext.Blocks{}, - Want: hclext.Blocks{ - { - Type: "terraform", + Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version1"}}, + Attributes: hclext.Attributes{"baz": &hclext.Attribute{Name: "baz2"}, "qux": &hclext.Attribute{Name: "qux"}}, }, }, + // Locals not present in the primaries are added. { - Type: "terraform", + Type: "locals", Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{"required_version": &hclext.Attribute{Name: "required_version2"}}, + Attributes: hclext.Attributes{"foo2": &hclext.Attribute{Name: "foo2"}}, }, }, }, @@ -1014,8 +1124,8 @@ func Test_overrideBlocks(t *testing.T) { }, }, Want: hclext.Blocks{ - // In both the required_version and required_providers settings, - // each override constraint entirely replaces the constraints for the same component in the original block. + // When overriding attributes, the last element in override takes precedence, + // so all attributes of primaries are overridden by required_version4. { Type: "terraform", Body: &hclext.BodyContent{ @@ -1031,93 +1141,6 @@ func Test_overrideBlocks(t *testing.T) { }, }, { - Name: "no override required_providers", - Primaries: hclext.Blocks{ - { - Type: "terraform", - Body: &hclext.BodyContent{ - Blocks: hclext.Blocks{ - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "aws": &hclext.Attribute{Name: "aws"}, - "google": &hclext.Attribute{Name: "google"}, - }, - }, - }, - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "azurerm": &hclext.Attribute{Name: "azurerm"}, - }, - }, - }, - }, - }, - }, - { - Type: "terraform", - Body: &hclext.BodyContent{ - Blocks: hclext.Blocks{ - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "google-beta": &hclext.Attribute{Name: "google-beta"}, - }, - }, - }, - }, - }, - }, - }, - Overrides: hclext.Blocks{}, - Want: hclext.Blocks{ - { - Type: "terraform", - Body: &hclext.BodyContent{ - Blocks: hclext.Blocks{ - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "aws": &hclext.Attribute{Name: "aws"}, - "google": &hclext.Attribute{Name: "google"}, - }, - }, - }, - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "azurerm": &hclext.Attribute{Name: "azurerm"}, - }, - }, - }, - }, - }, - }, - { - Type: "terraform", - Body: &hclext.BodyContent{ - Blocks: hclext.Blocks{ - { - Type: "required_providers", - Body: &hclext.BodyContent{ - Attributes: hclext.Attributes{ - "google-beta": &hclext.Attribute{Name: "google-beta"}, - }, - }, - }, - }, - }, - }, - }, - }, - { - // If the required_providers argument is set, its value is merged on an element-by-element basis Name: "override required_providers", Primaries: hclext.Blocks{ { @@ -1242,7 +1265,7 @@ func Test_overrideBlocks(t *testing.T) { }, }, }, - // Blocks not present in the primaries are added + // Blocks not present in the primaries are added. { Type: "terraform", Body: &hclext.BodyContent{ From 29578175d81ce53ed851c2dc468c17dd8800d4dc Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sun, 6 Oct 2024 08:13:04 +0000 Subject: [PATCH 7/9] Merge locals/required_providers if possible If there is only a single locals/required_providers in the primary, we will merge it rather than append it to maintain backwards compatibility. --- terraform/module.go | 68 +++++++-- terraform/module_test.go | 307 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 13 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index 27557b85f..dde3769b1 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -290,6 +290,10 @@ func overrideResourceBlock(primary, override *hclext.Block) { // it is returned to the caller with only those elements remaining. // This operation modifies the given override directly. func overrideLocalBlocks(primaries hclext.Blocks, override *hclext.Block) *hclext.Block { + // When there are multiple locals blocks, + // it is not obvious into which one the remaining local values ​​should be merged. + appendRemains := len(primaries) > 1 + // Tracks locals ​​that were not used to override. remains := hclext.Attributes{} for name, attr := range override.Body.Attributes { @@ -299,15 +303,21 @@ func overrideLocalBlocks(primaries hclext.Blocks, override *hclext.Block) *hclex // Overrides are applied on a value-by-value basis, ignoring which locals block they are defined in. for _, primary := range primaries { for name, attr := range override.Body.Attributes { - if _, exists := primary.Body.Attributes[name]; exists { + // Track the remaining local values ​​only if you need to append to them, + // otherwise simply merge them. + if appendRemains { + if _, exists := primary.Body.Attributes[name]; exists { + primary.Body.Attributes[name] = attr + delete(remains, name) + } + } else { primary.Body.Attributes[name] = attr - delete(remains, name) } } } // Any remaining locals that aren't overridden will be added as a new block. - if len(remains) > 0 { + if appendRemains && len(remains) > 0 { override.Body.Attributes = remains return override } @@ -322,6 +332,30 @@ func overrideLocalBlocks(primaries hclext.Blocks, override *hclext.Block) *hclex // it is returned to the caller with only those elements remaining. // This operation modifies the given override directly. func overrideTerraformBlocks(primaries hclext.Blocks, override *hclext.Block) *hclext.Block { + // When there are multiple required_providers blocks, + // it is not obvious into which one the remaining require providers ​​should be merged. + appendRemains := false + requiredProviderSeen := false + for _, primary := range primaries { + switch len(primary.Body.Blocks.ByType()["required_providers"]) { + case 0: + continue + + case 1: + // Found multiple terraform blocks with required_providers + if requiredProviderSeen { + appendRemains = true + break + } + requiredProviderSeen = true + + default: + // Found terraform block with multiple required_providers + appendRemains = true + break + } + } + // Tracks required providers ​​that were not used to override. remainRequiredProviders := override.Body.Blocks.ByType()["required_providers"] @@ -346,11 +380,17 @@ func overrideTerraformBlocks(primaries hclext.Blocks, override *hclext.Block) *h // If the required_providers argument is set, its value is merged on an element-by-element basis for _, override := range overridesByType[p.Type] { for name, attr := range override.Body.Attributes { - if _, exists := p.Body.Attributes[name]; exists { - p.Body.Attributes[name] = attr - for _, remain := range remainRequiredProviders { - delete(remain.Body.Attributes, name) + // Track the remaining required providers ​​only if you need to append to them, + // otherwise simply merge them. + if appendRemains { + if _, exists := p.Body.Attributes[name]; exists { + p.Body.Attributes[name] = attr + for _, remain := range remainRequiredProviders { + delete(remain.Body.Attributes, name) + } } + } else { + p.Body.Attributes[name] = attr } } } @@ -379,12 +419,14 @@ func overrideTerraformBlocks(primaries hclext.Blocks, override *hclext.Block) *h } // Any remaining required providers that aren't overridden will be added as a new block. - remainRequiredProviders = filterBlocks(remainRequiredProviders, func(b *hclext.Block) bool { - return len(b.Body.Attributes) > 0 - }) - if len(remainRequiredProviders) > 0 { - override.Body.Blocks = remainRequiredProviders - return override + if appendRemains { + remainRequiredProviders = filterBlocks(remainRequiredProviders, func(b *hclext.Block) bool { + return len(b.Body.Attributes) > 0 + }) + if len(remainRequiredProviders) > 0 { + override.Body.Blocks = remainRequiredProviders + return override + } } return nil } diff --git a/terraform/module_test.go b/terraform/module_test.go index 4a8bda5a8..e098cb0bf 100644 --- a/terraform/module_test.go +++ b/terraform/module_test.go @@ -1043,6 +1043,37 @@ func Test_overrideBlocks(t *testing.T) { }, { Name: "override locals", + Primaries: hclext.Blocks{ + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"foo": &hclext.Attribute{Name: "foo"}, "bar": &hclext.Attribute{Name: "bar"}}, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{"bar": &hclext.Attribute{Name: "bar2"}, "foo2": &hclext.Attribute{Name: "foo2"}}, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "locals", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "foo": &hclext.Attribute{Name: "foo"}, + "bar": &hclext.Attribute{Name: "bar2"}, + "foo2": &hclext.Attribute{Name: "foo2"}, + }, + }, + }, + }, + }, + { + Name: "override multiple locals", Primaries: hclext.Blocks{ { Type: "locals", @@ -1142,6 +1173,282 @@ func Test_overrideBlocks(t *testing.T) { }, { Name: "override required_providers", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google": &hclext.Attribute{Name: "google2"}, + "assert": &hclext.Attribute{Name: "assert"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "time": &hclext.Attribute{Name: "time"}, + }, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "google": &hclext.Attribute{Name: "google2"}, + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + "assert": &hclext.Attribute{Name: "assert"}, + "time": &hclext.Attribute{Name: "time"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override multiple required_providers", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + // Blocks not present in the primaries are added. + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override multiple terraform blocks with single required_providers", + Primaries: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + }, + Overrides: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + }, + }, + }, + }, + Want: hclext.Blocks{ + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "aws": &hclext.Attribute{Name: "aws2"}, + "google": &hclext.Attribute{Name: "google"}, + }, + }, + }, + }, + }, + }, + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "google-beta": &hclext.Attribute{Name: "google-beta"}, + }, + }, + }, + }, + }, + }, + // Blocks not present in the primaries are added. + { + Type: "terraform", + Body: &hclext.BodyContent{ + Blocks: hclext.Blocks{ + { + Type: "required_providers", + Body: &hclext.BodyContent{ + Attributes: hclext.Attributes{ + "azurerm": &hclext.Attribute{Name: "azurerm2"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "override multiple terraform blocks with multiple required_providers", Primaries: hclext.Blocks{ { Type: "terraform", From bd50eeeddc6e951f0d33aa8c58a26824933b53d5 Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sun, 6 Oct 2024 08:46:04 +0000 Subject: [PATCH 8/9] Do not sort override files in PartialContent For optimal performance, sort override files at parse time instead of sorting in PartialContent all the time. --- terraform/module.go | 20 +++++++------------- terraform/parser.go | 6 +++++- terraform/parser_test.go | 9 +++++++++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/terraform/module.go b/terraform/module.go index dde3769b1..b6ff59f35 100644 --- a/terraform/module.go +++ b/terraform/module.go @@ -2,9 +2,6 @@ package terraform import ( "fmt" - "maps" - "slices" - "sort" "strings" "github.com/hashicorp/hcl/v2" @@ -24,8 +21,9 @@ type Module struct { Sources map[string][]byte Files map[string]*hcl.File - primaries map[string]*hcl.File - overrides map[string]*hcl.File + primaries map[string]*hcl.File + overrides map[string]*hcl.File + overrideFilenames []string } func NewEmptyModule() *Module { @@ -40,8 +38,9 @@ func NewEmptyModule() *Module { Sources: map[string][]byte{}, Files: map[string]*hcl.File{}, - primaries: map[string]*hcl.File{}, - overrides: map[string]*hcl.File{}, + primaries: map[string]*hcl.File{}, + overrides: map[string]*hcl.File{}, + overrideFilenames: []string{}, } } @@ -142,13 +141,8 @@ func (m *Module) PartialContent(schema *hclext.BodySchema, ctx *Evaluator) (*hcl content.Blocks = append(content.Blocks, c.Blocks...) } - // If more than one override file defines the same top-level block, the overriding effect is compounded, - // with later blocks taking precedence over earlier blocks. // Overrides are processed in order first by filename (in lexicographical order) - // and then by position in each file. - overrideFilenames := slices.Collect(maps.Keys(m.overrides)) - sort.Strings(overrideFilenames) - for _, filename := range overrideFilenames { + for _, filename := range m.overrideFilenames { expanded, d := ctx.ExpandBlock(m.overrides[filename].Body, schema) diags = diags.Extend(d) c, d := hclext.PartialContent(expanded, schema) diff --git a/terraform/parser.go b/terraform/parser.go index d5cd91cd7..32b999b87 100644 --- a/terraform/parser.go +++ b/terraform/parser.go @@ -65,6 +65,7 @@ func (p *Parser) LoadConfigDir(baseDir, dir string) (*Module, hcl.Diagnostics) { } mod := NewEmptyModule() + mod.overrideFilenames = make([]string, len(overrides)) for _, path := range primaries { f, loadDiags := p.loadHCLFile(baseDir, path) @@ -78,7 +79,7 @@ func (p *Parser) LoadConfigDir(baseDir, dir string) (*Module, hcl.Diagnostics) { mod.Sources[realPath] = f.Bytes mod.Files[realPath] = f } - for _, path := range overrides { + for idx, path := range overrides { f, loadDiags := p.loadHCLFile(baseDir, path) diags = diags.Extend(loadDiags) if loadDiags.HasErrors() { @@ -89,7 +90,10 @@ func (p *Parser) LoadConfigDir(baseDir, dir string) (*Module, hcl.Diagnostics) { mod.overrides[realPath] = f mod.Sources[realPath] = f.Bytes mod.Files[realPath] = f + mod.overrideFilenames[idx] = realPath } + // Overrides are processed in order first by filename (in lexicographical order) + sort.Strings(mod.overrideFilenames) if diags.HasErrors() { return mod, diags } diff --git a/terraform/parser_test.go b/terraform/parser_test.go index ecddd1f9c..45a731844 100644 --- a/terraform/parser_test.go +++ b/terraform/parser_test.go @@ -39,6 +39,7 @@ func TestLoadConfigDir(t *testing.T) { "main_override.tf": {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: "main_override.tf"}})}, "override.tf": {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: "override.tf"}})}, }, + overrideFilenames: []string{"main_override.tf", "override.tf"}, Sources: map[string][]byte{ "main.tf": {}, "main_override.tf": {}, @@ -69,6 +70,7 @@ func TestLoadConfigDir(t *testing.T) { "main_override.tf.json": {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: "main_override.tf.json"}})}, "override.tf.json": {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: "override.tf.json"}})}, }, + overrideFilenames: []string{"main_override.tf.json", "override.tf.json"}, Sources: map[string][]byte{ "main.tf.json": []byte("{}"), "main_override.tf.json": []byte("{}"), @@ -99,6 +101,7 @@ func TestLoadConfigDir(t *testing.T) { filepath.Join("foo", "main_override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("foo", "main_override.tf")}})}, filepath.Join("foo", "override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("foo", "override.tf")}})}, }, + overrideFilenames: []string{filepath.Join("foo", "main_override.tf"), filepath.Join("foo", "override.tf")}, Sources: map[string][]byte{ filepath.Join("foo", "main.tf"): {}, filepath.Join("foo", "main_override.tf"): {}, @@ -129,6 +132,7 @@ func TestLoadConfigDir(t *testing.T) { filepath.Join("bar", "main_override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("bar", "main_override.tf")}})}, filepath.Join("bar", "override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("bar", "override.tf")}})}, }, + overrideFilenames: []string{filepath.Join("bar", "main_override.tf"), filepath.Join("bar", "override.tf")}, Sources: map[string][]byte{ filepath.Join("bar", "main.tf"): {}, filepath.Join("bar", "main_override.tf"): {}, @@ -159,6 +163,7 @@ func TestLoadConfigDir(t *testing.T) { filepath.Join("foo", "bar", "main_override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("foo", "bar", "main_override.tf")}})}, filepath.Join("foo", "bar", "override.tf"): {Body: hcltest.MockBody(&hcl.BodyContent{MissingItemRange: hcl.Range{Filename: filepath.Join("foo", "bar", "override.tf")}})}, }, + overrideFilenames: []string{filepath.Join("foo", "bar", "main_override.tf"), filepath.Join("foo", "bar", "override.tf")}, Sources: map[string][]byte{ filepath.Join("foo", "bar", "main.tf"): {}, filepath.Join("foo", "bar", "main_override.tf"): {}, @@ -216,6 +221,10 @@ func TestLoadConfigDir(t *testing.T) { t.Errorf("diff: %s", diff) } + if diff := cmp.Diff(mod.overrideFilenames, test.want.overrideFilenames); diff != "" { + t.Errorf("diff: %s", diff) + } + if diff := cmp.Diff(mod.Sources, test.want.Sources); diff != "" { t.Errorf("diff: %s", diff) } From a79a6839c27e13e1d0870f161b7501f78320201e Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Sun, 6 Oct 2024 13:20:46 +0000 Subject: [PATCH 9/9] Add E2E test for overriding required providers --- .../inspection/override/.tflint.hcl | 4 + .../inspection/override/result.json | 20 +++++ .../inspection/override/template.tf | 12 +++ .../inspection/override/template_override.tf | 8 ++ .../inspection/override/version_override.tf | 5 ++ plugin/stub-generator/sources/testing/main.go | 1 + .../rules/terraform_required_providers.go | 87 +++++++++++++++++++ 7 files changed, 137 insertions(+) create mode 100644 integrationtest/inspection/override/version_override.tf create mode 100644 plugin/stub-generator/sources/testing/rules/terraform_required_providers.go diff --git a/integrationtest/inspection/override/.tflint.hcl b/integrationtest/inspection/override/.tflint.hcl index e19f589dd..396bc0672 100644 --- a/integrationtest/inspection/override/.tflint.hcl +++ b/integrationtest/inspection/override/.tflint.hcl @@ -1,3 +1,7 @@ plugin "testing" { enabled = true } + +rule "terraform_required_providers" { + enabled = true +} diff --git a/integrationtest/inspection/override/result.json b/integrationtest/inspection/override/result.json index dfb6f374b..c5f563ba4 100644 --- a/integrationtest/inspection/override/result.json +++ b/integrationtest/inspection/override/result.json @@ -1,5 +1,25 @@ { "issues": [ + { + "rule": { + "name": "terraform_required_providers", + "severity": "error", + "link": "" + }, + "message": "required_providers: aws=2,azurerm=1,google=3,oracle=2", + "range": { + "filename": "template.tf", + "start": { + "line": 11, + "column": 3 + }, + "end": { + "line": 11, + "column": 21 + } + }, + "callers": [] + }, { "rule": { "name": "aws_instance_example_type", diff --git a/integrationtest/inspection/override/template.tf b/integrationtest/inspection/override/template.tf index 09d245ee3..55920459b 100644 --- a/integrationtest/inspection/override/template.tf +++ b/integrationtest/inspection/override/template.tf @@ -2,3 +2,15 @@ resource "aws_instance" "web" { ami = "ami-12345678" instance_type = "t2.micro" // Override by `template_override.tf` } + +terraform { + backend "s3" {} +} + +terraform { + required_providers { + aws = "1" // Override by `template_override.tf` + google = "1" // Override by `version_override.tf` + azurerm = "1" + } +} diff --git a/integrationtest/inspection/override/template_override.tf b/integrationtest/inspection/override/template_override.tf index c14a53457..3c6dbefc6 100644 --- a/integrationtest/inspection/override/template_override.tf +++ b/integrationtest/inspection/override/template_override.tf @@ -3,3 +3,11 @@ resource "aws_instance" "web" { instance_type = "m5.2xlarge" // aws_instance_example_type iam_instance_profile = "web-server" } + +terraform { + required_providers { + aws = "2" + google = "2" + oracle = "2" + } +} diff --git a/integrationtest/inspection/override/version_override.tf b/integrationtest/inspection/override/version_override.tf new file mode 100644 index 000000000..eeb64a5fd --- /dev/null +++ b/integrationtest/inspection/override/version_override.tf @@ -0,0 +1,5 @@ +terraform { + required_providers { + google = "3" + } +} diff --git a/plugin/stub-generator/sources/testing/main.go b/plugin/stub-generator/sources/testing/main.go index bfd71dd60..8bcd5a609 100644 --- a/plugin/stub-generator/sources/testing/main.go +++ b/plugin/stub-generator/sources/testing/main.go @@ -27,6 +27,7 @@ func main() { rules.NewTerraformAutofixRemoveLocalRule(), // should be former than terraform_autofix_comment because this rule changes the line number rules.NewTerraformAutofixCommentRule(), rules.NewAwsInstanceAutofixConflictRule(), // should be later than terraform_autofix_comment because this rule adds an issue for terraform_autofix_comment + rules.NewTerraformRequiredProvidersRule(), }, }, }) diff --git a/plugin/stub-generator/sources/testing/rules/terraform_required_providers.go b/plugin/stub-generator/sources/testing/rules/terraform_required_providers.go new file mode 100644 index 000000000..e3cbf4713 --- /dev/null +++ b/plugin/stub-generator/sources/testing/rules/terraform_required_providers.go @@ -0,0 +1,87 @@ +package rules + +import ( + "fmt" + "sort" + "strings" + + "github.com/terraform-linters/tflint-plugin-sdk/hclext" + "github.com/terraform-linters/tflint-plugin-sdk/tflint" +) + +// TerraformRequiredProviders checks whether ... +type TerraformRequiredProviders struct { + tflint.DefaultRule +} + +// NewTerraformRequiredProvidersRule returns a new rule +func NewTerraformRequiredProvidersRule() *TerraformRequiredProviders { + return &TerraformRequiredProviders{} +} + +// Name returns the rule name +func (r *TerraformRequiredProviders) Name() string { + return "terraform_required_providers" +} + +// Enabled returns whether the rule is enabled by default +func (r *TerraformRequiredProviders) Enabled() bool { + return false +} + +// Severity returns the rule severity +func (r *TerraformRequiredProviders) Severity() tflint.Severity { + return tflint.ERROR +} + +// Link returns the rule reference link +func (r *TerraformRequiredProviders) Link() string { + return "" +} + +// Check checks whether ... +func (r *TerraformRequiredProviders) Check(runner tflint.Runner) error { + module, err := runner.GetModuleContent(&hclext.BodySchema{ + Blocks: []hclext.BlockSchema{ + { + Type: "terraform", + Body: &hclext.BodySchema{ + Blocks: []hclext.BlockSchema{ + { + Type: "required_providers", + Body: &hclext.BodySchema{Mode: hclext.SchemaJustAttributesMode}, + }, + }, + }, + }, + }, + }, nil) + if err != nil { + return err + } + + for _, terraform := range module.Blocks { + for _, requiredProvider := range terraform.Body.Blocks { + ret := []string{} + for name, attr := range requiredProvider.Body.Attributes { + v, diags := attr.Expr.Value(nil) + if diags.HasErrors() { + return diags + } + ret = append(ret, fmt.Sprintf("%s=%s", name, v.AsString())) + } + sort.Strings(ret) + + err := runner.EmitIssue( + r, + fmt.Sprintf("required_providers: %s", strings.Join(ret, ",")), + requiredProvider.DefRange, + ) + if err != nil { + return err + } + } + } + + return nil +}