From 796e3b7ff16c74a79ec85b4014467129371899b4 Mon Sep 17 00:00:00 2001 From: Roman Kabaev Date: Sun, 10 Nov 2024 14:19:51 +0300 Subject: [PATCH 1/4] Added exclude_from_copy to config --- cli/commands/terraform/download_source.go | 17 +++- .../terraform/download_source_test.go | 2 +- cli/commands/terraform/file_copy_getter.go | 5 +- config/config.go | 8 +- config/config_as_cty.go | 2 + config/include.go | 12 +++ config/include_test.go | 11 +++ .../config-blocks-and-attributes.md | 4 + test/helpers/package.go | 2 +- test/integration_aws_test.go | 2 +- test/integration_gcp_test.go | 2 +- test/integration_json_test.go | 1 + test/integration_serial_aws_test.go | 2 +- test/integration_test.go | 1 + test/integration_tflint_test.go | 4 +- test/integration_windows_test.go | 4 +- util/file.go | 30 +++++- util/file_test.go | 95 ++++++++++++++++++- 18 files changed, 183 insertions(+), 21 deletions(-) diff --git a/cli/commands/terraform/download_source.go b/cli/commands/terraform/download_source.go index c0fa8eb729..881efa63ea 100644 --- a/cli/commands/terraform/download_source.go +++ b/cli/commands/terraform/download_source.go @@ -50,13 +50,18 @@ func downloadTerraformSource(ctx context.Context, source string, opts *options.T opts.Logger.Debugf("Copying files from %s into %s", opts.WorkingDir, terraformSource.WorkingDir) - var includeInCopy []string + var includeInCopy, excludeFromCopy []string + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil { includeInCopy = *terragruntConfig.Terraform.IncludeInCopy } + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil { + excludeFromCopy = *terragruntConfig.Terraform.ExcludeFromCopy + } + // Always include the .tflint.hcl file, if it exists includeInCopy = append(includeInCopy, tfLintConfig) - if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil { + if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy, excludeFromCopy); err != nil { return nil, err } @@ -213,12 +218,16 @@ func updateGetters(terragruntOptions *options.TerragruntOptions, terragruntConfi for getterName, getterValue := range getter.Getters { if getterName == "file" { - var includeInCopy []string + var includeInCopy, excludeFromCopy []string + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil { includeInCopy = *terragruntConfig.Terraform.IncludeInCopy } + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil { + includeInCopy = *terragruntConfig.Terraform.ExcludeFromCopy + } - client.Getters[getterName] = &FileCopyGetter{IncludeInCopy: includeInCopy, Logger: terragruntOptions.Logger} + client.Getters[getterName] = &FileCopyGetter{IncludeInCopy: includeInCopy, Logger: terragruntOptions.Logger, ExcludeFromCopy: excludeFromCopy} } else { client.Getters[getterName] = getterValue } diff --git a/cli/commands/terraform/download_source_test.go b/cli/commands/terraform/download_source_test.go index d1f8422218..e19b19f58a 100644 --- a/cli/commands/terraform/download_source_test.go +++ b/cli/commands/terraform/download_source_test.go @@ -494,6 +494,6 @@ func copyFolder(t *testing.T, src string, dest string) { logger := log.New() logger.SetOptions(log.WithOutput(io.Discard)) - err := util.CopyFolderContents(logger, filepath.FromSlash(src), filepath.FromSlash(dest), ".terragrunt-test", nil) + err := util.CopyFolderContents(logger, filepath.FromSlash(src), filepath.FromSlash(dest), ".terragrunt-test", nil, nil) require.NoError(t, err) } diff --git a/cli/commands/terraform/file_copy_getter.go b/cli/commands/terraform/file_copy_getter.go index 44a4b4edcc..83616041c9 100644 --- a/cli/commands/terraform/file_copy_getter.go +++ b/cli/commands/terraform/file_copy_getter.go @@ -21,7 +21,8 @@ type FileCopyGetter struct { // List of glob paths that should be included in the copy. This can be used to override the default behavior of // Terragrunt, which will skip hidden folders. - IncludeInCopy []string + IncludeInCopy []string + ExcludeFromCopy []string Logger log.Logger } @@ -42,7 +43,7 @@ func (g *FileCopyGetter) Get(dst string, u *url.URL) error { return errors.Errorf("source path must be a directory") } - return util.CopyFolderContents(g.Logger, path, dst, SourceManifestName, g.IncludeInCopy) + return util.CopyFolderContents(g.Logger, path, dst, SourceManifestName, g.IncludeInCopy, g.ExcludeFromCopy) } // GetFile The original FileGetter already knows how to do file copying so long as we set the Copy flag to true, so just diff --git a/config/config.go b/config/config.go index c124d95085..deff632a24 100644 --- a/config/config.go +++ b/config/config.go @@ -440,6 +440,7 @@ type ErrorHook struct { func (conf *Hook) String() string { return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands)) } + func (conf *ErrorHook) String() string { return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands)) } @@ -456,7 +457,8 @@ type TerraformConfig struct { // Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to // be defined and we want to make this optional. - IncludeInCopy *[]string `hcl:"include_in_copy,attr"` + IncludeInCopy *[]string `hcl:"include_in_copy,attr"` + ExcludeFromCopy *[]string `hcl:"exclude_from_copy,attr"` CopyTerraformLockFile *bool `hcl:"copy_terraform_lock_file,attr"` } @@ -1408,7 +1410,7 @@ func (cfg *TerragruntConfig) GetMapFieldMetadata(fieldType, fieldName string) (m return nil, false } - var result = make(map[string]string) + result := make(map[string]string) for key, value := range value { result[key] = fmt.Sprintf("%v", value) } @@ -1422,7 +1424,7 @@ func (cfg *TerragruntConfig) EngineOptions() (*options.EngineOptions, error) { return nil, nil } // in case of Meta is null, set empty meta - var meta = map[string]interface{}{} + meta := map[string]interface{}{} if cfg.Engine.Meta != nil { parsedMeta, err := ParseCtyValueToMap(*cfg.Engine.Meta) diff --git a/config/config_as_cty.go b/config/config_as_cty.go index 012c996802..0f8acfd2d8 100644 --- a/config/config_as_cty.go +++ b/config/config_as_cty.go @@ -555,6 +555,7 @@ type CtyTerraformConfig struct { ExtraArgs map[string]TerraformExtraArguments `cty:"extra_arguments"` Source *string `cty:"source"` IncludeInCopy *[]string `cty:"include_in_copy"` + ExcludeFromCopy *[]string `cty:"exclude_from_copy"` CopyTerraformLockFile *bool `cty:"copy_terraform_lock_file"` BeforeHooks map[string]Hook `cty:"before_hook"` AfterHooks map[string]Hook `cty:"after_hook"` @@ -570,6 +571,7 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) { configCty := CtyTerraformConfig{ Source: config.Source, IncludeInCopy: config.IncludeInCopy, + ExcludeFromCopy: config.ExcludeFromCopy, CopyTerraformLockFile: config.CopyTerraformLockFile, ExtraArgs: map[string]TerraformExtraArguments{}, BeforeHooks: map[string]Hook{}, diff --git a/config/include.go b/config/include.go index 276002aebc..4e17ee8e4f 100644 --- a/config/include.go +++ b/config/include.go @@ -503,6 +503,18 @@ func (cfg *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig, terragrun } } + if sourceConfig.Terraform.ExcludeFromCopy != nil { + srcList := *sourceConfig.Terraform.ExcludeFromCopy + + if cfg.Terraform.ExcludeFromCopy != nil { + targetList := *cfg.Terraform.ExcludeFromCopy + combinedList := append(srcList, targetList...) + cfg.Terraform.ExcludeFromCopy = &combinedList + } else { + cfg.Terraform.ExcludeFromCopy = &srcList + } + } + mergeExtraArgs(terragruntOptions, sourceConfig.Terraform.ExtraArgs, &cfg.Terraform.ExtraArgs) mergeHooks(terragruntOptions, sourceConfig.Terraform.BeforeHooks, &cfg.Terraform.BeforeHooks) diff --git a/config/include_test.go b/config/include_test.go index 7b31921a17..314279315c 100644 --- a/config/include_test.go +++ b/config/include_test.go @@ -152,6 +152,11 @@ func TestMergeConfigIntoIncludedConfig(t *testing.T) { &config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}}, &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}}, }, + { + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{ExcludeFromCopy: &[]string{"abc"}}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], ExcludeFromCopy: &[]string{"abc"}}}, + }, } for _, testCase := range testCases { @@ -324,6 +329,12 @@ func TestDeepMergeConfigIntoIncludedConfig(t *testing.T) { &config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}}, &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}}, }, + { + "terraform copy_terraform_lock_file", + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{ExcludeFromCopy: &[]string{"abc"}}}, + &config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], ExcludeFromCopy: &[]string{"abc"}}}, + }, } for _, tt := range tc { diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 1af6672683..7de59eaba1 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -110,6 +110,10 @@ The `terraform` block supports the following arguments: can specify that in this list to ensure it gets copied over to the scratch copy (e.g., `include_in_copy = [".python-version"]`). +- `exclude_from_copy` (attribute): A list of glob patterns (e.g., `["*.txt"]`) that should always be skipped when copying into the + OpenTofu/Terraform working directory. All examples valid for `include_in_copy` can be used here. + When `include_in_copy` is provided, you can still provide `exclude_from_copy` to skip provided glob, or nested glob from `include_in_copy` + - `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock file into your source repository from your working directory as described in [Lock File Handling]({{site.baseurl}}/docs/features/lock-file-handling/). This attribute allows you to disable the copy diff --git a/test/helpers/package.go b/test/helpers/package.go index 196f436d10..f48caba8f0 100644 --- a/test/helpers/package.go +++ b/test/helpers/package.go @@ -88,7 +88,7 @@ func CopyEnvironment(t *testing.T, environmentPath string, includeInCopy ...stri t.Logf("Copying %s to %s", environmentPath, tmpDir) - require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy)) + require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy, nil)) return tmpDir } diff --git a/test/integration_aws_test.go b/test/integration_aws_test.go index 0bf40c79c1..57a403673b 100644 --- a/test/integration_aws_test.go +++ b/test/integration_aws_test.go @@ -1012,7 +1012,7 @@ func TestAwsParallelStateInit(t *testing.T) { require.NoError(t, err) } for i := 0; i < 20; i++ { - err := util.CopyFolderContents(createLogger(), testFixtureParallelStateInit, tmpEnvPath, ".terragrunt-test", nil) + err := util.CopyFolderContents(createLogger(), testFixtureParallelStateInit, tmpEnvPath, ".terragrunt-test", nil, nil) require.NoError(t, err) err = os.Rename( path.Join(tmpEnvPath, "template"), diff --git a/test/integration_gcp_test.go b/test/integration_gcp_test.go index 521d8d63ae..3b47d54495 100644 --- a/test/integration_gcp_test.go +++ b/test/integration_gcp_test.go @@ -113,7 +113,7 @@ func TestGcpParallelStateInit(t *testing.T) { require.NoError(t, err) } for i := 0; i < 20; i++ { - err := util.CopyFolderContents(createLogger(), testFixtureGcsParallelStateInit, tmpEnvPath, ".terragrunt-test", nil) + err := util.CopyFolderContents(createLogger(), testFixtureGcsParallelStateInit, tmpEnvPath, ".terragrunt-test", nil, nil) require.NoError(t, err) err = os.Rename( path.Join(tmpEnvPath, "template"), diff --git a/test/integration_json_test.go b/test/integration_json_test.go index 0d4ce9f016..276215b7cb 100644 --- a/test/integration_json_test.go +++ b/test/integration_json_test.go @@ -435,6 +435,7 @@ func TestRenderJsonMetadataTerraform(t *testing.T) { "error_hook": map[string]interface{}{}, "extra_arguments": map[string]interface{}{}, "include_in_copy": nil, + "exclude_from_copy": nil, "source": "../terraform", "copy_terraform_lock_file": nil, }, diff --git a/test/integration_serial_aws_test.go b/test/integration_serial_aws_test.go index ae7dcb36af..ad0eaeaefa 100644 --- a/test/integration_serial_aws_test.go +++ b/test/integration_serial_aws_test.go @@ -104,7 +104,7 @@ func testRemoteFixtureParallelism(t *testing.T, parallelism int, numberOfModules t.Fatalf("Failed to create temp dir due to error: %v", err) } for i := 0; i < numberOfModules; i++ { - err := util.CopyFolderContents(createLogger(), testFixtureParallelism, tmpEnvPath, ".terragrunt-test", nil) + err := util.CopyFolderContents(createLogger(), testFixtureParallelism, tmpEnvPath, ".terragrunt-test", nil, nil) if err != nil { return "", 0, err } diff --git a/test/integration_test.go b/test/integration_test.go index d7debb672e..11636b70db 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -2395,6 +2395,7 @@ func TestReadTerragruntConfigFull(t *testing.T) { map[string]interface{}{ "source": "./delorean", "include_in_copy": []interface{}{"time_machine.*"}, + "exclude_from_copy": []interface{}{"excluded_time_machine.*"}, "copy_terraform_lock_file": true, "extra_arguments": map[string]interface{}{ "var-files": map[string]interface{}{ diff --git a/test/integration_tflint_test.go b/test/integration_tflint_test.go index 927057ef32..ce320244ca 100644 --- a/test/integration_tflint_test.go +++ b/test/integration_tflint_test.go @@ -98,7 +98,7 @@ func TestTflintInitSameModule(t *testing.T) { // generate multiple "app" modules that will be initialized in parallel for i := 0; i < 50; i++ { appPath := util.JoinPath(modulePath, "dev", fmt.Sprintf("app-%d", i)) - err := util.CopyFolderContents(createLogger(), appTemplate, appPath, ".terragrunt-test", []string{}) + err := util.CopyFolderContents(createLogger(), appTemplate, appPath, ".terragrunt-test", []string{}, []string{}) require.NoError(t, err) } helpers.RunTerragrunt(t, "terragrunt run-all init --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir "+runPath) @@ -203,7 +203,7 @@ func CopyEnvironmentWithTflint(t *testing.T, environmentPath string) string { t.Logf("Copying %s to %s", environmentPath, tmpDir) - require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"})) + require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}, []string{})) return tmpDir } diff --git a/test/integration_windows_test.go b/test/integration_windows_test.go index acb4601e98..7b2da09162 100644 --- a/test/integration_windows_test.go +++ b/test/integration_windows_test.go @@ -179,7 +179,7 @@ func CopyEnvironmentToPath(t *testing.T, environmentPath, targetPath string) { t.Fatalf("Failed to create temp dir %s due to error %v", targetPath, err) } - copyErr := util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(targetPath, environmentPath), ".terragrunt-test", nil) + copyErr := util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(targetPath, environmentPath), ".terragrunt-test", nil, nil) require.NoError(t, copyErr) } @@ -191,7 +191,7 @@ func CopyEnvironmentWithTflint(t *testing.T, environmentPath string) string { t.Logf("Copying %s to %s", environmentPath, tmpDir) - require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"})) + require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}, []string{})) return tmpDir } diff --git a/util/file.go b/util/file.go index 29ef468739..8daee9f082 100644 --- a/util/file.go +++ b/util/file.go @@ -239,6 +239,15 @@ func listContainsElementWithPrefix(list []string, elementPrefix string) bool { return false } +func pathContainsPrefix(path string, prefixes []string) bool { + for _, element := range prefixes { + if strings.HasPrefix(path, element) { + return true + } + } + return false +} + // Takes apbsolute glob path and returns an array of expanded relative paths func expandGlobPath(source, absoluteGlobPath string) ([]string, error) { includeExpandedGlobs := []string{} @@ -276,7 +285,7 @@ func expandGlobPath(source, absoluteGlobPath string) ([]string, error) { // CopyFolderContents copies the files and folders within the source folder into the destination folder. Note that hidden files and folders // (those starting with a dot) will be skipped. Will create a specified manifest file that contains paths of all copied files. -func CopyFolderContents(logger log.Logger, source, destination, manifestFile string, includeInCopy []string) error { +func CopyFolderContents(logger log.Logger, source, destination, manifestFile string, includeInCopy []string, excludeFromCopy []string) error { // Expand all the includeInCopy glob paths, converting the globbed results to relative paths so that they work in // the copy filter. includeExpandedGlobs := []string{} @@ -292,11 +301,28 @@ func CopyFolderContents(logger log.Logger, source, destination, manifestFile str includeExpandedGlobs = append(includeExpandedGlobs, expandGlob...) } + excludeExpandedGlobs := []string{} + + for _, excludeGlob := range excludeFromCopy { + globPath := filepath.Join(source, excludeGlob) + + expandGlob, err := expandGlobPath(source, globPath) + if err != nil { + return errors.New(err) + } + + excludeExpandedGlobs = append(excludeExpandedGlobs, expandGlob...) + } + return CopyFolderContentsWithFilter(logger, source, destination, manifestFile, func(absolutePath string) bool { relativePath, err := GetPathRelativeTo(absolutePath, source) - if err == nil && listContainsElementWithPrefix(includeExpandedGlobs, relativePath) { + + if err == nil && listContainsElementWithPrefix(includeExpandedGlobs, relativePath) && !pathContainsPrefix(relativePath, excludeExpandedGlobs) { return true } + if err == nil && pathContainsPrefix(relativePath, excludeExpandedGlobs) { + return false + } return !TerragruntExcludes(filepath.FromSlash(relativePath)) }) diff --git a/util/file_test.go b/util/file_test.go index dc612a21bf..edb1d0884a 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -362,7 +362,100 @@ func TestIncludeInCopy(t *testing.T) { assert.NoError(t, os.WriteFile(path, fileContent, 0644)) } - require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", includeInCopy)) + require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", includeInCopy, nil)) + + for i, tt := range tc { + tt := tt + + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + + _, err := os.Stat(filepath.Join(destination, tt.path)) + assert.True(t, + tt.copyExpected && err == nil || + !tt.copyExpected && errors.Is(err, os.ErrNotExist), + "Unexpected copy result for file '%s' (should be copied: '%t') - got error: %s", tt.path, tt.copyExpected, err) + }) + } +} + +func TestExcludeFromCopy(t *testing.T) { + t.Parallel() + + excludeFromCopy := []string{"module/region2", "**/exclude-me-here", "**/app1"} + + tc := []struct { + path string + copyExpected bool + }{ + {"/app/terragrunt.hcl", true}, + {"/module/main.tf", true}, + {"/module/region1/info.txt", true}, + {"/module/region1/project2-1/app1/f2-dot-f2.txt", false}, + {"/module/region3/project3-1/f1-2-levels.txt", true}, + {"/module/region3/project3-1/app1/exclude-me-here/file.txt", false}, + {"/module/region3/project3-2/f0/f0-3-levels.txt", true}, + {"/module/region2/project2-1/app2/f2-dot-f2.txt", false}, + {"/module/region2/project2-1/readme.txt", false}, + {"/module/region2/project2-2/f2-dot-f0.txt", false}, + } + + tempDir := t.TempDir() + source := filepath.Join(tempDir, "source") + destination := filepath.Join(tempDir, "destination") + + fileContent := []byte("source file") + for _, tt := range tc { + path := filepath.Join(source, tt.path) + assert.NoError(t, os.MkdirAll(filepath.Dir(path), os.ModePerm)) + assert.NoError(t, os.WriteFile(path, fileContent, 0644)) + } + + require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", nil, excludeFromCopy)) + + for i, tt := range tc { + tt := tt + + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + + _, err := os.Stat(filepath.Join(destination, tt.path)) + assert.True(t, + tt.copyExpected && err == nil || + !tt.copyExpected && errors.Is(err, os.ErrNotExist), + "Unexpected copy result for file '%s' (should be copied: '%t') - got error: %s", tt.path, tt.copyExpected, err) + }) + } +} + +func TestExcludeIncludeBehaviourPriority(t *testing.T) { + t.Parallel() + + includeInCopy := []string{"_module/.region2", "_module/.region3"} + excludeFromCopy := []string{"**/.project2-2", "_module/.region3"} + + tc := []struct { + path string + copyExpected bool + }{ + {"/_module/.region2/.project2-1/app2/f2-dot-f2.txt", true}, + {"/_module/.region2/.project2-1/readme.txt", true}, + {"/_module/.region2/.project2-2/f2-dot-f0.txt", false}, + {"/_module/.region3/.project2-1/readme.txt", false}, + } + + tempDir := t.TempDir() + source := filepath.Join(tempDir, "source") + destination := filepath.Join(tempDir, "destination") + + fileContent := []byte("source file") + for _, tt := range tc { + path := filepath.Join(source, tt.path) + assert.NoError(t, os.MkdirAll(filepath.Dir(path), os.ModePerm)) + assert.NoError(t, os.WriteFile(path, fileContent, 0644)) + } + + require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", includeInCopy, excludeFromCopy)) for i, tt := range tc { tt := tt From a78ecfbe0a66ad246b282a1bf376f342ee57284c Mon Sep 17 00:00:00 2001 From: Roman Kabaev Date: Wed, 8 Jan 2025 00:34:37 +0300 Subject: [PATCH 2/4] Changed README info for more clear behaviour explanation on both include and exclude --- docs/_docs/04_reference/config-blocks-and-attributes.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 7de59eaba1..3433780a1f 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -111,8 +111,10 @@ The `terraform` block supports the following arguments: (e.g., `include_in_copy = [".python-version"]`). - `exclude_from_copy` (attribute): A list of glob patterns (e.g., `["*.txt"]`) that should always be skipped when copying into the - OpenTofu/Terraform working directory. All examples valid for `include_in_copy` can be used here. - When `include_in_copy` is provided, you can still provide `exclude_from_copy` to skip provided glob, or nested glob from `include_in_copy` + OpenTofu/Terraform working directory. All examples valid for `include_in_copy` can be used here. + + *Note that using `include_in_copy` and `exclude_from_copy` are not mutually exclusive.* + If a file matches a pattern in both `include_in_copy` and `exclude_from_copy`, it will not be included. If you would like to ensure that the file _is_ included, make sure the patterns you use for `include_in_copy` do not match the patterns in `exclude_from_copy`. - `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock file into your source repository from your working directory as described in From 545a631224d713be1ff332a1f51c8111bb6fe561 Mon Sep 17 00:00:00 2001 From: Roman Kabaev Date: Wed, 8 Jan 2025 14:47:05 +0300 Subject: [PATCH 3/4] Linting error fixes --- cli/commands/terraform/download_source.go | 19 +++++++++++++-- .../config-blocks-and-attributes.md | 4 ++-- test/helpers/package.go | 5 +++- util/file.go | 15 ++++++++++-- util/file_test.go | 23 ++++++++----------- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/cli/commands/terraform/download_source.go b/cli/commands/terraform/download_source.go index 881efa63ea..52fa96ff4a 100644 --- a/cli/commands/terraform/download_source.go +++ b/cli/commands/terraform/download_source.go @@ -55,13 +55,23 @@ func downloadTerraformSource(ctx context.Context, source string, opts *options.T if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil { includeInCopy = *terragruntConfig.Terraform.IncludeInCopy } + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil { excludeFromCopy = *terragruntConfig.Terraform.ExcludeFromCopy } // Always include the .tflint.hcl file, if it exists includeInCopy = append(includeInCopy, tfLintConfig) - if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy, excludeFromCopy); err != nil { + + err = util.CopyFolderContents( + opts.Logger, + opts.WorkingDir, + terraformSource.WorkingDir, + ModuleManifestName, + includeInCopy, + excludeFromCopy, + ) + if err != nil { return nil, err } @@ -223,11 +233,16 @@ func updateGetters(terragruntOptions *options.TerragruntOptions, terragruntConfi if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil { includeInCopy = *terragruntConfig.Terraform.IncludeInCopy } + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil { includeInCopy = *terragruntConfig.Terraform.ExcludeFromCopy } - client.Getters[getterName] = &FileCopyGetter{IncludeInCopy: includeInCopy, Logger: terragruntOptions.Logger, ExcludeFromCopy: excludeFromCopy} + client.Getters[getterName] = &FileCopyGetter{ + IncludeInCopy: includeInCopy, + Logger: terragruntOptions.Logger, + ExcludeFromCopy: excludeFromCopy, + } } else { client.Getters[getterName] = getterValue } diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 3433780a1f..6191fa23df 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -114,7 +114,7 @@ The `terraform` block supports the following arguments: OpenTofu/Terraform working directory. All examples valid for `include_in_copy` can be used here. *Note that using `include_in_copy` and `exclude_from_copy` are not mutually exclusive.* - If a file matches a pattern in both `include_in_copy` and `exclude_from_copy`, it will not be included. If you would like to ensure that the file _is_ included, make sure the patterns you use for `include_in_copy` do not match the patterns in `exclude_from_copy`. + If a file matches a pattern in both `include_in_copy` and `exclude_from_copy`, it will not be included. If you would like to ensure that the file *is* included, make sure the patterns you use for `include_in_copy` do not match the patterns in `exclude_from_copy`. - `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock file into your source repository from your working directory as described in @@ -438,7 +438,7 @@ For the `s3` backend, the following additional properties are supported in the ` - `dynamodb_table` - (Optional) The name of a DynamoDB table to use for state locking and consistency. The table must have a primary key named LockID. If not present, locking will be disabled. - `skip_bucket_versioning`: When `true`, the S3 bucket that is created to store the state will not be versioned. - `skip_bucket_ssencryption`: When `true`, the S3 bucket that is created to store the state will not be configured with server-side encryption. -- `skip_bucket_accesslogging`: _DEPRECATED_ If provided, will be ignored. A log warning will be issued in the console output to notify the user. +- `skip_bucket_accesslogging`: *DEPRECATED* If provided, will be ignored. A log warning will be issued in the console output to notify the user. - `skip_bucket_root_access`: When `true`, the S3 bucket that is created will not be configured with bucket policies that allow access to the root AWS user. - `skip_bucket_enforced_tls`: When `true`, the S3 bucket that is created will not be configured with a bucket policy that enforces access to the bucket via a TLS connection. - `skip_bucket_public_access_blocking`: When `true`, the S3 bucket that is created will not have public access blocking enabled. diff --git a/test/helpers/package.go b/test/helpers/package.go index f48caba8f0..9113c78e0e 100644 --- a/test/helpers/package.go +++ b/test/helpers/package.go @@ -88,7 +88,10 @@ func CopyEnvironment(t *testing.T, environmentPath string, includeInCopy ...stri t.Logf("Copying %s to %s", environmentPath, tmpDir) - require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy, nil)) + require.NoError( + t, + util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy, nil), + ) return tmpDir } diff --git a/util/file.go b/util/file.go index 8daee9f082..3b6710d305 100644 --- a/util/file.go +++ b/util/file.go @@ -245,6 +245,7 @@ func pathContainsPrefix(path string, prefixes []string) bool { return true } } + return false } @@ -285,7 +286,14 @@ func expandGlobPath(source, absoluteGlobPath string) ([]string, error) { // CopyFolderContents copies the files and folders within the source folder into the destination folder. Note that hidden files and folders // (those starting with a dot) will be skipped. Will create a specified manifest file that contains paths of all copied files. -func CopyFolderContents(logger log.Logger, source, destination, manifestFile string, includeInCopy []string, excludeFromCopy []string) error { +func CopyFolderContents( + logger log.Logger, + source, + destination, + manifestFile string, + includeInCopy []string, + excludeFromCopy []string, +) error { // Expand all the includeInCopy glob paths, converting the globbed results to relative paths so that they work in // the copy filter. includeExpandedGlobs := []string{} @@ -316,10 +324,13 @@ func CopyFolderContents(logger log.Logger, source, destination, manifestFile str return CopyFolderContentsWithFilter(logger, source, destination, manifestFile, func(absolutePath string) bool { relativePath, err := GetPathRelativeTo(absolutePath, source) + pathHasPrefix := pathContainsPrefix(relativePath, excludeExpandedGlobs) - if err == nil && listContainsElementWithPrefix(includeExpandedGlobs, relativePath) && !pathContainsPrefix(relativePath, excludeExpandedGlobs) { + listHasElementWithPrefix := listContainsElementWithPrefix(includeExpandedGlobs, relativePath) + if err == nil && listHasElementWithPrefix && !pathHasPrefix { return true } + if err == nil && pathContainsPrefix(relativePath, excludeExpandedGlobs) { return false } diff --git a/util/file_test.go b/util/file_test.go index edb1d0884a..cd3b6e6004 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -365,7 +365,6 @@ func TestIncludeInCopy(t *testing.T) { require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", includeInCopy, nil)) for i, tt := range tc { - tt := tt t.Run(strconv.Itoa(i), func(t *testing.T) { t.Parallel() @@ -384,7 +383,7 @@ func TestExcludeFromCopy(t *testing.T) { excludeFromCopy := []string{"module/region2", "**/exclude-me-here", "**/app1"} - tc := []struct { + testCases := []struct { path string copyExpected bool }{ @@ -405,7 +404,7 @@ func TestExcludeFromCopy(t *testing.T) { destination := filepath.Join(tempDir, "destination") fileContent := []byte("source file") - for _, tt := range tc { + for _, tt := range testCases { path := filepath.Join(source, tt.path) assert.NoError(t, os.MkdirAll(filepath.Dir(path), os.ModePerm)) assert.NoError(t, os.WriteFile(path, fileContent, 0644)) @@ -413,17 +412,15 @@ func TestExcludeFromCopy(t *testing.T) { require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", nil, excludeFromCopy)) - for i, tt := range tc { - tt := tt - + for i, testCase := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { t.Parallel() - _, err := os.Stat(filepath.Join(destination, tt.path)) + _, err := os.Stat(filepath.Join(destination, testCase.path)) assert.True(t, - tt.copyExpected && err == nil || - !tt.copyExpected && errors.Is(err, os.ErrNotExist), - "Unexpected copy result for file '%s' (should be copied: '%t') - got error: %s", tt.path, tt.copyExpected, err) + testCase.copyExpected && err == nil || + !testCase.copyExpected && errors.Is(err, os.ErrNotExist), + "Unexpected copy result for file '%s' (should be copied: '%t') - got error: %s", testCase.path, testCase.copyExpected, err) }) } } @@ -434,7 +431,7 @@ func TestExcludeIncludeBehaviourPriority(t *testing.T) { includeInCopy := []string{"_module/.region2", "_module/.region3"} excludeFromCopy := []string{"**/.project2-2", "_module/.region3"} - tc := []struct { + testCases := []struct { path string copyExpected bool }{ @@ -449,7 +446,7 @@ func TestExcludeIncludeBehaviourPriority(t *testing.T) { destination := filepath.Join(tempDir, "destination") fileContent := []byte("source file") - for _, tt := range tc { + for _, tt := range testCases { path := filepath.Join(source, tt.path) assert.NoError(t, os.MkdirAll(filepath.Dir(path), os.ModePerm)) assert.NoError(t, os.WriteFile(path, fileContent, 0644)) @@ -457,7 +454,7 @@ func TestExcludeIncludeBehaviourPriority(t *testing.T) { require.NoError(t, util.CopyFolderContents(log.New(), source, destination, ".terragrunt-test", includeInCopy, excludeFromCopy)) - for i, tt := range tc { + for i, tt := range testCases { tt := tt t.Run(strconv.Itoa(i), func(t *testing.T) { From 8c83b114d00da5d5fb794358c3575b36c6f44681 Mon Sep 17 00:00:00 2001 From: Roman Kabaev Date: Wed, 8 Jan 2025 17:52:57 +0300 Subject: [PATCH 4/4] Fixed broken test --- test/fixtures/read-config/full/source.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/fixtures/read-config/full/source.hcl b/test/fixtures/read-config/full/source.hcl index 0835d44691..183ecee33c 100644 --- a/test/fixtures/read-config/full/source.hcl +++ b/test/fixtures/read-config/full/source.hcl @@ -26,6 +26,7 @@ remote_state { terraform { source = "./delorean" include_in_copy = ["time_machine.*"] + exclude_from_copy = ["excluded_time_machine.*"] copy_terraform_lock_file = true extra_arguments "var-files" {