From c3eb11dfd3041d7dcd14921a9e388d0112e898f4 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:56:07 -0500 Subject: [PATCH 01/12] feat: Adding experiment mode --- cli/commands/flags.go | 35 ++++++++ internal/experiment/experiment.go | 113 +++++++++++++++++++++++++ internal/experiment/experiment_test.go | 81 ++++++++++++++++++ options/options.go | 7 ++ 4 files changed, 236 insertions(+) create mode 100644 internal/experiment/experiment.go create mode 100644 internal/experiment/experiment_test.go diff --git a/cli/commands/flags.go b/cli/commands/flags.go index 095d8afb16..35ac2a4862 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -6,6 +6,7 @@ import ( "github.com/gruntwork-io/go-commons/collections" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/internal/strict" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/pkg/cli" @@ -157,6 +158,13 @@ const ( TerragruntStrictControlFlagName = "strict-control" TerragruntStrictControlEnvName = "TERRAGRUNT_STRICT_CONTROL" + // Experiment Mode related flags/envs + TerragruntExperimentModeFlagName = "experiment-mode" + TerragruntExperimentModeEnvName = "TERRAGRUNT_EXPERIMENT_MODE" + + TerragruntExperimentFlagName = "experiment" + TerragruntExperimentEnvName = "TERRAGRUNT_EXPERIMENT" + // Terragrunt Provider Cache related flags/envs TerragruntProviderCacheFlagName = "terragrunt-provider-cache" @@ -519,6 +527,33 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { return nil }, }, + // Experiment Mode flags + &cli.BoolFlag{ + Name: TerragruntExperimentModeFlagName, + EnvVar: TerragruntExperimentModeEnvName, + Destination: &opts.ExperimentMode, + Usage: "Enables experiment mode for Terragrunt. For more information, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode .", + }, + &cli.SliceFlag[string]{ + Name: TerragruntExperimentFlagName, + EnvVar: TerragruntExperimentEnvName, + Usage: "Enables specific experiments. For a list of available experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode .", + Action: func(ctx *cli.Context, val []string) error { + experiments := experiment.NewExperiments() + warning, err := experiments.ValidateExperimentNames(val) + if err != nil { + return cli.NewExitError(err, 1) + } + + if warning != "" { + log.Warn(warning) + } + + opts.Experiments = experiments + + return nil + }, + }, // Terragrunt Provider Cache flags &cli.BoolFlag{ Name: TerragruntProviderCacheFlagName, diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go new file mode 100644 index 0000000000..e36556f468 --- /dev/null +++ b/internal/experiment/experiment.go @@ -0,0 +1,113 @@ +// Package experiment provides utilities used by Terragrunt to support an "experiment" mode. +// By default experiment mode is disabled, but when enabled, experimental features can be enabled. +// These features are not yet stable and may change in the future. +// +// Note that any behavior outlined here should be documented in /docs/_docs/04_reference/experiment-mode.md +// +// That is how users will know what to expect when they enable experiment mode, and how to customize it. +package experiment + +import ( + "strings" +) + +// NewExperiments returns a new Experiments map with all experiments disabled. +// +// Bottom values for each experiment are the defaults, so only the names of experiments need to be set. +func NewExperiments() Experiments { + return Experiments{ + Symlinks: Experiment{ + Name: Symlinks, + }, + } +} + +// Experiment represents an experiment that can be enabled. +// When the experiment is enabled, Terragrunt will behave in a way that uses some experimental functionality. +type Experiment struct { + // Enabled determines if the experiment is enabled. + Enabled bool + // Name is the name of the experiment. + Name string + // Status is the status of the experiment. + Status int +} + +func (e Experiment) String() string { + return e.Name +} + +const ( + // Symlinks is the experiment that allows symlinks to be used in Terragrunt configurations. + Symlinks = "symlinks" +) + +const ( + // StatusOngoing is the status of an experiment that is ongoing. + StatusOngoing = iota + // StatusCompleted is the status of an experiment that is completed. + StatusCompleted +) + +type Experiments map[string]Experiment + +// ValidateExperimentNames validates the given slice of experiment names are valid. +func (e Experiments) ValidateExperimentNames(experimentNames []string) (string, error) { + completedExperiments := []string{} + invalidExperiments := []string{} + + for _, name := range experimentNames { + experiment, ok := e[name] + if !ok { + invalidExperiments = append(invalidExperiments, name) + continue + } + + if experiment.Status == StatusCompleted { + completedExperiments = append(completedExperiments, name) + } + } + + var warning string + if len(completedExperiments) > 0 { + warning = CompletedExperimentsWarning{ + ExperimentNames: completedExperiments, + }.String() + } + + var err error + if len(invalidExperiments) > 0 { + err = InvalidExperimentsError{ + ExperimentNames: invalidExperiments, + } + } + + return warning, err +} + +// CompletedExperimentsWarning is a warning that is returned when completed experiments are requested. +type CompletedExperimentsWarning struct { + ExperimentNames []string +} + +func (e CompletedExperimentsWarning) String() string { + return "The following experiment(s) are already completed: " + strings.Join(e.ExperimentNames, ", ") + ". Please remove any completed experiments, as setting them no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode" +} + +// InvalidExperimentsError is an error that is returned when an invalid experiments are requested. +type InvalidExperimentsError struct { + ExperimentNames []string +} + +func (e InvalidExperimentsError) Error() string { + return "The following experiment(s) are invalid: " + strings.Join(e.ExperimentNames, ", ") + ". For a list of all valid experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode" +} + +// Evaluate returns true if either the experiment is enabled, or experiment mode is enabled. +func (experiment Experiment) Evaluate(experimentMode bool) bool { + if experimentMode { + return true + } + + return experiment.Enabled +} diff --git a/internal/experiment/experiment_test.go b/internal/experiment/experiment_test.go new file mode 100644 index 0000000000..3ec62926ca --- /dev/null +++ b/internal/experiment/experiment_test.go @@ -0,0 +1,81 @@ +package experiment_test + +import ( + "testing" + + "github.com/gruntwork-io/terragrunt/internal/experiment" + "github.com/stretchr/testify/assert" +) + +func TestValidateExperiments(t *testing.T) { + t.Parallel() + + tc := []struct { + name string + experiments experiment.Experiments + experimentNames []string + expectedWarning string + expectedError error + }{ + { + name: "no experiments", + experiments: experiment.NewExperiments(), + experimentNames: []string{}, + expectedWarning: "", + expectedError: nil, + }, + { + name: "valid experiment", + experiments: experiment.NewExperiments(), + experimentNames: []string{experiment.Symlinks}, + expectedWarning: "", + expectedError: nil, + }, + { + name: "invalid experiment", + experiments: experiment.NewExperiments(), + experimentNames: []string{"invalid"}, + expectedWarning: "", + expectedError: experiment.InvalidExperimentsError{ + ExperimentNames: []string{"invalid"}, + }, + }, + { + name: "completed experiment", + experiments: experiment.Experiments{ + experiment.Symlinks: experiment.Experiment{ + Name: experiment.Symlinks, + Status: experiment.StatusCompleted, + }, + }, + experimentNames: []string{experiment.Symlinks}, + expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove this experiment flag, as it no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", + expectedError: nil, + }, + { + name: "invalid and completed experiment", + experiments: experiment.Experiments{ + experiment.Symlinks: experiment.Experiment{ + Name: experiment.Symlinks, + Status: experiment.StatusCompleted, + }, + }, + experimentNames: []string{"invalid", experiment.Symlinks}, + expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove this experiment flag, as it no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", + expectedError: experiment.InvalidExperimentsError{ + ExperimentNames: []string{"invalid"}, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + warning, err := tt.experiments.ValidateExperimentNames(tt.experimentNames) + + assert.Equal(t, tt.expectedWarning, warning) + assert.Equal(t, tt.expectedError, err) + }) + } +} diff --git a/options/options.go b/options/options.go index 9c96433ece..e74578b450 100644 --- a/options/options.go +++ b/options/options.go @@ -13,6 +13,7 @@ import ( "time" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/pkg/log" "github.com/gruntwork-io/terragrunt/pkg/log/format" "github.com/gruntwork-io/terragrunt/pkg/log/format/placeholders" @@ -364,6 +365,12 @@ type TerragruntOptions struct { // StrictControls is a slice of strict controls enabled. StrictControls []string + // ExperimentMode is a flag to enable experiment mode for terragrunt. + ExperimentMode bool + + // Experiments is a map of experiments, and their status. + Experiments experiment.Experiments + // FeatureFlags is a map of feature flags to enable. FeatureFlags map[string]string From 0e273b575515c10e6c17df808f56803b1fb14f38 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:22:20 -0500 Subject: [PATCH 02/12] feat: Reintroducing symlink work behind experiment flag --- cli/commands/catalog/action.go | 6 +- cli/commands/catalog/module/repo.go | 18 +++-- cli/commands/catalog/module/repo_test.go | 2 +- cli/commands/terraform/download_source.go | 18 +++-- config/config.go | 16 +++-- config/config_helpers.go | 6 +- config/dependency.go | 10 ++- config/variable.go | 6 +- internal/experiment/experiment.go | 4 +- options/options.go | 2 + terraform/source.go | 80 ++++++++++++++++------- util/file.go | 11 +++- 12 files changed, 128 insertions(+), 51 deletions(-) diff --git a/cli/commands/catalog/action.go b/cli/commands/catalog/action.go index d3a7aa5ead..f67efbf264 100644 --- a/cli/commands/catalog/action.go +++ b/cli/commands/catalog/action.go @@ -10,6 +10,7 @@ import ( "github.com/gruntwork-io/terragrunt/cli/commands/catalog/tui" "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/util" ) @@ -39,7 +40,10 @@ func Run(ctx context.Context, opts *options.TerragruntOptions, repoURL string) e for _, repoURL := range repoURLs { tempDir := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, util.EncodeBase64Sha1(repoURL))) - repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir) + experiment := opts.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) + + repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir, walkWithSymlinks) if err != nil { return err } diff --git a/cli/commands/catalog/module/repo.go b/cli/commands/catalog/module/repo.go index 22cc3abc45..fe097d69f7 100644 --- a/cli/commands/catalog/module/repo.go +++ b/cli/commands/catalog/module/repo.go @@ -42,13 +42,16 @@ type Repo struct { remoteURL string branchName string + + walkWithSymlinks bool } -func NewRepo(ctx context.Context, logger log.Logger, cloneURL, tempDir string) (*Repo, error) { +func NewRepo(ctx context.Context, logger log.Logger, cloneURL, tempDir string, walkWithSymlinks bool) (*Repo, error) { repo := &Repo{ - logger: logger, - cloneURL: cloneURL, - path: tempDir, + logger: logger, + cloneURL: cloneURL, + path: tempDir, + walkWithSymlinks: walkWithSymlinks, } if err := repo.clone(ctx); err != nil { @@ -84,7 +87,12 @@ func (repo *Repo) FindModules(ctx context.Context) (Modules, error) { continue } - err := util.WalkWithSymlinks(modulesPath, + walkFunc := filepath.Walk + if repo.walkWithSymlinks { + walkFunc = util.WalkWithSymlinks + } + + err := walkFunc(modulesPath, func(dir string, remote os.FileInfo, err error) error { if err != nil { return err diff --git a/cli/commands/catalog/module/repo_test.go b/cli/commands/catalog/module/repo_test.go index 51863d54ed..81e163d30a 100644 --- a/cli/commands/catalog/module/repo_test.go +++ b/cli/commands/catalog/module/repo_test.go @@ -63,7 +63,7 @@ func TestFindModules(t *testing.T) { ctx := context.Background() - repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "") + repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "", false) require.NoError(t, err) modules, err := repo.FindModules(ctx) diff --git a/cli/commands/terraform/download_source.go b/cli/commands/terraform/download_source.go index 9b64350068..c0fa8eb729 100644 --- a/cli/commands/terraform/download_source.go +++ b/cli/commands/terraform/download_source.go @@ -12,6 +12,7 @@ import ( "github.com/gruntwork-io/terragrunt/cli/commands" "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/terraform" "github.com/gruntwork-io/terragrunt/util" @@ -34,17 +35,20 @@ const fileURIScheme = "file://" // // See the NewTerraformSource method for how we determine the temporary folder so we can reuse it across multiple // runs of Terragrunt to avoid downloading everything from scratch every time. -func downloadTerraformSource(ctx context.Context, source string, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (*options.TerragruntOptions, error) { - terraformSource, err := terraform.NewSource(source, terragruntOptions.DownloadDir, terragruntOptions.WorkingDir, terragruntOptions.Logger) +func downloadTerraformSource(ctx context.Context, source string, opts *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (*options.TerragruntOptions, error) { + experiment := opts.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) + + terraformSource, err := terraform.NewSource(source, opts.DownloadDir, opts.WorkingDir, opts.Logger, walkWithSymlinks) if err != nil { return nil, err } - if err := DownloadTerraformSourceIfNecessary(ctx, terraformSource, terragruntOptions, terragruntConfig); err != nil { + if err := DownloadTerraformSourceIfNecessary(ctx, terraformSource, opts, terragruntConfig); err != nil { return nil, err } - terragruntOptions.Logger.Debugf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.WorkingDir) + opts.Logger.Debugf("Copying files from %s into %s", opts.WorkingDir, terraformSource.WorkingDir) var includeInCopy []string if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil { @@ -52,16 +56,16 @@ func downloadTerraformSource(ctx context.Context, source string, terragruntOptio } // Always include the .tflint.hcl file, if it exists includeInCopy = append(includeInCopy, tfLintConfig) - if err := util.CopyFolderContents(terragruntOptions.Logger, terragruntOptions.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil { + if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil { return nil, err } - updatedTerragruntOptions, err := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) + updatedTerragruntOptions, err := opts.Clone(opts.TerragruntConfigPath) if err != nil { return nil, err } - terragruntOptions.Logger.Debugf("Setting working directory to %s", terraformSource.WorkingDir) + opts.Logger.Debugf("Setting working directory to %s", terraformSource.WorkingDir) updatedTerragruntOptions.WorkingDir = terraformSource.WorkingDir return updatedTerragruntOptions, nil diff --git a/config/config.go b/config/config.go index 746d969dc1..c124d95085 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,7 @@ import ( "github.com/gruntwork-io/terragrunt/pkg/log/writer" "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/telemetry" @@ -656,10 +657,17 @@ func GetDefaultConfigPath(workingDir string) string { // FindConfigFilesInPath returns a list of all Terragrunt config files in the given path or any subfolder of the path. A file is a Terragrunt // config file if it has a name as returned by the DefaultConfigPath method -func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { +func FindConfigFilesInPath(rootPath string, opts *options.TerragruntOptions) ([]string, error) { configFiles := []string{} - err := util.WalkWithSymlinks(rootPath, func(path string, info os.FileInfo, err error) error { + experiment := opts.Experiments[experiment.Symlinks] + + walkFunc := filepath.Walk + if experiment.Evaluate(opts.ExperimentMode) { + walkFunc = util.WalkWithSymlinks + } + + err := walkFunc(rootPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -668,13 +676,13 @@ func FindConfigFilesInPath(rootPath string, terragruntOptions *options.Terragrun return nil } - if ok, err := isTerragruntModuleDir(path, terragruntOptions); err != nil { + if ok, err := isTerragruntModuleDir(path, opts); err != nil { return err } else if !ok { return filepath.SkipDir } - for _, configFile := range append(DefaultTerragruntConfigPaths, filepath.Base(terragruntOptions.TerragruntConfigPath)) { + for _, configFile := range append(DefaultTerragruntConfigPaths, filepath.Base(opts.TerragruntConfigPath)) { if !filepath.IsAbs(configFile) { configFile = util.JoinPath(path, configFile) } diff --git a/config/config_helpers.go b/config/config_helpers.go index 1d1bf40fb9..2b67e5a61d 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -24,6 +24,7 @@ import ( "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/internal/locks" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/shell" @@ -573,7 +574,10 @@ func getWorkingDir(ctx *ParsingContext) (string, error) { return ctx.TerragruntOptions.WorkingDir, nil } - source, err := terraform.NewSource(sourceURL, ctx.TerragruntOptions.DownloadDir, ctx.TerragruntOptions.WorkingDir, ctx.TerragruntOptions.Logger) + experiment := ctx.TerragruntOptions.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(ctx.TerragruntOptions.ExperimentMode) + + source, err := terraform.NewSource(sourceURL, ctx.TerragruntOptions.DownloadDir, ctx.TerragruntOptions.WorkingDir, ctx.TerragruntOptions.Logger, walkWithSymlinks) if err != nil { return "", err } diff --git a/config/dependency.go b/config/dependency.go index 83a804074c..69bd799f15 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -14,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -789,7 +790,7 @@ func canGetRemoteState(remoteState *remote.RemoteState) bool { // terragruntAlreadyInit returns true if it detects that the module specified by the given terragrunt configuration is // already initialized with the terraform source. This will also return the working directory where you can run // terraform. -func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configPath string, ctx *ParsingContext) (bool, string, error) { +func terragruntAlreadyInit(opts *options.TerragruntOptions, configPath string, ctx *ParsingContext) (bool, string, error) { // We need to first determine the working directory where the terraform source should be located. This is dependent // on the source field of the terraform block in the config. terraformBlockTGConfig, err := PartialParseConfigFile(ctx.WithDecodeList(TerraformSource), configPath, nil) @@ -799,7 +800,7 @@ func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configP var workingDir string - sourceURL, err := GetTerraformSourceURL(terragruntOptions, terraformBlockTGConfig) + sourceURL, err := GetTerraformSourceURL(opts, terraformBlockTGConfig) if err != nil { return false, "", err } @@ -813,7 +814,10 @@ func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configP workingDir = filepath.Dir(configPath) } } else { - terraformSource, err := terraform.NewSource(sourceURL, terragruntOptions.DownloadDir, terragruntOptions.WorkingDir, terragruntOptions.Logger) + experiment := opts.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) + + terraformSource, err := terraform.NewSource(sourceURL, opts.DownloadDir, opts.WorkingDir, opts.Logger, walkWithSymlinks) if err != nil { return false, "", err } diff --git a/config/variable.go b/config/variable.go index 90e7499ebf..0c8f65615c 100644 --- a/config/variable.go +++ b/config/variable.go @@ -6,6 +6,7 @@ import ( "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/util" "github.com/hashicorp/hcl/v2" @@ -25,8 +26,11 @@ type ParsedVariable struct { // ParseVariables - parse variables from tf files. func ParseVariables(opts *options.TerragruntOptions, directoryPath string) ([]*ParsedVariable, error) { + experiment := opts.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) + // list all tf files - tfFiles, err := util.ListTfFiles(directoryPath) + tfFiles, err := util.ListTfFiles(directoryPath, walkWithSymlinks) if err != nil { return nil, errors.New(err) } diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go index e36556f468..9e31adfb53 100644 --- a/internal/experiment/experiment.go +++ b/internal/experiment/experiment.go @@ -104,10 +104,10 @@ func (e InvalidExperimentsError) Error() string { } // Evaluate returns true if either the experiment is enabled, or experiment mode is enabled. -func (experiment Experiment) Evaluate(experimentMode bool) bool { +func (e Experiment) Evaluate(experimentMode bool) bool { if experimentMode { return true } - return experiment.Enabled + return e.Enabled } diff --git a/options/options.go b/options/options.go index e74578b450..e43a6ad69f 100644 --- a/options/options.go +++ b/options/options.go @@ -493,6 +493,8 @@ func NewTerragruntOptionsWithWriters(stdout, stderr io.Writer) *TerragruntOption JSONOutputFolder: "", FeatureFlags: map[string]string{}, ReadFiles: xsync.NewMapOf[string, []string](), + ExperimentMode: false, + Experiments: experiment.NewExperiments(), } } diff --git a/terraform/source.go b/terraform/source.go index 8df0d940b8..6a2eab5bc8 100644 --- a/terraform/source.go +++ b/terraform/source.go @@ -39,7 +39,11 @@ type Source struct { // The path to a file in DownloadDir that stores the version number of the code VersionFile string + // Logger to use for logging Logger log.Logger + + // WalkWithSymlinks controls whether to walk symlinks in the downloaded source + WalkWithSymlinks bool } func (src *Source) String() string { @@ -58,31 +62,60 @@ func (src Source) EncodeSourceVersion() (string, error) { sourceHash := sha256.New() sourceDir := filepath.Clean(src.CanonicalSourceURL.Path) - err := util.WalkWithSymlinks(sourceDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - // If we've encountered an error while walking the tree, give up - return err - } + var err error + if src.WalkWithSymlinks { + err = util.WalkWithSymlinks(sourceDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If we've encountered an error while walking the tree, give up + return err + } + + if info.IsDir() { + // We don't use any info from directories to calculate our hash + return nil + } + // avoid checking files in .terragrunt-cache directory since contents is auto-generated + if strings.Contains(path, util.TerragruntCacheDir) { + return nil + } + // avoid checking files in .terraform directory since contents is auto-generated + if info.Name() == util.TerraformLockFile { + return nil + } + + fileModified := info.ModTime().UnixMicro() + hashContents := fmt.Sprintf("%s:%d", path, fileModified) + sourceHash.Write([]byte(hashContents)) - if info.IsDir() { - // We don't use any info from directories to calculate our hash - return nil - } - // avoid checking files in .terragrunt-cache directory since contents is auto-generated - if strings.Contains(path, util.TerragruntCacheDir) { - return nil - } - // avoid checking files in .terraform directory since contents is auto-generated - if info.Name() == util.TerraformLockFile { return nil - } + }) + } else { + err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If we've encountered an error while walking the tree, give up + return err + } + + if info.IsDir() { + // We don't use any info from directories to calculate our hash + return nil + } + // avoid checking files in .terragrunt-cache directory since contents is auto-generated + if strings.Contains(path, util.TerragruntCacheDir) { + return nil + } + // avoid checking files in .terraform directory since contents is auto-generated + if info.Name() == util.TerraformLockFile { + return nil + } + + fileModified := info.ModTime().UnixMicro() + hashContents := fmt.Sprintf("%s:%d", path, fileModified) + sourceHash.Write([]byte(hashContents)) - fileModified := info.ModTime().UnixMicro() - hashContents := fmt.Sprintf("%s:%d", path, fileModified) - sourceHash.Write([]byte(hashContents)) - - return nil - }) + return nil + }) + } if err == nil { hash := hex.EncodeToString(sourceHash.Sum(nil)) @@ -146,7 +179,7 @@ func (src Source) WriteVersionFile() error { // 1. Always download source URLs pointing to local file paths. // 2. Only download source URLs pointing to remote paths if /T/W/H doesn't already exist or, if it does exist, if the // version number in /T/W/H/.terragrunt-source-version doesn't match the current version. -func NewSource(source string, downloadDir string, workingDir string, logger log.Logger) (*Source, error) { +func NewSource(source string, downloadDir string, workingDir string, logger log.Logger, walkWithSymlinks bool) (*Source, error) { canonicalWorkingDir, err := util.CanonicalPath(workingDir, "") if err != nil { return nil, err @@ -190,6 +223,7 @@ func NewSource(source string, downloadDir string, workingDir string, logger log. WorkingDir: updatedWorkingDir, VersionFile: versionFile, Logger: logger, + WalkWithSymlinks: walkWithSymlinks, }, nil } diff --git a/util/file.go b/util/file.go index 076530b966..647641a845 100644 --- a/util/file.go +++ b/util/file.go @@ -637,10 +637,15 @@ func (err PathIsNotFile) Error() string { } // ListTfFiles returns a list of all TF files in the specified directory. -func ListTfFiles(directoryPath string) ([]string, error) { +func ListTfFiles(directoryPath string, walkWithSymlinks bool) ([]string, error) { var tfFiles []string - err := WalkWithSymlinks(directoryPath, func(path string, info os.FileInfo, err error) error { + walkFunc := filepath.Walk + if walkWithSymlinks { + walkFunc = WalkWithSymlinks + } + + err := walkFunc(directoryPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -651,8 +656,8 @@ func ListTfFiles(directoryPath string) ([]string, error) { return nil }) - return tfFiles, err + } // IsDirectoryEmpty - returns true if the given path exists and is a empty directory. From bdb4e54412d6d170d6f440c1b0e715882a1cd2f1 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:34:27 -0500 Subject: [PATCH 03/12] fix: Fixing `NewRepo` usage --- test/integration_catalog_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration_catalog_test.go b/test/integration_catalog_test.go index 0bdd7e6aab..ee1f8a2d78 100644 --- a/test/integration_catalog_test.go +++ b/test/integration_catalog_test.go @@ -22,10 +22,10 @@ func TestCatalogGitRepoUpdate(t *testing.T) { tempDir := t.TempDir() - _, err := module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir) + _, err := module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir, false) require.NoError(t, err) - _, err = module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir) + _, err = module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir, false) require.NoError(t, err) } @@ -36,7 +36,7 @@ func TestScaffoldGitRepo(t *testing.T) { tempDir := t.TempDir() - repo, err := module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir) + repo, err := module.NewRepo(ctx, log.New(), "github.com/gruntwork-io/terraform-fake-modules.git", tempDir, false) require.NoError(t, err) modules, err := repo.FindModules(ctx) @@ -51,7 +51,7 @@ func TestScaffoldGitModule(t *testing.T) { tempDir := t.TempDir() - repo, err := module.NewRepo(ctx, log.New(), "https://github.com/gruntwork-io/terraform-fake-modules.git", tempDir) + repo, err := module.NewRepo(ctx, log.New(), "https://github.com/gruntwork-io/terraform-fake-modules.git", tempDir, false) require.NoError(t, err) modules, err := repo.FindModules(ctx) @@ -89,7 +89,7 @@ func TestScaffoldGitModuleHttps(t *testing.T) { tempDir := t.TempDir() - repo, err := module.NewRepo(ctx, log.New(), "https://github.com/gruntwork-io/terraform-fake-modules", tempDir) + repo, err := module.NewRepo(ctx, log.New(), "https://github.com/gruntwork-io/terraform-fake-modules", tempDir, false) require.NoError(t, err) modules, err := repo.FindModules(ctx) From 90bb8589507ca5c9d4d9382673bfdc6d0c99ef39 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:39:44 -0500 Subject: [PATCH 04/12] fix: Fixing `NewSource` usage --- test/integration_download_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration_download_test.go b/test/integration_download_test.go index d7b3fcf111..e9124e635c 100644 --- a/test/integration_download_test.go +++ b/test/integration_download_test.go @@ -236,7 +236,7 @@ func TestCustomLockFile(t *testing.T) { source := "../custom-lock-file-module" downloadDir := util.JoinPath(rootPath, helpers.TerragruntCache) - result, err := tfsource.NewSource(source, downloadDir, rootPath, createLogger()) + result, err := tfsource.NewSource(source, downloadDir, rootPath, createLogger(), false) require.NoError(t, err) lockFilePath := util.JoinPath(result.WorkingDir, util.TerraformLockFile) From fbb832ba073630211015c6f5978e4c73c5f7a960 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:45:00 -0500 Subject: [PATCH 05/12] fix: Updating symlink tests now that they're red --- test/integration_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index fc023e471c..8bbb3c3338 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -820,21 +820,21 @@ func TestTerragruntStackCommandsWithSymlinks(t *testing.T) { helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath) // perform the first initialization - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --experiment symlinks --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) require.NoError(t, err) assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache") // perform the second initialization and make sure that the cache is not downloaded again - _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --experiment symlinks --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) require.NoError(t, err) assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache") // validate the modules - _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all validate --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all validate --experiment symlinks --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) require.NoError(t, err) assert.Contains(t, stderr, "Module ./a") assert.Contains(t, stderr, "Module ./b") @@ -844,7 +844,7 @@ func TestTerragruntStackCommandsWithSymlinks(t *testing.T) { require.NoError(t, os.Chtimes(util.JoinPath(disjointSymlinksEnvironmentPath, "module/main.tf"), time.Now(), time.Now())) // perform the initialization and make sure that the cache is downloaded again - _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --experiment symlinks --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) require.NoError(t, err) assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") @@ -873,12 +873,12 @@ func TestTerragruntOutputModuleGroupsWithSymlinks(t *testing.T) { }`, disjointSymlinksEnvironmentPath) helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath) - stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output-module-groups --terragrunt-working-dir %s apply", disjointSymlinksEnvironmentPath)) + stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output-module-groups --experiment symlinks --terragrunt-working-dir %s apply", disjointSymlinksEnvironmentPath)) require.NoError(t, err) output := strings.ReplaceAll(stdout, " ", "") expectedOutput := strings.ReplaceAll(strings.ReplaceAll(expectedApplyOutput, "\t", ""), " ", "") - assert.True(t, strings.Contains(strings.TrimSpace(output), strings.TrimSpace(expectedOutput))) + assert.Contains(t, strings.TrimSpace(output), strings.TrimSpace(expectedOutput)) } func TestInvalidSource(t *testing.T) { From e6820a693a212bc9f3663b1f05c44f927c08d11c Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:25:12 -0500 Subject: [PATCH 06/12] fix: Linting --- util/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/file.go b/util/file.go index 647641a845..29ef468739 100644 --- a/util/file.go +++ b/util/file.go @@ -656,8 +656,8 @@ func ListTfFiles(directoryPath string, walkWithSymlinks bool) ([]string, error) return nil }) - return tfFiles, err + return tfFiles, err } // IsDirectoryEmpty - returns true if the given path exists and is a empty directory. From c8c4602535ebbd3681b22e90b9534028df14a1a0 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:25:03 -0500 Subject: [PATCH 07/12] fix: Actually set experiments after flag parsing... --- cli/commands/flags.go | 4 ++++ internal/experiment/experiment.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cli/commands/flags.go b/cli/commands/flags.go index 35ac2a4862..96d3057264 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -549,6 +549,10 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { log.Warn(warning) } + if err := experiments.EnableExperiments(val); err != nil { + return cli.NewExitError(err, 1) + } + opts.Experiments = experiments return nil diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go index 9e31adfb53..6b148c8c49 100644 --- a/internal/experiment/experiment.go +++ b/internal/experiment/experiment.go @@ -52,12 +52,12 @@ const ( type Experiments map[string]Experiment // ValidateExperimentNames validates the given slice of experiment names are valid. -func (e Experiments) ValidateExperimentNames(experimentNames []string) (string, error) { +func (e *Experiments) ValidateExperimentNames(experimentNames []string) (string, error) { completedExperiments := []string{} invalidExperiments := []string{} for _, name := range experimentNames { - experiment, ok := e[name] + experiment, ok := (*e)[name] if !ok { invalidExperiments = append(invalidExperiments, name) continue @@ -85,6 +85,30 @@ func (e Experiments) ValidateExperimentNames(experimentNames []string) (string, return warning, err } +// EnableExperiments enables the given experiments. +func (e *Experiments) EnableExperiments(experimentNames []string) error { + invalidExperiments := []string{} + + for _, name := range experimentNames { + experiment, ok := (*e)[name] + if !ok { + invalidExperiments = append(invalidExperiments, name) + continue + } + + experiment.Enabled = true + (*e)[name] = experiment + } + + if len(invalidExperiments) > 0 { + return InvalidExperimentsError{ + ExperimentNames: invalidExperiments, + } + } + + return nil +} + // CompletedExperimentsWarning is a warning that is returned when completed experiments are requested. type CompletedExperimentsWarning struct { ExperimentNames []string From 943bf49a4c231c5981ff303b0786a49992f81a42 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:34:41 -0500 Subject: [PATCH 08/12] fix: Fixing experiments test --- internal/experiment/experiment_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/experiment/experiment_test.go b/internal/experiment/experiment_test.go index 3ec62926ca..033d33040b 100644 --- a/internal/experiment/experiment_test.go +++ b/internal/experiment/experiment_test.go @@ -49,7 +49,7 @@ func TestValidateExperiments(t *testing.T) { }, }, experimentNames: []string{experiment.Symlinks}, - expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove this experiment flag, as it no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", + expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove any completed experiments, as setting them no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", expectedError: nil, }, { @@ -61,7 +61,7 @@ func TestValidateExperiments(t *testing.T) { }, }, experimentNames: []string{"invalid", experiment.Symlinks}, - expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove this experiment flag, as it no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", + expectedWarning: "The following experiment(s) are already completed: symlinks. Please remove any completed experiments, as setting them no longer does anything. For a list of all ongoing experiments, and the outcomes of previous experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode", expectedError: experiment.InvalidExperimentsError{ ExperimentNames: []string{"invalid"}, }, From eeedca66321415b46d159b096d4ed5978fa6446d Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:52:12 -0500 Subject: [PATCH 09/12] fix: Reintroducing symlink work behind experiment flag --- docs/_docs/04_reference/cli-options.md | 41 +++++++++ docs/_docs/04_reference/experiments.md | 87 +++++++++++++++++++ docs/_docs/04_reference/strict-mode.md | 45 ++++++++-- docs/_docs/04_reference/supported-versions.md | 2 +- 4 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 docs/_docs/04_reference/experiments.md diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 2938b355af..69f2a9bab8 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -95,6 +95,10 @@ This page documents the CLI commands and options available with Terragrunt: - [terragrunt-forward-tf-stdout](#terragrunt-forward-tf-stdout) - [terragrunt-no-destroy-dependencies-check](#terragrunt-no-destroy-dependencies-check) - [feature](#feature) + - [experiment](#experiment) + - [experiment-mode](#experiment-mode) + - [strict-control](#strict-control) + - [strict-mode](#strict-mode) ## CLI commands @@ -1759,3 +1763,40 @@ Setting feature flags through environment variables: export TERRAGRUNT_FEATURE=int_feature_flag=123,bool_feature_flag=true,string_feature_flag=app1 terragrunt apply ``` + +### experiment + +**CLI Arg**: `--experiment`
+**Environment Variable**: `TERRAGRUNT_EXPERIMENT`
+ +Enable experimental features in Terragrunt before they're stable. + +For more information, see the [Experiments](/docs/reference/experiments) documentation. + +### experiment-mode + +**CLI Arg**: `--experiment-mode`
+**Environment Variable**: `TERRAGRUNT_EXPERIMENT_MODE`
+ +Enable all experimental features in Terragrunt before they're stable. + +For more information, see the [Experiments](/docs/reference/experiments) documentation. + +### strict-control + +**CLI Arg**: `--strict-control`
+**Environment Variable**: `TERRAGRUNT_STRICT_CONTROL`
+ +Enable strict controls that opt-in future breaking changes in Terragrunt. + +For more information, see the [Strict Mode](/docs/reference/strict-mode) documentation. + +### strict-mode + +**CLI Arg**: `--strict-mode`
+**Environment Variable**: `TERRAGRUNT_STRICT_MODE`
+ +Enable all strict controls that opt-in future breaking changes in Terragrunt. + +For more information, see the [Strict Mode](/docs/reference/strict-mode) documentation. + diff --git a/docs/_docs/04_reference/experiments.md b/docs/_docs/04_reference/experiments.md new file mode 100644 index 0000000000..753fd3fb61 --- /dev/null +++ b/docs/_docs/04_reference/experiments.md @@ -0,0 +1,87 @@ +--- +layout: collection-browser-doc +title: Experiments +category: reference +categories_url: reference +excerpt: >- + Opt-in to experimental features before they're stable. +tags: ["CLI"] +order: 405 +nav_title: Documentation +nav_title_link: /docs/ +--- + +Terragrunt supports operating in a mode referred to as "Experiment Mode". + +Experiment Mode is a set of controls that can be enabled to opt-in to experimental features before they're stable. +These features are subject to change and may be removed or altered at any time. +They generally provide early access to new features or changes that are being considered for inclusion in future releases. + +Those experiments will be documented here so that you know the following: +1. What the experiment is. +2. What the experiment does. +3. How to provide feedback on the experiment. +4. What criteria must be met for the experiment to be considered stable. + +Sometimes, the criteria for an experiment to be considered stable is unknown, as there may not be a clear path to stabilization. In that case, this will be noted in the experiment documentation, and collaboration with the community will be encouraged to help determine the future of the experiment. + +## Controlling Experiment Mode + +The simplest way to enable experiment mode is to set the [experiment-mode](/docs/reference/cli-options/#experiment-mode) flag. + +This will enable experiment mode for all Terragrunt commands, for all experiments (note that this isn't generally recommended, unless you are following Terragrunt development closely and are prepared for the possibility of breaking changes). + +```bash +$ terragrunt plan --experiment-mode +``` + +You can also use the environment variable, which can be more useful in CI/CD pipelines: + +```bash +$ TERRAGRUNT_EXPERIMENT_MODE='true' terragrunt plan +``` + +Instead of enabling experiment mode, you can also enable specific experiments by setting the [experiment](/docs/reference/cli-options/#experiment) +flag to a value that's specific to a experiment. +This can allow you to experiment with a specific unstable feature that you think might be useful to you. + +```bash +$ terragrunt plan --experiment symlinks +``` + +Again, you can also use the environment variable, which can be more useful in CI/CD pipelines: + +```bash +$ TERRAGRUNT_EXPERIMENT='symlinks' terragrunt plan +``` + +You can also enable multiple experiments at once with a comma delimited list. + +**TODO**: Will add an example here once there's more than one officially supported experiment. The existing experiments are scattered throughout configuration, so they need to be pulled into this system first. + +## Active Experiments + +The following strict mode controls are available: + +- [symlinks](#symlinks) + +### symlinks + +Support symlink resolution for Terragrunt units. + +#### What it does + +By default, Terragrunt will ignore symlinks when determining which units it should run. By enabling this experiment, Terragrunt will resolve symlinks and add them to the list of units being run. + +#### How to provide feedback + +Provide your feedback on the [Experiment: Symlinks](https://github.com/gruntwork-io/terragrunt/discussions/3671) discussion. + +#### Criteria for stabilization + +To stabilize this feature, the following need to be resolved, at a minimum: + +- [ ] Ensure that symlink support continues to work for users referencing symlinks in flags. See [#3622](https://github.com/gruntwork-io/terragrunt/issues/3622). + - [ ] Add integration tests for all filesystem flags to confirm support with symlinks (or document the fact that they cannot be supported). +- [ ] Ensure that MacOS integration tests still work. See [#3616](https://github.com/gruntwork-io/terragrunt/issues/3616). + - [ ] Add integration tests for MacOS in CI. diff --git a/docs/_docs/04_reference/strict-mode.md b/docs/_docs/04_reference/strict-mode.md index 330036a1d0..ae75fc69e8 100644 --- a/docs/_docs/04_reference/strict-mode.md +++ b/docs/_docs/04_reference/strict-mode.md @@ -21,9 +21,16 @@ future versions of Terragrunt. Whenever possible, Terragrunt will initially provide you with a warning when you use a deprecated feature, without throwing an error. However, in Strict Mode, these warnings will be converted to errors, which will cause the Terragrunt command to fail. +A good practice for using strict controls is to enable Strict Mode in your CI/CD pipelines for lower environments +to catch any deprecated features early on. This allows you to fix them before they become a problem +in production in a future Terragrunt release. + +If you are unsure about the impact of enabling strict controls, you can enable them for specific controls to +gradually increase your confidence in the future compatibility of your Terragrunt usage. + ## Controlling Strict Mode -The simplest way to enable strict mode is to set the `TERRAGRUNT_STRICT_MODE` environment variable to `true`. +The simplest way to enable strict mode is to set the [strict-mode](/docs/reference/cli-options/#strict-mode) flag. This will enable strict mode for all Terragrunt commands, for all strict mode controls. @@ -32,26 +39,54 @@ $ terragrunt plan-all 15:26:08.585 WARN The `plan-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead. ``` +```bash +$ terragrunt --strict-mode plan-all +15:26:23.685 ERROR The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead. +``` + +You can also use the environment variable, which can be more useful in CI/CD pipelines: + ```bash $ TERRAGRUNT_STRICT_MODE='true' terragrunt plan-all 15:26:23.685 ERROR The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead. ``` -Instead of setting this environment variable, you can also enable strict mode for specific controls by setting the `TERRAGRUNT_STRICT_CONTROL` -environment variable to a value that's specific to a particular strict control. +Instead of enabling strict mode like this, you can also enable specific strict controls by setting the [strict-control](/docs/reference/cli-options/#strict-control) +flag to a value that's specific to a particular strict control. This can allow you to gradually increase your confidence in the future compatibility of your Terragrunt usage. ```bash -$ TERRAGRUNT_STRICT_CONTROL='apply-all' terragrunt plan-all +$ terragrunt plan-all --strict-control apply-all 15:26:08.585 WARN The `plan-all` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead. ``` +```bash +$ terragrunt plan-all --strict-control plan-all +15:26:23.685 ERROR The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead. +``` + +Again, you can also use the environment variable, which might be more useful in CI/CD pipelines: + ```bash $ TERRAGRUNT_STRICT_CONTROL='plan-all' terragrunt plan-all 15:26:23.685 ERROR The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead. ``` -You can also enable multiple strict controls at once with a comma delimited list. +You can enable multiple strict controls at once: + +```bash +$ terragrunt plan-all --strict-control plan-all --strict-control apply-all +15:26:23.685 ERROR The `plan-all` command is no longer supported. Use `terragrunt run-all plan` instead. +15:26:46.521 ERROR Unable to determine underlying exit code, so Terragrunt will exit with error code 1 +``` + +```bash +$ terragrunt apply-all --strict-control plan-all --strict-control apply-all +15:26:46.564 ERROR The `apply-all` command is no longer supported. Use `terragrunt run-all apply` instead. +15:26:46.564 ERROR Unable to determine underlying exit code, so Terragrunt will exit with error code 1 +``` + +You can also enable multiple strict controls at once when using the environment variable by using a comma delimited list. ```bash $ TERRAGRUNT_STRICT_CONTROL='plan-all,apply-all' bash -c 'terragrunt plan-all; terragrunt apply-all' diff --git a/docs/_docs/04_reference/supported-versions.md b/docs/_docs/04_reference/supported-versions.md index bdaa462a29..487ca449a6 100644 --- a/docs/_docs/04_reference/supported-versions.md +++ b/docs/_docs/04_reference/supported-versions.md @@ -5,7 +5,7 @@ category: reference categories_url: reference excerpt: Learn which Terraform and OpenTofu versions are compatible with which versions of Terragrunt. tags: [ "install" ] -order: 405 +order: 406 nav_title: Documentation nav_title_link: /docs/ --- From c489f59295ccbe553baabf445859f0dec4451e34 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:57:58 -0500 Subject: [PATCH 10/12] fix: Markdown linting --- docs/_docs/04_reference/cli-options.md | 1 - docs/_docs/04_reference/experiments.md | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 69f2a9bab8..ddf4f40b2d 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -1799,4 +1799,3 @@ For more information, see the [Strict Mode](/docs/reference/strict-mode) documen Enable all strict controls that opt-in future breaking changes in Terragrunt. For more information, see the [Strict Mode](/docs/reference/strict-mode) documentation. - diff --git a/docs/_docs/04_reference/experiments.md b/docs/_docs/04_reference/experiments.md index 753fd3fb61..6ad063d26b 100644 --- a/docs/_docs/04_reference/experiments.md +++ b/docs/_docs/04_reference/experiments.md @@ -18,6 +18,7 @@ These features are subject to change and may be removed or altered at any time. They generally provide early access to new features or changes that are being considered for inclusion in future releases. Those experiments will be documented here so that you know the following: + 1. What the experiment is. 2. What the experiment does. 3. How to provide feedback on the experiment. @@ -32,13 +33,13 @@ The simplest way to enable experiment mode is to set the [experiment-mode](/docs This will enable experiment mode for all Terragrunt commands, for all experiments (note that this isn't generally recommended, unless you are following Terragrunt development closely and are prepared for the possibility of breaking changes). ```bash -$ terragrunt plan --experiment-mode +terragrunt plan --experiment-mode ``` You can also use the environment variable, which can be more useful in CI/CD pipelines: ```bash -$ TERRAGRUNT_EXPERIMENT_MODE='true' terragrunt plan +TERRAGRUNT_EXPERIMENT_MODE='true' terragrunt plan ``` Instead of enabling experiment mode, you can also enable specific experiments by setting the [experiment](/docs/reference/cli-options/#experiment) @@ -46,13 +47,13 @@ flag to a value that's specific to a experiment. This can allow you to experiment with a specific unstable feature that you think might be useful to you. ```bash -$ terragrunt plan --experiment symlinks +terragrunt plan --experiment symlinks ``` Again, you can also use the environment variable, which can be more useful in CI/CD pipelines: ```bash -$ TERRAGRUNT_EXPERIMENT='symlinks' terragrunt plan +TERRAGRUNT_EXPERIMENT='symlinks' terragrunt plan ``` You can also enable multiple experiments at once with a comma delimited list. From 950638ae705b8f5ac1bcc19e2d4461bd6b847ac6 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:53:59 -0500 Subject: [PATCH 11/12] fix: Addressing review feedback --- cli/commands/catalog/action.go | 6 +++--- internal/experiment/experiment.go | 10 ++++++---- options/options.go | 5 +++++ terraform/source.go | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cli/commands/catalog/action.go b/cli/commands/catalog/action.go index f67efbf264..d5a60b9341 100644 --- a/cli/commands/catalog/action.go +++ b/cli/commands/catalog/action.go @@ -37,12 +37,12 @@ func Run(ctx context.Context, opts *options.TerragruntOptions, repoURL string) e var modules module.Modules + experiment := opts.Experiments[experiment.Symlinks] + walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) + for _, repoURL := range repoURLs { tempDir := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, util.EncodeBase64Sha1(repoURL))) - experiment := opts.Experiments[experiment.Symlinks] - walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode) - repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir, walkWithSymlinks) if err != nil { return err diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go index 6b148c8c49..36abce36d4 100644 --- a/internal/experiment/experiment.go +++ b/internal/experiment/experiment.go @@ -9,6 +9,8 @@ package experiment import ( "strings" + + "github.com/gruntwork-io/terragrunt/internal/errors" ) // NewExperiments returns a new Experiments map with all experiments disabled. @@ -77,9 +79,9 @@ func (e *Experiments) ValidateExperimentNames(experimentNames []string) (string, var err error if len(invalidExperiments) > 0 { - err = InvalidExperimentsError{ + err = errors.New(InvalidExperimentsError{ ExperimentNames: invalidExperiments, - } + }) } return warning, err @@ -101,9 +103,9 @@ func (e *Experiments) EnableExperiments(experimentNames []string) error { } if len(invalidExperiments) > 0 { - return InvalidExperimentsError{ + return errors.New(InvalidExperimentsError{ ExperimentNames: invalidExperiments, - } + }) } return nil diff --git a/options/options.go b/options/options.go index e43a6ad69f..182977e911 100644 --- a/options/options.go +++ b/options/options.go @@ -655,6 +655,11 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp EngineLogLevel: opts.EngineLogLevel, EngineSkipChecksumCheck: opts.EngineSkipChecksumCheck, Engine: cloneEngineOptions(opts.Engine), + ExperimentMode: opts.ExperimentMode, + // This doesn't have to be deep cloned, as the same experiments + // are used across all units in a `run-all`. If that changes in + // the future, we can deep clone this as well. + Experiments: opts.Experiments, // copy array StrictControls: util.CloneStringList(opts.StrictControls), FeatureFlags: opts.FeatureFlags, diff --git a/terraform/source.go b/terraform/source.go index 6a2eab5bc8..96aefdf3a6 100644 --- a/terraform/source.go +++ b/terraform/source.go @@ -46,7 +46,7 @@ type Source struct { WalkWithSymlinks bool } -func (src *Source) String() string { +func (src Source) String() string { return fmt.Sprintf("Source{CanonicalSourceURL = %v, DownloadDir = %v, WorkingDir = %v, VersionFile = %v}", src.CanonicalSourceURL, src.DownloadDir, src.WorkingDir, src.VersionFile) } From 28fab5ac02586a4c71da32de50bfb6a7b26b3c74 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:24:49 -0500 Subject: [PATCH 12/12] fix: Just check the string, even though I don't think that's right --- internal/experiment/experiment_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/experiment/experiment_test.go b/internal/experiment/experiment_test.go index 033d33040b..77cc85ae61 100644 --- a/internal/experiment/experiment_test.go +++ b/internal/experiment/experiment_test.go @@ -5,6 +5,7 @@ import ( "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestValidateExperiments(t *testing.T) { @@ -75,7 +76,12 @@ func TestValidateExperiments(t *testing.T) { warning, err := tt.experiments.ValidateExperimentNames(tt.experimentNames) assert.Equal(t, tt.expectedWarning, warning) - assert.Equal(t, tt.expectedError, err) + + if tt.expectedError != nil { + require.EqualError(t, err, tt.expectedError.Error()) + } else { + require.NoError(t, err) + } }) } }