fix: Reintroduce symlink work behind experiment flag

cli/commands/catalog/action.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/cli/commands/catalog/tui"
 	"github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/util"
 )
@@ -36,10 +37,13 @@ func Run(ctx context.Context, opts *options.TerragruntOptions, repoURL string) e
 
 	var modules module.Modules
 
+	experiment := opts.Experiments[experiment.Symlinks]
+	walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode)
+
 	for _, repoURL := range repoURLs {
 		tempDir := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, util.EncodeBase64Sha1(repoURL)))
 
-		repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir)
+		repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir, walkWithSymlinks)
 		if err != nil {
 			return err
 		}

cli/commands/catalog/module/repo.go

@@ -42,13 +42,16 @@ type Repo struct {
 
 	remoteURL  string
 	branchName string
+
+	walkWithSymlinks bool
 }
 
-func NewRepo(ctx context.Context, logger log.Logger, cloneURL, tempDir string) (*Repo, error) {
+func NewRepo(ctx context.Context, logger log.Logger, cloneURL, tempDir string, walkWithSymlinks bool) (*Repo, error) {
 	repo := &Repo{
-		logger:   logger,
-		cloneURL: cloneURL,
-		path:     tempDir,
+		logger:           logger,
+		cloneURL:         cloneURL,
+		path:             tempDir,
+		walkWithSymlinks: walkWithSymlinks,
 	}
 
 	if err := repo.clone(ctx); err != nil {
@@ -84,7 +87,12 @@ func (repo *Repo) FindModules(ctx context.Context) (Modules, error) {
 			continue
 		}
 
-		err := util.WalkWithSymlinks(modulesPath,
+		walkFunc := filepath.Walk
+		if repo.walkWithSymlinks {
+			walkFunc = util.WalkWithSymlinks
+		}
+
+		err := walkFunc(modulesPath,
 			func(dir string, remote os.FileInfo, err error) error {
 				if err != nil {
 					return err

cli/commands/catalog/module/repo_test.go

@@ -63,7 +63,7 @@ func TestFindModules(t *testing.T) {
 
 			ctx := context.Background()
 
-			repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "")
+			repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "", false)
 			require.NoError(t, err)
 
 			modules, err := repo.FindModules(ctx)

cli/commands/flags.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gruntwork-io/go-commons/collections"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/internal/strict"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/pkg/cli"
@@ -157,6 +158,13 @@ const (
 	TerragruntStrictControlFlagName = "strict-control"
 	TerragruntStrictControlEnvName  = "TERRAGRUNT_STRICT_CONTROL"
 
+	// Experiment Mode related flags/envs
+	TerragruntExperimentModeFlagName = "experiment-mode"
+	TerragruntExperimentModeEnvName  = "TERRAGRUNT_EXPERIMENT_MODE"
+
+	TerragruntExperimentFlagName = "experiment"
+	TerragruntExperimentEnvName  = "TERRAGRUNT_EXPERIMENT"
+
 	// Terragrunt Provider Cache related flags/envs
 
 	TerragruntProviderCacheFlagName = "terragrunt-provider-cache"
@@ -519,6 +527,37 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
 				return nil
 			},
 		},
+		// Experiment Mode flags
+		&cli.BoolFlag{
+			Name:        TerragruntExperimentModeFlagName,
+			EnvVar:      TerragruntExperimentModeEnvName,
+			Destination: &opts.ExperimentMode,
+			Usage:       "Enables experiment mode for Terragrunt. For more information, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode .",
+		},
+		&cli.SliceFlag[string]{
+			Name:   TerragruntExperimentFlagName,
+			EnvVar: TerragruntExperimentEnvName,
+			Usage:  "Enables specific experiments. For a list of available experiments, see https://terragrunt.gruntwork.io/docs/reference/experiment-mode .",
+			Action: func(ctx *cli.Context, val []string) error {
+				experiments := experiment.NewExperiments()
+				warning, err := experiments.ValidateExperimentNames(val)
+				if err != nil {
+					return cli.NewExitError(err, 1)
+				}
+
+				if warning != "" {
+					log.Warn(warning)
+				}
+
+				if err := experiments.EnableExperiments(val); err != nil {
+					return cli.NewExitError(err, 1)
+				}
+
+				opts.Experiments = experiments
+
+				return nil
+			},
+		},
 		// Terragrunt Provider Cache flags
 		&cli.BoolFlag{
 			Name:        TerragruntProviderCacheFlagName,

cli/commands/terraform/download_source.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/cli/commands"
 	"github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/terraform"
 	"github.com/gruntwork-io/terragrunt/util"
@@ -34,34 +35,37 @@ const fileURIScheme = "file://"
 //
 // See the NewTerraformSource method for how we determine the temporary folder so we can reuse it across multiple
 // runs of Terragrunt to avoid downloading everything from scratch every time.
-func downloadTerraformSource(ctx context.Context, source string, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (*options.TerragruntOptions, error) {
-	terraformSource, err := terraform.NewSource(source, terragruntOptions.DownloadDir, terragruntOptions.WorkingDir, terragruntOptions.Logger)
+func downloadTerraformSource(ctx context.Context, source string, opts *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) (*options.TerragruntOptions, error) {
+	experiment := opts.Experiments[experiment.Symlinks]
+	walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode)
+
+	terraformSource, err := terraform.NewSource(source, opts.DownloadDir, opts.WorkingDir, opts.Logger, walkWithSymlinks)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := DownloadTerraformSourceIfNecessary(ctx, terraformSource, terragruntOptions, terragruntConfig); err != nil {
+	if err := DownloadTerraformSourceIfNecessary(ctx, terraformSource, opts, terragruntConfig); err != nil {
 		return nil, err
 	}
 
-	terragruntOptions.Logger.Debugf("Copying files from %s into %s", terragruntOptions.WorkingDir, terraformSource.WorkingDir)
+	opts.Logger.Debugf("Copying files from %s into %s", opts.WorkingDir, terraformSource.WorkingDir)
 
 	var includeInCopy []string
 	if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil {
 		includeInCopy = *terragruntConfig.Terraform.IncludeInCopy
 	}
 	// Always include the .tflint.hcl file, if it exists
 	includeInCopy = append(includeInCopy, tfLintConfig)
-	if err := util.CopyFolderContents(terragruntOptions.Logger, terragruntOptions.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil {
+	if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil {
 		return nil, err
 	}
 
-	updatedTerragruntOptions, err := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
+	updatedTerragruntOptions, err := opts.Clone(opts.TerragruntConfigPath)
 	if err != nil {
 		return nil, err
 	}
 
-	terragruntOptions.Logger.Debugf("Setting working directory to %s", terraformSource.WorkingDir)
+	opts.Logger.Debugf("Setting working directory to %s", terraformSource.WorkingDir)
 	updatedTerragruntOptions.WorkingDir = terraformSource.WorkingDir
 
 	return updatedTerragruntOptions, nil

config/config.go

@@ -16,6 +16,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/pkg/log/writer"
 
 	"github.com/gruntwork-io/terragrunt/internal/cache"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/telemetry"
 
@@ -656,10 +657,17 @@ func GetDefaultConfigPath(workingDir string) string {
 
 // FindConfigFilesInPath returns a list of all Terragrunt config files in the given path or any subfolder of the path. A file is a Terragrunt
 // config file if it has a name as returned by the DefaultConfigPath method
-func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) {
+func FindConfigFilesInPath(rootPath string, opts *options.TerragruntOptions) ([]string, error) {
 	configFiles := []string{}
 
-	err := util.WalkWithSymlinks(rootPath, func(path string, info os.FileInfo, err error) error {
+	experiment := opts.Experiments[experiment.Symlinks]
+
+	walkFunc := filepath.Walk
+	if experiment.Evaluate(opts.ExperimentMode) {
+		walkFunc = util.WalkWithSymlinks
+	}
+
+	err := walkFunc(rootPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -668,13 +676,13 @@ func FindConfigFilesInPath(rootPath string, terragruntOptions *options.Terragrun
 			return nil
 		}
 
-		if ok, err := isTerragruntModuleDir(path, terragruntOptions); err != nil {
+		if ok, err := isTerragruntModuleDir(path, opts); err != nil {
 			return err
 		} else if !ok {
 			return filepath.SkipDir
 		}
 
-		for _, configFile := range append(DefaultTerragruntConfigPaths, filepath.Base(terragruntOptions.TerragruntConfigPath)) {
+		for _, configFile := range append(DefaultTerragruntConfigPaths, filepath.Base(opts.TerragruntConfigPath)) {
 			if !filepath.IsAbs(configFile) {
 				configFile = util.JoinPath(path, configFile)
 			}

config/config_helpers.go

@@ -24,6 +24,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/config/hclparse"
 	"github.com/gruntwork-io/terragrunt/internal/cache"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/internal/locks"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/shell"
@@ -573,7 +574,10 @@ func getWorkingDir(ctx *ParsingContext) (string, error) {
 		return ctx.TerragruntOptions.WorkingDir, nil
 	}
 
-	source, err := terraform.NewSource(sourceURL, ctx.TerragruntOptions.DownloadDir, ctx.TerragruntOptions.WorkingDir, ctx.TerragruntOptions.Logger)
+	experiment := ctx.TerragruntOptions.Experiments[experiment.Symlinks]
+	walkWithSymlinks := experiment.Evaluate(ctx.TerragruntOptions.ExperimentMode)
+
+	source, err := terraform.NewSource(sourceURL, ctx.TerragruntOptions.DownloadDir, ctx.TerragruntOptions.WorkingDir, ctx.TerragruntOptions.Logger, walkWithSymlinks)
 	if err != nil {
 		return "", err
 	}

config/dependency.go

@@ -14,6 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 
 	"github.com/gruntwork-io/terragrunt/internal/cache"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -789,7 +790,7 @@ func canGetRemoteState(remoteState *remote.RemoteState) bool {
 // terragruntAlreadyInit returns true if it detects that the module specified by the given terragrunt configuration is
 // already initialized with the terraform source. This will also return the working directory where you can run
 // terraform.
-func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configPath string, ctx *ParsingContext) (bool, string, error) {
+func terragruntAlreadyInit(opts *options.TerragruntOptions, configPath string, ctx *ParsingContext) (bool, string, error) {
 	// We need to first determine the working directory where the terraform source should be located. This is dependent
 	// on the source field of the terraform block in the config.
 	terraformBlockTGConfig, err := PartialParseConfigFile(ctx.WithDecodeList(TerraformSource), configPath, nil)
@@ -799,7 +800,7 @@ func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configP
 
 	var workingDir string
 
-	sourceURL, err := GetTerraformSourceURL(terragruntOptions, terraformBlockTGConfig)
+	sourceURL, err := GetTerraformSourceURL(opts, terraformBlockTGConfig)
 	if err != nil {
 		return false, "", err
 	}
@@ -813,7 +814,10 @@ func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configP
 			workingDir = filepath.Dir(configPath)
 		}
 	} else {
-		terraformSource, err := terraform.NewSource(sourceURL, terragruntOptions.DownloadDir, terragruntOptions.WorkingDir, terragruntOptions.Logger)
+		experiment := opts.Experiments[experiment.Symlinks]
+		walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode)
+
+		terraformSource, err := terraform.NewSource(sourceURL, opts.DownloadDir, opts.WorkingDir, opts.Logger, walkWithSymlinks)
 		if err != nil {
 			return false, "", err
 		}

config/variable.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gruntwork-io/terragrunt/config/hclparse"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/util"
 	"github.com/hashicorp/hcl/v2"
@@ -25,8 +26,11 @@ type ParsedVariable struct {
 
 // ParseVariables - parse variables from tf files.
 func ParseVariables(opts *options.TerragruntOptions, directoryPath string) ([]*ParsedVariable, error) {
+	experiment := opts.Experiments[experiment.Symlinks]
+	walkWithSymlinks := experiment.Evaluate(opts.ExperimentMode)
+
 	// list all tf files
-	tfFiles, err := util.ListTfFiles(directoryPath)
+	tfFiles, err := util.ListTfFiles(directoryPath, walkWithSymlinks)
 	if err != nil {
 		return nil, errors.New(err)
 	}

docs/_docs/04_reference/cli-options.md

@@ -95,6 +95,10 @@ This page documents the CLI commands and options available with Terragrunt:
   - [terragrunt-forward-tf-stdout](#terragrunt-forward-tf-stdout)
   - [terragrunt-no-destroy-dependencies-check](#terragrunt-no-destroy-dependencies-check)
   - [feature](#feature)
+  - [experiment](#experiment)
+  - [experiment-mode](#experiment-mode)
+  - [strict-control](#strict-control)
+  - [strict-mode](#strict-mode)
 
 ## CLI commands
 
@@ -1759,3 +1763,39 @@ Setting feature flags through environment variables:
 export TERRAGRUNT_FEATURE=int_feature_flag=123,bool_feature_flag=true,string_feature_flag=app1
 terragrunt apply
 ```
+
+### experiment
+
+**CLI Arg**: `--experiment`<br/>
+**Environment Variable**: `TERRAGRUNT_EXPERIMENT`<br/>
+
+Enable experimental features in Terragrunt before they're stable.
+
+For more information, see the [Experiments](/docs/reference/experiments) documentation.
+
+### experiment-mode
+
+**CLI Arg**: `--experiment-mode`<br/>
+**Environment Variable**: `TERRAGRUNT_EXPERIMENT_MODE`<br/>
+
+Enable all experimental features in Terragrunt before they're stable.
+
+For more information, see the [Experiments](/docs/reference/experiments) documentation.
+
+### strict-control
+
+**CLI Arg**: `--strict-control`<br/>
+**Environment Variable**: `TERRAGRUNT_STRICT_CONTROL`<br/>
+
+Enable strict controls that opt-in future breaking changes in Terragrunt.
+
+For more information, see the [Strict Mode](/docs/reference/strict-mode) documentation.
+
+### strict-mode
+
+**CLI Arg**: `--strict-mode`<br/>
+**Environment Variable**: `TERRAGRUNT_STRICT_MODE`<br/>
+
+Enable all strict controls that opt-in future breaking changes in Terragrunt.
+
+For more information, see the [Strict Mode](/docs/reference/strict-mode) documentation.