Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add skip evaluation of locals for strict included modules #3012

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

Add skip evaluation of locals for strict included modules #3012

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da888cb
Add skip of modules in case of strict include
denis256 Mar 19, 2024
0bab3f9
Add collection of transient modules
denis256 Mar 20, 2024
bffadb4
Merge remote-tracking branch 'origin/master' into bug/skip-strict-mod…
denis256 Mar 22, 2024
877f751
Merge remote-tracking branch 'origin/master' into bug/skip-strict-mod…
denis256 Mar 26, 2024
44ba550
Merge remote-tracking branch 'origin/master' into bug/skip-strict-mod…
denis256 Apr 3, 2024
95b10ad
Merge remote-tracking branch 'origin/master' into bug/skip-strict-mod…
denis256 Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 33 additions & 11 deletions configstack/module.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,14 @@ func ResolveTerraformModules(terragruntConfigPaths []string, terragruntOptions *
return nil, err
}

transientDependencies := map[string]bool{}

var modules map[string]*TerraformModule
err = telemetry.Telemetry(terragruntOptions, "resolve_modules", map[string]interface{}{
"working_dir": terragruntOptions.WorkingDir,
}, func(childCtx context.Context) error {

result, err := resolveModules(canonicalTerragruntConfigPaths, terragruntOptions, childTerragruntConfig, howThesePathsWereFound)
result, err := resolveModules(&transientDependencies, canonicalTerragruntConfigPaths, terragruntOptions, childTerragruntConfig, howThesePathsWereFound)
if err != nil {
return err
}
Expand All @@ -79,7 +81,7 @@ func ResolveTerraformModules(terragruntConfigPaths []string, terragruntOptions *
err = telemetry.Telemetry(terragruntOptions, "resolve_external_dependencies_for_modules", map[string]interface{}{
"working_dir": terragruntOptions.WorkingDir,
}, func(childCtx context.Context) error {
result, err := resolveExternalDependenciesForModules(modules, map[string]*TerraformModule{}, 0, terragruntOptions, childTerragruntConfig)
result, err := resolveExternalDependenciesForModules(&transientDependencies, modules, map[string]*TerraformModule{}, 0, terragruntOptions, childTerragruntConfig)
if err != nil {
return err
}
Expand Down Expand Up @@ -277,15 +279,15 @@ func flagModulesThatDontInclude(modules []*TerraformModule, terragruntOptions *o
// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
// into a TerraformModule struct. Note that this method will NOT fill in the Dependencies field of the TerraformModule
// struct (see the crosslinkDependencies method for that). Return a map from module path to TerraformModule struct.
func resolveModules(canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howTheseModulesWereFound string) (map[string]*TerraformModule, error) {
func resolveModules(transientDependencies *map[string]bool, canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howTheseModulesWereFound string) (map[string]*TerraformModule, error) {
moduleMap := map[string]*TerraformModule{}
for _, terragruntConfigPath := range canonicalTerragruntConfigPaths {
var module *TerraformModule
err := telemetry.Telemetry(terragruntOptions, "resolve_terraform_module", map[string]interface{}{
"config_path": terragruntConfigPath,
"working_dir": terragruntOptions.WorkingDir,
}, func(childCtx context.Context) error {
m, err := resolveTerraformModule(terragruntConfigPath, moduleMap, terragruntOptions, childTerragruntConfig, howTheseModulesWereFound)
m, err := resolveTerraformModule(transientDependencies, terragruntConfigPath, moduleMap, terragruntOptions, childTerragruntConfig, howTheseModulesWereFound)
if err != nil {
return err
}
Expand All @@ -297,13 +299,19 @@ func resolveModules(canonicalTerragruntConfigPaths []string, terragruntOptions *
}
if module != nil {
moduleMap[module.Path] = module

if module.Config.DependentModulesPath != nil {
for _, dep := range module.Config.DependentModulesPath {
(*transientDependencies)[*dep] = true
}
}
var dependencies map[string]*TerraformModule
err := telemetry.Telemetry(terragruntOptions, "resolve_dependencies_for_module", map[string]interface{}{
"config_path": terragruntConfigPath,
"working_dir": terragruntOptions.WorkingDir,
"module_path": module.Path,
}, func(childCtx context.Context) error {
deps, err := resolveDependenciesForModule(module, moduleMap, terragruntOptions, childTerragruntConfig, true)
deps, err := resolveDependenciesForModule(transientDependencies, module, moduleMap, terragruntOptions, childTerragruntConfig, true)
if err != nil {
return err
}
Expand All @@ -323,7 +331,7 @@ func resolveModules(canonicalTerragruntConfigPaths []string, terragruntOptions *
// Create a TerraformModule struct for the Terraform module specified by the given Terragrunt configuration file path.
// Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the
// crosslinkDependencies method for that).
func resolveTerraformModule(terragruntConfigPath string, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThisModuleWasFound string) (*TerraformModule, error) {
func resolveTerraformModule(transientDependencies *map[string]bool, terragruntConfigPath string, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThisModuleWasFound string) (*TerraformModule, error) {
modulePath, err := util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".")
if err != nil {
return nil, err
Expand All @@ -333,6 +341,20 @@ func resolveTerraformModule(terragruntConfigPath string, moduleMap map[string]*T
return nil, nil
}

// in case of strict include, allow only modules that are included in the terragruntOptions.ModulesThatInclude
if terragruntOptions.StrictInclude {
var found = false
for _, includePath := range terragruntOptions.IncludeDirs {
if modulePath == includePath {
found = true
break
}
}
if !found {
terragruntOptions.Logger.Debugf("Module %s is not included in the --terragrunt-include-dir and will be skipped.", modulePath)
return nil, nil
}
}
// Clone the options struct so we don't modify the original one. This is especially important as run-all operations
// happen concurrently.
opts := terragruntOptions.Clone(terragruntConfigPath)
Expand Down Expand Up @@ -421,7 +443,7 @@ func resolveTerraformModule(terragruntConfigPath string, moduleMap map[string]*T
// environment the user is trying to apply-all or destroy-all. Therefore, this method also confirms whether the user wants
// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in
// the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that).
func resolveExternalDependenciesForModules(moduleMap map[string]*TerraformModule, modulesAlreadyProcessed map[string]*TerraformModule, recursionLevel int, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig) (map[string]*TerraformModule, error) {
func resolveExternalDependenciesForModules(transientDependencies *map[string]bool, moduleMap map[string]*TerraformModule, modulesAlreadyProcessed map[string]*TerraformModule, recursionLevel int, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig) (map[string]*TerraformModule, error) {
allExternalDependencies := map[string]*TerraformModule{}
modulesToSkip := mergeMaps(moduleMap, modulesAlreadyProcessed)

Expand All @@ -433,7 +455,7 @@ func resolveExternalDependenciesForModules(moduleMap map[string]*TerraformModule
sortedKeys := getSortedKeys(moduleMap)
for _, key := range sortedKeys {
module := moduleMap[key]
externalDependencies, err := resolveDependenciesForModule(module, modulesToSkip, terragruntOptions, childTerragruntConfig, false)
externalDependencies, err := resolveDependenciesForModule(transientDependencies, module, modulesToSkip, terragruntOptions, childTerragruntConfig, false)
if err != nil {
return externalDependencies, err
}
Expand All @@ -457,7 +479,7 @@ func resolveExternalDependenciesForModules(moduleMap map[string]*TerraformModule
}

if len(allExternalDependencies) > 0 {
recursiveDependencies, err := resolveExternalDependenciesForModules(allExternalDependencies, moduleMap, recursionLevel+1, terragruntOptions, childTerragruntConfig)
recursiveDependencies, err := resolveExternalDependenciesForModules(transientDependencies, allExternalDependencies, moduleMap, recursionLevel+1, terragruntOptions, childTerragruntConfig)
if err != nil {
return allExternalDependencies, err
}
Expand All @@ -472,7 +494,7 @@ var existingModules = config.NewCache[*map[string]*TerraformModule]()
// resolveDependenciesForModule looks through the dependencies of the given module and resolve the dependency paths listed in the module's config.
// If `skipExternal` is true, the func returns only dependencies that are inside of the current working directory, which means they are part of the environment the
// user is trying to apply-all or destroy-all. Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that).
func resolveDependenciesForModule(module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, chilTerragruntConfig *config.TerragruntConfig, skipExternal bool) (map[string]*TerraformModule, error) {
func resolveDependenciesForModule(transientDependencies *map[string]bool, module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions, chilTerragruntConfig *config.TerragruntConfig, skipExternal bool) (map[string]*TerraformModule, error) {
if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 {
return map[string]*TerraformModule{}, nil
}
Expand Down Expand Up @@ -501,7 +523,7 @@ func resolveDependenciesForModule(module *TerraformModule, moduleMap map[string]
}

howThesePathsWereFound := fmt.Sprintf("dependency of module at '%s'", module.Path)
result, err := resolveModules(externalTerragruntConfigPaths, terragruntOptions, chilTerragruntConfig, howThesePathsWereFound)
result, err := resolveModules(transientDependencies, externalTerragruntConfigPaths, terragruntOptions, chilTerragruntConfig, howThesePathsWereFound)
if err != nil {
return nil, err
}
Expand Down