From 25c3b8db3b8a634a4de8393f62779f63a3f19bd1 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Tue, 14 Aug 2018 15:22:11 +0200 Subject: [PATCH 1/4] Speed up init usage When we use init to download modules, set `-get=false`, `-get-plugins=false`, and `-backend=false` so that all of those can be handled in the second call to `init` (if necessary). I tried this before in https://github.com/gruntwork-io/terragrunt/pull/516, but had to revert it due to a Terraform bug (https://github.com/hashicorp/terraform/issues/18460). I realize now that I can simply catch the error from that Terraform bug and ignore it! --- cli/cli_app.go | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/cli/cli_app.go b/cli/cli_app.go index cfbb0157c2..6aa97eef81 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -343,13 +343,30 @@ func runTerragruntWithConfig(terragruntOptions *options.TerragruntOptions, terra return errors.NewMultiError(beforeHookErrors, terraformError, postHookErrors) } +var moduleNotFoundErr = regexp.MustCompile(`Error: Error loading modules: module .+?: not found, may need to run 'terraform init'`) + func runTerraformCommandIfNoErrors(possibleErrors error, terragruntOptions *options.TerragruntOptions) error { - if possibleErrors == nil { - return shell.RunTerraformCommand(terragruntOptions, terragruntOptions.TerraformCliArgs...) + if possibleErrors != nil { + terragruntOptions.Logger.Println("Errors encountered running before_hooks. Not running terraform.") + return nil } - terragruntOptions.Logger.Println("Errors encountered running before_hooks. Not running terraform.") - return nil + // Workaround for https://github.com/hashicorp/terraform/issues/18460. Calling 'terraform init -get=false' + // sometimes results in Terraform trying to download/validate modules anyway, so we need to ignore that error. + if util.ListContainsElement(terragruntOptions.TerraformCliArgs, "init") && util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-get=false") { + out, err := shell.RunTerraformCommandAndCaptureOutput(terragruntOptions, terragruntOptions.TerraformCliArgs...) + terragruntOptions.Logger.Println(out) + + // If we got an error and the error output included this error message, ignore the error and keep going + if err != nil && len(moduleNotFoundErr.FindStringSubmatch(out)) > 0 { + terragruntOptions.Logger.Println("Ignoring error from call to init, as this is a known Terraform bug: https://github.com/hashicorp/terraform/issues/18460") + return nil + } + + return err + } + + return shell.RunTerraformCommand(terragruntOptions, terragruntOptions.TerraformCliArgs...) } // Prepare for running 'terraform init' by @@ -506,6 +523,11 @@ func runTerraformInit(terragruntOptions *options.TerragruntOptions, terragruntCo } } + // We will run init separately to download modules, plugins, backend state, etc, so don't run it at this point + initOptions.AppendTerraformCliArgs("-get=false") + initOptions.AppendTerraformCliArgs("-get-plugins=false") + initOptions.AppendTerraformCliArgs("-backend=false") + v0_10_0, err := version.NewVersion("v0.10.0") if err != nil { return err @@ -513,10 +535,10 @@ func runTerraformInit(terragruntOptions *options.TerragruntOptions, terragruntCo if terragruntOptions.TerraformVersion.LessThan(v0_10_0) { // Terraform versions < 0.10.0 specified the module source as an argument (rather than the -from-module option) - initOptions.AppendTerraformCliArgs(terraformSource.CanonicalSourceURL.String()) + initOptions.AppendTerraformCliArgs(terraformSource.CanonicalSourceURL.String(), "-no-color") } else { // Terraform versions >= 0.10.0 specify the module source using the -from-module option - initOptions.AppendTerraformCliArgs("-from-module=" + terraformSource.CanonicalSourceURL.String()) + initOptions.AppendTerraformCliArgs("-from-module="+terraformSource.CanonicalSourceURL.String(), "-no-color") } initOptions.AppendTerraformCliArgs(terraformSource.DownloadDir) } From 8f504b42dcce661712beb8feec4c6e84431d00c5 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Tue, 14 Aug 2018 15:29:25 +0200 Subject: [PATCH 2/4] Fix error message check --- cli/cli_app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cli_app.go b/cli/cli_app.go index 6aa97eef81..89cf92b08d 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -343,7 +343,7 @@ func runTerragruntWithConfig(terragruntOptions *options.TerragruntOptions, terra return errors.NewMultiError(beforeHookErrors, terraformError, postHookErrors) } -var moduleNotFoundErr = regexp.MustCompile(`Error: Error loading modules: module .+?: not found, may need to run 'terraform init'`) +var moduleNotFoundErr = regexp.MustCompile(`Error loading modules: module .+?: not found, may need to run 'terraform init'`) func runTerraformCommandIfNoErrors(possibleErrors error, terragruntOptions *options.TerragruntOptions) error { if possibleErrors != nil { From 1942596a04cac6be4952ee490a352076413f2807 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Tue, 14 Aug 2018 23:36:48 +0200 Subject: [PATCH 3/4] Send output to stderr instead of polluting stdout --- cli/cli_app.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/cli_app.go b/cli/cli_app.go index 89cf92b08d..23804a3d8c 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -355,7 +355,9 @@ func runTerraformCommandIfNoErrors(possibleErrors error, terragruntOptions *opti // sometimes results in Terraform trying to download/validate modules anyway, so we need to ignore that error. if util.ListContainsElement(terragruntOptions.TerraformCliArgs, "init") && util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-get=false") { out, err := shell.RunTerraformCommandAndCaptureOutput(terragruntOptions, terragruntOptions.TerraformCliArgs...) - terragruntOptions.Logger.Println(out) + + // Write the log output to stderr to make sure we don't pollute stdout + terragruntOptions.ErrWriter.Write([]byte(out)) // If we got an error and the error output included this error message, ignore the error and keep going if err != nil && len(moduleNotFoundErr.FindStringSubmatch(out)) > 0 { From e0069c0990a33b05d709e544dd93497377d2c9f3 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Wed, 15 Aug 2018 00:09:55 +0200 Subject: [PATCH 4/4] Add check for missing plugins --- cli/cli_app.go | 9 ++++++--- test/integration_test.go | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/cli_app.go b/cli/cli_app.go index 23804a3d8c..a91dd96b19 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -351,16 +351,19 @@ func runTerraformCommandIfNoErrors(possibleErrors error, terragruntOptions *opti return nil } - // Workaround for https://github.com/hashicorp/terraform/issues/18460. Calling 'terraform init -get=false' + // Workaround for https://github.com/hashicorp/terraform/issues/18460. Calling 'terraform init -get=false ' // sometimes results in Terraform trying to download/validate modules anyway, so we need to ignore that error. - if util.ListContainsElement(terragruntOptions.TerraformCliArgs, "init") && util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-get=false") { + if util.ListContainsElement(terragruntOptions.TerraformCliArgs, "init") && + util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-get=false") && + util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-get-plugins=false") && + util.ListContainsElement(terragruntOptions.TerraformCliArgs, "-backend=false") { out, err := shell.RunTerraformCommandAndCaptureOutput(terragruntOptions, terragruntOptions.TerraformCliArgs...) // Write the log output to stderr to make sure we don't pollute stdout terragruntOptions.ErrWriter.Write([]byte(out)) // If we got an error and the error output included this error message, ignore the error and keep going - if err != nil && len(moduleNotFoundErr.FindStringSubmatch(out)) > 0 { + if err != nil && (len(moduleNotFoundErr.FindStringSubmatch(out)) > 0 || strings.Contains(out, "Missing required providers.")) { terragruntOptions.Logger.Println("Ignoring error from call to init, as this is a known Terraform bug: https://github.com/hashicorp/terraform/issues/18460") return nil } diff --git a/test/integration_test.go b/test/integration_test.go index a094249f58..170150c50c 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -948,7 +948,9 @@ func copyEnvironment(t *testing.T, environmentPath string) string { return err } - if info.IsDir() { + // The info.Mode() check is to catch symlinks to directories: + // https://stackoverflow.com/questions/32908277/fileinfo-isdir-isnt-detecting-directory + if info.IsDir() || (info.Mode()&os.ModeSymlink) == os.ModeSymlink { return nil }