diff --git a/.gitignore b/.gitignore index e6cd03ffea..cbaafb3a74 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor *.tfstate *.tfstate.backup *.out +.terragrunt-cache \ No newline at end of file diff --git a/README.md b/README.md index de1508206a..651dc66aad 100644 --- a/README.md +++ b/README.md @@ -263,12 +263,13 @@ terragrunt apply When Terragrunt finds the `terraform` block with a `source` parameter in `live/qa/app/terraform.tfvars` file, it will: -1. Download the configurations specified via the `source` parameter into a temporary folder. This downloading is done - by using the [terraform init command](https://www.terraform.io/docs/commands/init.html), so the `source` parameter - supports the exact same syntax as the [module source](https://www.terraform.io/docs/modules/sources.html) parameter, - including local file paths, Git URLs, and Git URLs with `ref` parameters (useful for checking out a specific tag, - commit, or branch of Git repo). Terragrunt will download all the code in the repo (i.e. the part before the - double-slash `//`) so that relative paths work correctly between modules in that repo. +1. Download the configurations specified via the `source` parameter into the `--terragrunt-download-dir` folder (by + default, `.terragrunt-cache` in the working directory, which we recommend adding to `.gitignore`). This downloading + is done by using the [terraform init command](https://www.terraform.io/docs/commands/init.html), so the `source` + parameter supports the exact same syntax as the [module source](https://www.terraform.io/docs/modules/sources.html) + parameter, including local file paths, Git URLs, and Git URLs with `ref` parameters (useful for checking out a + specific tag, commit, or branch of Git repo). Terragrunt will download all the code in the repo (i.e. the part + before the double-slash `//`) so that relative paths work correctly between modules in that repo. 1. Copy all files from the current working directory into the temporary folder. This way, Terraform will automatically read in the variables defined in the `terraform.tfvars` file. @@ -1759,6 +1760,10 @@ start with the prefix `--terragrunt-`. The currently available options are: commands, this parameter has a different meaning: Terragrunt will apply or destroy all the Terraform modules in the subfolders of the `terragrunt-working-dir`, running `terraform` in the root of each module it finds. +* `--terragrunt-download-dir`: The path where to download Terraform code when using [remote Terraform + configurations](#keep-your-terraform-code-dry). Default is `.terragrunt-cache` in the working directory. We recommend + adding this folder to your `.gitignore`. + * `--terragrunt-source`: Download Terraform configurations from the specified source into a temporary folder, and run Terraform in that temporary folder. May also be specified via the `TERRAGRUNT_SOURCE` environment variable. The source should use the same syntax as the [Terraform module source](https://www.terraform.io/docs/modules/sources.html) diff --git a/cli/args.go b/cli/args.go index 52a8c6cc7d..fa7cd4a555 100644 --- a/cli/args.go +++ b/cli/args.go @@ -45,6 +45,15 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer) return nil, err } + downloadDirRaw, err := parseStringArg(args, OPT_DOWNLOAD_DIR, util.JoinPath(workingDir, ".terragrunt-cache")) + if err != nil { + return nil, err + } + downloadDir, err := filepath.Abs(downloadDirRaw) + if err != nil { + return nil, errors.WithStackTrace(err) + } + terragruntConfigPath, err := parseStringArg(args, OPT_TERRAGRUNT_CONFIG, os.Getenv("TERRAGRUNT_CONFIG")) if err != nil { return nil, err @@ -85,6 +94,7 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer) opts.NonInteractive = parseBooleanArg(args, OPT_NON_INTERACTIVE, os.Getenv("TF_INPUT") == "false" || os.Getenv("TF_INPUT") == "0") opts.TerraformCliArgs = filterTerragruntArgs(args) opts.WorkingDir = filepath.ToSlash(workingDir) + opts.DownloadDir = filepath.ToSlash(downloadDir) opts.Logger = util.CreateLoggerWithWriter(errWriter, "") opts.RunTerragrunt = runTerragrunt opts.Source = terraformSource diff --git a/cli/cli_app.go b/cli/cli_app.go index 9933e26646..6832203323 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -18,6 +18,7 @@ import ( "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/util" version "github.com/hashicorp/go-version" + "github.com/mattn/go-zglob" "github.com/urfave/cli" ) @@ -26,13 +27,14 @@ const OPT_TERRAGRUNT_TFPATH = "terragrunt-tfpath" const OPT_TERRAGRUNT_NO_AUTO_INIT = "terragrunt-no-auto-init" const OPT_NON_INTERACTIVE = "terragrunt-non-interactive" const OPT_WORKING_DIR = "terragrunt-working-dir" +const OPT_DOWNLOAD_DIR = "terragrunt-download-dir" const OPT_TERRAGRUNT_SOURCE = "terragrunt-source" const OPT_TERRAGRUNT_SOURCE_UPDATE = "terragrunt-source-update" const OPT_TERRAGRUNT_IAM_ROLE = "terragrunt-iam-role" const OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS = "terragrunt-ignore-dependency-errors" var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_NO_AUTO_INIT} -var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE} +var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_DOWNLOAD_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE} const CMD_PLAN_ALL = "plan-all" const CMD_APPLY_ALL = "apply-all" @@ -106,6 +108,7 @@ GLOBAL OPTIONS: terragrunt-no-auto-init Don't automatically run 'terraform init' during other terragrunt commands. You must run 'terragrunt init' manually. terragrunt-non-interactive Assume "yes" for all prompts. terragrunt-working-dir The path to the Terraform templates. Default is current directory. + terragrunt-download-dir The path where to download Terraform code. Default is .terragrunt-cache in the working directory. terragrunt-source Download Terraform configurations from the specified source into a temporary folder, and run Terraform in that temporary folder. terragrunt-source-update Delete the contents of the temporary folder to clear out any old, cached source code before downloading new source code into it. terragrunt-iam-role Assume the specified IAM role before executing Terraform. Can also be set via the TERRAGRUNT_IAM_ROLE environment variable. @@ -226,6 +229,10 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error { } } + if err := checkFolderContainsTerraformCode(terragruntOptions); err != nil { + return err + } + if terragruntConfig.RemoteState != nil { if err := checkTerraformCodeDefinesBackend(terragruntOptions, terragruntConfig.RemoteState.Backend); err != nil { return err @@ -373,6 +380,19 @@ func prepareInitCommand(terragruntOptions *options.TerragruntOptions, terragrunt return nil } +func checkFolderContainsTerraformCode(terragruntOptions *options.TerragruntOptions) error { + files, err := zglob.Glob(fmt.Sprintf("%s/**/*.tf", terragruntOptions.WorkingDir)) + if err != nil { + return errors.WithStackTrace(err) + } + + if len(files) == 0 { + return errors.WithStackTrace(NoTerraformFilesFound(terragruntOptions.WorkingDir)) + } + + return nil +} + // Check that the specified Terraform code defines a backend { ... } block and return an error if doesn't func checkTerraformCodeDefinesBackend(terragruntOptions *options.TerragruntOptions, backendType string) error { terraformBackendRegexp, err := regexp.Compile(fmt.Sprintf(`backend[[:blank:]]+"%s"`, backendType)) @@ -477,11 +497,16 @@ func runTerraformInit(terragruntOptions *options.TerragruntOptions, terragruntCo if downloadSource { initOptions.WorkingDir = terraformSource.WorkingDir if !util.FileExists(terraformSource.WorkingDir) { - if err := os.MkdirAll(terraformSource.WorkingDir, 0777); err != nil { + if err := os.MkdirAll(terraformSource.WorkingDir, 0700); err != nil { return errors.WithStackTrace(err) } } + // We will run init separately to download modules, plugins, backend state, etc, so don't run it at this point + initOptions.AppendTerraformCliArgs("-get=false") + initOptions.AppendTerraformCliArgs("-get-plugins=false") + initOptions.AppendTerraformCliArgs("-backend=false") + v0_10_0, err := version.NewVersion("v0.10.0") if err != nil { return err @@ -681,3 +706,9 @@ type BackendNotDefined struct { func (err BackendNotDefined) Error() string { return fmt.Sprintf("Found remote_state settings in %s but no backend block in the Terraform code in %s. You must define a backend block (it can be empty!) in your Terraform code or your remote state settings will have no effect! It should look something like this:\n\nterraform {\n backend \"%s\" {}\n}\n\n", err.Opts.TerragruntConfigPath, err.Opts.WorkingDir, err.BackendType) } + +type NoTerraformFilesFound string + +func (path NoTerraformFilesFound) Error() string { + return fmt.Sprintf("Did not find any Terraform files (*.tf) in %s", string(path)) +} diff --git a/cli/download_source.go b/cli/download_source.go index 67ae77dd2e..575650681a 100644 --- a/cli/download_source.go +++ b/cli/download_source.go @@ -266,21 +266,40 @@ func getForcedGetter(sourceUrl string) (string, string) { // Splits a source URL into the root repo and the path. The root repo is the part of the URL before the double-slash // (//), which typically represents the root of a modules repo (e.g. github.com/foo/infrastructure-modules) and the // path is everything after the double slash. If there is no double-slash in the URL, the root repo is the entire -// sourceUrl and the path is an empty string. +// sourceUrl before the final slash and the path is everything after the final slash. func splitSourceUrl(sourceUrl *url.URL, terragruntOptions *options.TerragruntOptions) (*url.URL, string, error) { + sourceUrlModifiedPath, err := parseSourceUrl(strings.TrimSuffix(sourceUrl.String(), string(filepath.Separator))) + if err != nil { + return nil, "", errors.WithStackTrace(err) + } + pathSplitOnDoubleSlash := strings.SplitN(sourceUrl.Path, "//", 2) if len(pathSplitOnDoubleSlash) > 1 { - sourceUrlModifiedPath, err := parseSourceUrl(sourceUrl.String()) - if err != nil { - return nil, "", errors.WithStackTrace(err) - } - sourceUrlModifiedPath.Path = pathSplitOnDoubleSlash[0] return sourceUrlModifiedPath, pathSplitOnDoubleSlash[1], nil } else { - terragruntOptions.Logger.Printf("WARNING: no double-slash (//) found in source URL %s. Relative paths in downloaded Terraform code may not work.", sourceUrl.Path) - return sourceUrl, "", nil + // We use terragrunt init -from-module=XXX to download remote Terraform configurations from XXX. If you don't + // have a double slash in XXX, the -from-module code tries to do some sort of validation on your code right + // after downloading, which will fail if you're running it with -get=false, -backend=false, etc (we run it this + // way as we call init shortly after and don't want to do all these steps twice). Therefore, we inject a double + // slash here to avoid this validation failure. + + terragruntOptions.Logger.Printf("WARNING: no double-slash (//) found in source URL %s. Will insert one, but note that relative paths in downloaded Terraform code may not work.", sourceUrl.Path) + + parts := strings.SplitAfter(sourceUrlModifiedPath.Path, string(filepath.Separator)) + + everythingBeforeFinalSlash := parts[0 : len(parts)-1] + everythingAfterFinalSlash := parts[len(parts)-1] + + sourceUrlModifiedPath.Path = strings.Join(everythingBeforeFinalSlash, "") + + // Remove trailing slashes, so long as the path isn't just a single slash + if len(sourceUrlModifiedPath.Path) > 1 { + sourceUrlModifiedPath.Path = strings.TrimSuffix(sourceUrlModifiedPath.Path, string(filepath.Separator)) + } + + return sourceUrlModifiedPath, everythingAfterFinalSlash, nil } } diff --git a/cli/download_source_test.go b/cli/download_source_test.go index 85e23e54a1..d16e5890b6 100644 --- a/cli/download_source_test.go +++ b/cli/download_source_test.go @@ -13,6 +13,7 @@ import ( "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAlreadyHaveLatestCodeLocalFilePath(t *testing.T) { @@ -158,6 +159,54 @@ func TestDownloadTerraformSourceIfNecessaryRemoteUrlOverrideSource(t *testing.T) testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, true, "# Hello, World") } +func TestSplitSourceUrl(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + sourceUrl string + expectedRootRepo string + expectedModulePath string + }{ + {"root-path-only-no-double-slash", "/foo", "/", "foo"}, + {"parent-path-one-child-no-double-slash", "/foo/bar", "/foo", "bar"}, + {"parent-path-multiple-children-no-double-slash", "/foo/bar/baz/blah", "/foo/bar/baz", "blah"}, + {"relative-path-no-children-no-double-slash", "../foo", "..", "foo"}, + {"relative-path-one-child-no-double-slash", "../foo/bar", "../foo", "bar"}, + {"relative-path-multiple-children-no-double-slash", "../foo/bar/baz/blah", "../foo/bar/baz", "blah"}, + {"root-path-only-with-double-slash", "/foo//", "/foo", ""}, + {"parent-path-one-child-with-double-slash", "/foo//bar", "/foo", "bar"}, + {"parent-path-multiple-children-with-double-slash", "/foo/bar//baz/blah", "/foo/bar", "baz/blah"}, + {"relative-path-no-children-with-double-slash", "..//foo", "..", "foo"}, + {"relative-path-one-child-with-double-slash", "../foo//bar", "../foo", "bar"}, + {"relative-path-multiple-children-with-double-slash", "../foo/bar//baz/blah", "../foo/bar", "baz/blah"}, + {"parent-url-one-child-no-double-slash", "ssh://git@github.com:foo/modules.git/foo", "ssh://git@github.com:foo/modules.git", "foo"}, + {"parent-url-multiple-children-no-double-slash", "ssh://git@github.com:foo/modules.git/foo/bar/baz/blah", "ssh://git@github.com:foo/modules.git/foo/bar/baz", "blah"}, + {"parent-url-one-child-with-double-slash", "ssh://git@github.com:foo/modules.git//foo", "ssh://git@github.com:foo/modules.git", "foo"}, + {"parent-url-multiple-children-with-double-slash", "ssh://git@github.com:foo/modules.git//foo/bar/baz/blah", "ssh://git@github.com:foo/modules.git", "foo/bar/baz/blah"}, + } + + for _, testCase := range testCases { + // Save a local copy in scope so all the tests don't run the final item in the loop + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + sourceUrl, err := url.Parse(testCase.sourceUrl) + require.NoError(t, err) + + terragruntOptions, err := options.NewTerragruntOptionsForTest("testing") + require.NoError(t, err) + + actualRootRepo, actualModulePath, err := splitSourceUrl(sourceUrl, terragruntOptions) + require.NoError(t, err) + + assert.Equal(t, testCase.expectedRootRepo, actualRootRepo.String()) + assert.Equal(t, testCase.expectedModulePath, actualModulePath) + }) + } +} + func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, downloadDir string, sourceUpdate bool, expectedFileContents string) { terraformSource := &TerraformSource{ CanonicalSourceURL: parseUrl(t, canonicalUrl), diff --git a/options/options.go b/options/options.go index 27b8cdf8e5..677d09c9a7 100644 --- a/options/options.go +++ b/options/options.go @@ -2,7 +2,6 @@ package options import ( "fmt" - "github.com/mitchellh/go-homedir" "io" "log" "os" @@ -90,14 +89,11 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro logger := util.CreateLogger("") - homedir, err := homedir.Dir() + downloadDir, err := filepath.Abs(filepath.Join(workingDir, ".terragrunt-cache")) if err != nil { - logger.Printf("error: %v\n", err) - return nil, err + return nil, errors.WithStackTrace(err) } - downloadDir := filepath.Join(homedir, ".terragrunt") - return &TerragruntOptions{ TerragruntConfigPath: terragruntConfigPath, TerraformPath: "terraform",