Merge pull request #516 from gruntwork-io/cache-dir

Change how Terragrunt downloads remote Terraform configurations

.gitignore

@@ -7,3 +7,4 @@ vendor
 *.tfstate
 *.tfstate.backup
 *.out
+.terragrunt-cache

README.md

@@ -263,12 +263,13 @@ terragrunt apply
 
 When Terragrunt finds the `terraform` block with a `source` parameter in `live/qa/app/terraform.tfvars` file, it will:
 
-1. Download the configurations specified via the `source` parameter into a temporary folder. This downloading is done
-   by using the [terraform init command](https://www.terraform.io/docs/commands/init.html), so the `source` parameter
-   supports the exact same syntax as the [module source](https://www.terraform.io/docs/modules/sources.html) parameter,
-   including local file paths, Git URLs, and Git URLs with `ref` parameters (useful for checking out a specific tag,
-   commit, or branch of Git repo). Terragrunt will download all the code in the repo (i.e. the part before the
-   double-slash `//`) so that relative paths work correctly between modules in that repo.
+1. Download the configurations specified via the `source` parameter into the `--terragrunt-download-dir` folder (by
+   default, `.terragrunt-cache` in the working directory, which we recommend adding to `.gitignore`). This downloading
+   is done by using the [terraform init command](https://www.terraform.io/docs/commands/init.html), so the `source`
+   parameter supports the exact same syntax as the [module source](https://www.terraform.io/docs/modules/sources.html)
+   parameter, including local file paths, Git URLs, and Git URLs with `ref` parameters (useful for checking out a
+   specific tag, commit, or branch of Git repo). Terragrunt will download all the code in the repo (i.e. the part
+   before the double-slash `//`) so that relative paths work correctly between modules in that repo.
 
 1. Copy all files from the current working directory into the temporary folder. This way, Terraform will automatically
    read in the variables defined in the `terraform.tfvars` file.
@@ -1759,6 +1760,10 @@ start with the prefix `--terragrunt-`. The currently available options are:
   commands, this parameter has a different meaning: Terragrunt will apply or destroy all the Terraform modules in the 
   subfolders of the `terragrunt-working-dir`, running `terraform` in the root of each module it finds.
 
+* `--terragrunt-download-dir`: The path where to download Terraform code when using [remote Terraform
+  configurations](#keep-your-terraform-code-dry). Default is `.terragrunt-cache` in the working directory. We recommend
+  adding this folder to your `.gitignore`.
+
 * `--terragrunt-source`: Download Terraform configurations from the specified source into a temporary folder, and run
   Terraform in that temporary folder. May also be specified via the `TERRAGRUNT_SOURCE` environment variable. The
   source should use the same syntax as the [Terraform module source](https://www.terraform.io/docs/modules/sources.html)

cli/args.go

@@ -45,6 +45,15 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer)
 		return nil, err
 	}
 
+	downloadDirRaw, err := parseStringArg(args, OPT_DOWNLOAD_DIR, util.JoinPath(workingDir, ".terragrunt-cache"))
+	if err != nil {
+		return nil, err
+	}
+	downloadDir, err := filepath.Abs(downloadDirRaw)
+	if err != nil {
+		return nil, errors.WithStackTrace(err)
+	}
+
 	terragruntConfigPath, err := parseStringArg(args, OPT_TERRAGRUNT_CONFIG, os.Getenv("TERRAGRUNT_CONFIG"))
 	if err != nil {
 		return nil, err
@@ -85,6 +94,7 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer)
 	opts.NonInteractive = parseBooleanArg(args, OPT_NON_INTERACTIVE, os.Getenv("TF_INPUT") == "false" || os.Getenv("TF_INPUT") == "0")
 	opts.TerraformCliArgs = filterTerragruntArgs(args)
 	opts.WorkingDir = filepath.ToSlash(workingDir)
+	opts.DownloadDir = filepath.ToSlash(downloadDir)
 	opts.Logger = util.CreateLoggerWithWriter(errWriter, "")
 	opts.RunTerragrunt = runTerragrunt
 	opts.Source = terraformSource

cli/cli_app.go

@@ -18,6 +18,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/util"
 	version "github.com/hashicorp/go-version"
+	"github.com/mattn/go-zglob"
 	"github.com/urfave/cli"
 )
 
@@ -26,13 +27,14 @@ const OPT_TERRAGRUNT_TFPATH = "terragrunt-tfpath"
 const OPT_TERRAGRUNT_NO_AUTO_INIT = "terragrunt-no-auto-init"
 const OPT_NON_INTERACTIVE = "terragrunt-non-interactive"
 const OPT_WORKING_DIR = "terragrunt-working-dir"
+const OPT_DOWNLOAD_DIR = "terragrunt-download-dir"
 const OPT_TERRAGRUNT_SOURCE = "terragrunt-source"
 const OPT_TERRAGRUNT_SOURCE_UPDATE = "terragrunt-source-update"
 const OPT_TERRAGRUNT_IAM_ROLE = "terragrunt-iam-role"
 const OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS = "terragrunt-ignore-dependency-errors"
 
 var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_NO_AUTO_INIT}
-var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE}
+var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_DOWNLOAD_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE}
 
 const CMD_PLAN_ALL = "plan-all"
 const CMD_APPLY_ALL = "apply-all"
@@ -106,6 +108,7 @@ GLOBAL OPTIONS:
    terragrunt-no-auto-init              Don't automatically run 'terraform init' during other terragrunt commands. You must run 'terragrunt init' manually.
    terragrunt-non-interactive           Assume "yes" for all prompts.
    terragrunt-working-dir               The path to the Terraform templates. Default is current directory.
+   terragrunt-download-dir              The path where to download Terraform code. Default is .terragrunt-cache in the working directory.
    terragrunt-source                    Download Terraform configurations from the specified source into a temporary folder, and run Terraform in that temporary folder.
    terragrunt-source-update             Delete the contents of the temporary folder to clear out any old, cached source code before downloading new source code into it.
    terragrunt-iam-role             		Assume the specified IAM role before executing Terraform. Can also be set via the TERRAGRUNT_IAM_ROLE environment variable.
@@ -226,6 +229,10 @@ func runTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 		}
 	}
 
+	if err := checkFolderContainsTerraformCode(terragruntOptions); err != nil {
+		return err
+	}
+
 	if terragruntConfig.RemoteState != nil {
 		if err := checkTerraformCodeDefinesBackend(terragruntOptions, terragruntConfig.RemoteState.Backend); err != nil {
 			return err
@@ -373,6 +380,19 @@ func prepareInitCommand(terragruntOptions *options.TerragruntOptions, terragrunt
 	return nil
 }
 
+func checkFolderContainsTerraformCode(terragruntOptions *options.TerragruntOptions) error {
+	files, err := zglob.Glob(fmt.Sprintf("%s/**/*.tf", terragruntOptions.WorkingDir))
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+
+	if len(files) == 0 {
+		return errors.WithStackTrace(NoTerraformFilesFound(terragruntOptions.WorkingDir))
+	}
+
+	return nil
+}
+
 // Check that the specified Terraform code defines a backend { ... } block and return an error if doesn't
 func checkTerraformCodeDefinesBackend(terragruntOptions *options.TerragruntOptions, backendType string) error {
 	terraformBackendRegexp, err := regexp.Compile(fmt.Sprintf(`backend[[:blank:]]+"%s"`, backendType))
@@ -477,11 +497,16 @@ func runTerraformInit(terragruntOptions *options.TerragruntOptions, terragruntCo
 	if downloadSource {
 		initOptions.WorkingDir = terraformSource.WorkingDir
 		if !util.FileExists(terraformSource.WorkingDir) {
-			if err := os.MkdirAll(terraformSource.WorkingDir, 0777); err != nil {
+			if err := os.MkdirAll(terraformSource.WorkingDir, 0700); err != nil {
 				return errors.WithStackTrace(err)
 			}
 		}
 
+		// We will run init separately to download modules, plugins, backend state, etc, so don't run it at this point
+		initOptions.AppendTerraformCliArgs("-get=false")
+		initOptions.AppendTerraformCliArgs("-get-plugins=false")
+		initOptions.AppendTerraformCliArgs("-backend=false")
+
 		v0_10_0, err := version.NewVersion("v0.10.0")
 		if err != nil {
 			return err
@@ -681,3 +706,9 @@ type BackendNotDefined struct {
 func (err BackendNotDefined) Error() string {
 	return fmt.Sprintf("Found remote_state settings in %s but no backend block in the Terraform code in %s. You must define a backend block (it can be empty!) in your Terraform code or your remote state settings will have no effect! It should look something like this:\n\nterraform {\n  backend \"%s\" {}\n}\n\n", err.Opts.TerragruntConfigPath, err.Opts.WorkingDir, err.BackendType)
 }
+
+type NoTerraformFilesFound string
+
+func (path NoTerraformFilesFound) Error() string {
+	return fmt.Sprintf("Did not find any Terraform files (*.tf) in %s", string(path))
+}

cli/download_source.go

@@ -266,21 +266,40 @@ func getForcedGetter(sourceUrl string) (string, string) {
 // Splits a source URL into the root repo and the path. The root repo is the part of the URL before the double-slash
 // (//), which typically represents the root of a modules repo (e.g. github.com/foo/infrastructure-modules) and the
 // path is everything after the double slash. If there is no double-slash in the URL, the root repo is the entire
-// sourceUrl and the path is an empty string.
+// sourceUrl before the final slash and the path is everything after the final slash.
 func splitSourceUrl(sourceUrl *url.URL, terragruntOptions *options.TerragruntOptions) (*url.URL, string, error) {
+	sourceUrlModifiedPath, err := parseSourceUrl(strings.TrimSuffix(sourceUrl.String(), string(filepath.Separator)))
+	if err != nil {
+		return nil, "", errors.WithStackTrace(err)
+	}
+
 	pathSplitOnDoubleSlash := strings.SplitN(sourceUrl.Path, "//", 2)
 
 	if len(pathSplitOnDoubleSlash) > 1 {
-		sourceUrlModifiedPath, err := parseSourceUrl(sourceUrl.String())
-		if err != nil {
-			return nil, "", errors.WithStackTrace(err)
-		}
-
 		sourceUrlModifiedPath.Path = pathSplitOnDoubleSlash[0]
 		return sourceUrlModifiedPath, pathSplitOnDoubleSlash[1], nil
 	} else {
-		terragruntOptions.Logger.Printf("WARNING: no double-slash (//) found in source URL %s. Relative paths in downloaded Terraform code may not work.", sourceUrl.Path)
-		return sourceUrl, "", nil
+		// We use terragrunt init -from-module=XXX to download remote Terraform configurations from XXX. If you don't
+		// have a double slash in XXX, the -from-module code tries to do some sort of validation on your code right
+		// after downloading, which will fail if you're running it with -get=false, -backend=false, etc (we run it this
+		// way as we call init shortly after and don't want to do all these steps twice). Therefore, we inject a double
+		// slash here to avoid this validation failure.
+
+		terragruntOptions.Logger.Printf("WARNING: no double-slash (//) found in source URL %s. Will insert one, but note that relative paths in downloaded Terraform code may not work.", sourceUrl.Path)
+
+		parts := strings.SplitAfter(sourceUrlModifiedPath.Path, string(filepath.Separator))
+
+		everythingBeforeFinalSlash := parts[0 : len(parts)-1]
+		everythingAfterFinalSlash := parts[len(parts)-1]
+
+		sourceUrlModifiedPath.Path = strings.Join(everythingBeforeFinalSlash, "")
+
+		// Remove trailing slashes, so long as the path isn't just a single slash
+		if len(sourceUrlModifiedPath.Path) > 1 {
+			sourceUrlModifiedPath.Path = strings.TrimSuffix(sourceUrlModifiedPath.Path, string(filepath.Separator))
+		}
+
+		return sourceUrlModifiedPath, everythingAfterFinalSlash, nil
 	}
 }
 

cli/download_source_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/util"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAlreadyHaveLatestCodeLocalFilePath(t *testing.T) {
@@ -158,6 +159,54 @@ func TestDownloadTerraformSourceIfNecessaryRemoteUrlOverrideSource(t *testing.T)
 	testDownloadTerraformSourceIfNecessary(t, canonicalUrl, downloadDir, true, "# Hello, World")
 }
 
+func TestSplitSourceUrl(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name               string
+		sourceUrl          string
+		expectedRootRepo   string
+		expectedModulePath string
+	}{
+		{"root-path-only-no-double-slash", "/foo", "/", "foo"},
+		{"parent-path-one-child-no-double-slash", "/foo/bar", "/foo", "bar"},
+		{"parent-path-multiple-children-no-double-slash", "/foo/bar/baz/blah", "/foo/bar/baz", "blah"},
+		{"relative-path-no-children-no-double-slash", "../foo", "..", "foo"},
+		{"relative-path-one-child-no-double-slash", "../foo/bar", "../foo", "bar"},
+		{"relative-path-multiple-children-no-double-slash", "../foo/bar/baz/blah", "../foo/bar/baz", "blah"},
+		{"root-path-only-with-double-slash", "/foo//", "/foo", ""},
+		{"parent-path-one-child-with-double-slash", "/foo//bar", "/foo", "bar"},
+		{"parent-path-multiple-children-with-double-slash", "/foo/bar//baz/blah", "/foo/bar", "baz/blah"},
+		{"relative-path-no-children-with-double-slash", "..//foo", "..", "foo"},
+		{"relative-path-one-child-with-double-slash", "../foo//bar", "../foo", "bar"},
+		{"relative-path-multiple-children-with-double-slash", "../foo/bar//baz/blah", "../foo/bar", "baz/blah"},
+		{"parent-url-one-child-no-double-slash", "ssh://[email protected]:foo/modules.git/foo", "ssh://[email protected]:foo/modules.git", "foo"},
+		{"parent-url-multiple-children-no-double-slash", "ssh://[email protected]:foo/modules.git/foo/bar/baz/blah", "ssh://[email protected]:foo/modules.git/foo/bar/baz", "blah"},
+		{"parent-url-one-child-with-double-slash", "ssh://[email protected]:foo/modules.git//foo", "ssh://[email protected]:foo/modules.git", "foo"},
+		{"parent-url-multiple-children-with-double-slash", "ssh://[email protected]:foo/modules.git//foo/bar/baz/blah", "ssh://[email protected]:foo/modules.git", "foo/bar/baz/blah"},
+	}
+
+	for _, testCase := range testCases {
+		// Save a local copy in scope so all the tests don't run the final item in the loop
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			t.Parallel()
+
+			sourceUrl, err := url.Parse(testCase.sourceUrl)
+			require.NoError(t, err)
+
+			terragruntOptions, err := options.NewTerragruntOptionsForTest("testing")
+			require.NoError(t, err)
+
+			actualRootRepo, actualModulePath, err := splitSourceUrl(sourceUrl, terragruntOptions)
+			require.NoError(t, err)
+
+			assert.Equal(t, testCase.expectedRootRepo, actualRootRepo.String())
+			assert.Equal(t, testCase.expectedModulePath, actualModulePath)
+		})
+	}
+}
+
 func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, downloadDir string, sourceUpdate bool, expectedFileContents string) {
 	terraformSource := &TerraformSource{
 		CanonicalSourceURL: parseUrl(t, canonicalUrl),

options/options.go

@@ -2,7 +2,6 @@ package options
 
 import (
 	"fmt"
-	"github.com/mitchellh/go-homedir"
 	"io"
 	"log"
 	"os"
@@ -90,14 +89,11 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro
 
 	logger := util.CreateLogger("")
 
-	homedir, err := homedir.Dir()
+	downloadDir, err := filepath.Abs(filepath.Join(workingDir, ".terragrunt-cache"))
 	if err != nil {
-		logger.Printf("error: %v\n", err)
-		return nil, err
+		return nil, errors.WithStackTrace(err)
 	}
 
-	downloadDir := filepath.Join(homedir, ".terragrunt")
-
 	return &TerragruntOptions{
 		TerragruntConfigPath:   terragruntConfigPath,
 		TerraformPath:          "terraform",