Fixing generation of provider_installation block used for provider …

…caching (#3280)

* fix: generating provider installation methods

* chore: update integration tests

* chore: delete duplicate log

* fix: unit test

* chore: code improvements

* chore: code improvement

* chore: integration test improvement

* chore: code improvements

* fix: test assets

* fix: integration test

* fix: integration test

* fix: integration test

* fix: integration test

cli/commands/terraform/creds/providers/externalcmd/provider.go

@@ -43,7 +43,6 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden
 		args = parts[1:]
 	}
 
-	provider.terragruntOptions.Logger.Infof("Executing %s to obtain Terragrunt credentials", provider.Name())
 	output, err := shell.RunShellCommandWithOutput(ctx, provider.terragruntOptions, "", true, false, command, args...)
 	if err != nil {
 		return nil, err

cli/provider_cache.go

@@ -105,17 +105,19 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 	for _, registryName := range opts.ProviderCacheRegistryNames {
 		excludeAddrs = append(excludeAddrs, fmt.Sprintf("%s/*/*", registryName))
 	}
-
 	for _, method := range cliCfg.ProviderInstallation.Methods {
 		switch method := method.(type) {
 		case *cliconfig.ProviderInstallationFilesystemMirror:
 			providerHandlers = append(providerHandlers, handlers.NewProviderFilesystemMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
+			method.AppendExclude(excludeAddrs)
 		case *cliconfig.ProviderInstallationNetworkMirror:
 			providerHandlers = append(providerHandlers, handlers.NewProviderNetworkMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
+			method.AppendExclude(excludeAddrs)
 		case *cliconfig.ProviderInstallationDirect:
 			providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, method))
+			method.RemoveExclude(excludeAddrs)
+			continue
 		}
-		method.AppendExclude(excludeAddrs)
 	}
 	providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect)))
 

dynamodb/dynamo_lock_table.go

@@ -2,10 +2,13 @@ package dynamodb
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
 	"github.com/gruntwork-io/go-commons/errors"
 	"github.com/gruntwork-io/terragrunt/aws_helper"
@@ -156,12 +159,34 @@ func tagTableIfTagsGiven(tags map[string]string, tableArn *string, client *dynam
 }
 
 // Delete the given table in DynamoDB
-func DeleteTable(tableName string, client *dynamodb.DynamoDB) error {
+func DeleteTable(tableName string, dbClient *dynamodb.DynamoDB) error {
+	const (
+		maxRetries    = 5
+		minRetryDelay = time.Second
+	)
+
 	tableCreateDeleteSemaphore.Acquire()
 	defer tableCreateDeleteSemaphore.Release()
 
-	_, err := client.DeleteTable(&dynamodb.DeleteTableInput{TableName: aws.String(tableName)})
-	return err
+	req, _ := dbClient.DeleteTableRequest(&dynamodb.DeleteTableInput{TableName: aws.String(tableName)})
+	// It is not always able to delete a table the first attempt, error: `StatusCode: 400, Attempt to change a resource which is still in use: Table tags are being updated: terragrunt_test_*`
+	req.Retryer = &DeleteTableRetryer{DefaultRetryer: client.DefaultRetryer{
+		NumMaxRetries: maxRetries,
+		MinRetryDelay: minRetryDelay,
+	}}
+	return req.Send()
+
+}
+
+type DeleteTableRetryer struct {
+	client.DefaultRetryer
+}
+
+func (retryer DeleteTableRetryer) ShouldRetry(req *request.Request) bool {
+	if req.HTTPResponse.StatusCode == http.StatusBadRequest {
+		return true
+	}
+	return retryer.DefaultRetryer.ShouldRetry(req)
 }
 
 // Return true if the given error is the error message returned by AWS when the resource already exists and is being

terraform/cliconfig/config.go

@@ -91,11 +91,11 @@ func (cfg *Config) AddHost(name string, services map[string]string) {
 //			exclude = ["example.com/*/*"]
 //		}
 //	}
-func (cfg *Config) AddProviderInstallationMethods(methods ...ProviderInstallationMethod) {
+func (cfg *Config) AddProviderInstallationMethods(newMethods ...ProviderInstallationMethod) {
 	if cfg.ProviderInstallation == nil {
 		cfg.ProviderInstallation = &ProviderInstallation{}
 	}
-	cfg.ProviderInstallation.Methods = append(cfg.ProviderInstallation.Methods, methods...)
+	cfg.ProviderInstallation.Methods = cfg.ProviderInstallation.Methods.Merge(newMethods...)
 }
 
 // Save marshalls and saves CLI config with the given config path.

terraform/cliconfig/provider_installation.go

@@ -1,16 +1,59 @@
 package cliconfig
 
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/gruntwork-io/terragrunt/util"
+)
+
 // ProviderInstallation is the structure of the "provider_installation" nested block within the CLI configuration.
 type ProviderInstallation struct {
-	Methods []ProviderInstallationMethod `hcl:",block"`
+	Methods ProviderInstallationMethods `hcl:",block"`
+}
+
+type ProviderInstallationMethods []ProviderInstallationMethod
+
+func (methods ProviderInstallationMethods) Merge(withMethods ...ProviderInstallationMethod) ProviderInstallationMethods {
+	mergedMethods := methods
+
+	for _, withMethod := range withMethods {
+		var isMerged bool
+
+		for _, method := range methods {
+			if method.Merge(withMethod) {
+				isMerged = true
+				break
+			}
+		}
+
+		if !isMerged {
+			mergedMethods = append(mergedMethods, withMethod)
+		}
+	}
+
+	// place the `direct` method at the very end.
+	sort.Slice(mergedMethods, func(i, j int) bool {
+		if _, ok := mergedMethods[j].(*ProviderInstallationDirect); ok {
+			return true
+		}
+		return false
+	})
+
+	return mergedMethods
 }
 
 // ProviderInstallationMethod is an interface type representing the different installation path types and represents an installation method block inside a provider_installation block. The concrete implementations of this interface are:
 //
 //	ProviderInstallationDirect:           install from the provider's origin registry
 //	ProviderInstallationFilesystemMirror: install from a local filesystem mirror
 type ProviderInstallationMethod interface {
+	fmt.Stringer
+	AppendInclude(addrs []string)
 	AppendExclude(addrs []string)
+	RemoveExclude(addrs []string)
+	Merge(with ProviderInstallationMethod) bool
 }
 
 type ProviderInstallationDirect struct {
@@ -35,11 +78,54 @@ func NewProviderInstallationDirect(include, exclude []string) *ProviderInstallat
 	return res
 }
 
+func (method *ProviderInstallationDirect) Merge(with ProviderInstallationMethod) bool {
+	if with, ok := with.(*ProviderInstallationDirect); ok {
+		if with.Exclude != nil {
+			method.AppendExclude(*with.Exclude)
+		}
+		if with.Include != nil {
+			method.AppendInclude(*with.Include)
+		}
+		return true
+	}
+
+	return false
+}
+
+func (method *ProviderInstallationDirect) AppendInclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
+	if method.Include == nil {
+		method.Include = &[]string{}
+	}
+	*method.Include = util.RemoveDuplicatesFromList(append(*method.Include, addrs...))
+}
+
 func (method *ProviderInstallationDirect) AppendExclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
 	if method.Exclude == nil {
 		method.Exclude = &[]string{}
 	}
-	*method.Exclude = append(*method.Exclude, addrs...)
+	*method.Exclude = util.RemoveDuplicatesFromList(append(*method.Exclude, addrs...))
+}
+
+func (method *ProviderInstallationDirect) RemoveExclude(addrs []string) {
+	if len(addrs) == 0 || method.Exclude == nil {
+		return
+	}
+	*method.Exclude = util.RemoveSublistFromList(*method.Exclude, addrs)
+
+	if len(*method.Exclude) == 0 {
+		method.Exclude = nil
+	}
+}
+
+func (method *ProviderInstallationDirect) String() string {
+	b, _ := json.Marshal(method) //nolint:errcheck
+	return string(b)
 }
 
 type ProviderInstallationFilesystemMirror struct {
@@ -66,11 +152,54 @@ func NewProviderInstallationFilesystemMirror(path string, include, exclude []str
 	return res
 }
 
+func (method *ProviderInstallationFilesystemMirror) Merge(with ProviderInstallationMethod) bool {
+	if with, ok := with.(*ProviderInstallationFilesystemMirror); ok && method.Path == with.Path {
+		if with.Exclude != nil {
+			method.AppendExclude(*with.Exclude)
+		}
+		if with.Include != nil {
+			method.AppendInclude(*with.Include)
+		}
+		return true
+	}
+
+	return false
+}
+
+func (method *ProviderInstallationFilesystemMirror) AppendInclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
+	if method.Include == nil {
+		method.Include = &[]string{}
+	}
+	*method.Include = util.RemoveDuplicatesFromList(append(*method.Include, addrs...))
+}
+
 func (method *ProviderInstallationFilesystemMirror) AppendExclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
 	if method.Exclude == nil {
 		method.Exclude = &[]string{}
 	}
-	*method.Exclude = append(*method.Exclude, addrs...)
+	*method.Exclude = util.RemoveDuplicatesFromList(append(*method.Exclude, addrs...))
+}
+
+func (method *ProviderInstallationFilesystemMirror) RemoveExclude(addrs []string) {
+	if len(addrs) == 0 || method.Exclude == nil {
+		return
+	}
+	*method.Exclude = util.RemoveSublistFromList(*method.Exclude, addrs)
+
+	if len(*method.Exclude) == 0 {
+		method.Exclude = nil
+	}
+}
+
+func (method *ProviderInstallationFilesystemMirror) String() string {
+	b, _ := json.Marshal(method) //nolint:errcheck
+	return string(b)
 }
 
 type ProviderInstallationNetworkMirror struct {
@@ -97,9 +226,52 @@ func NewProviderInstallationNetworkMirror(url string, include, exclude []string)
 	return res
 }
 
+func (method *ProviderInstallationNetworkMirror) Merge(with ProviderInstallationMethod) bool {
+	if with, ok := with.(*ProviderInstallationNetworkMirror); ok && method.URL == with.URL {
+		if with.Exclude != nil {
+			method.AppendExclude(*with.Exclude)
+		}
+		if with.Include != nil {
+			method.AppendInclude(*with.Include)
+		}
+		return true
+	}
+
+	return false
+}
+
+func (method *ProviderInstallationNetworkMirror) AppendInclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
+	if method.Include == nil {
+		method.Include = &[]string{}
+	}
+	*method.Include = util.RemoveDuplicatesFromList(append(*method.Include, addrs...))
+}
+
 func (method *ProviderInstallationNetworkMirror) AppendExclude(addrs []string) {
+	if len(addrs) == 0 {
+		return
+	}
 	if method.Exclude == nil {
 		method.Exclude = &[]string{}
 	}
-	*method.Exclude = append(*method.Exclude, addrs...)
+	*method.Exclude = util.RemoveDuplicatesFromList(append(*method.Exclude, addrs...))
+}
+
+func (method *ProviderInstallationNetworkMirror) RemoveExclude(addrs []string) {
+	if len(addrs) == 0 || method.Exclude == nil {
+		return
+	}
+	*method.Exclude = util.RemoveSublistFromList(*method.Exclude, addrs)
+
+	if len(*method.Exclude) == 0 {
+		method.Exclude = nil
+	}
+}
+
+func (method *ProviderInstallationNetworkMirror) String() string {
+	b, _ := json.Marshal(method) //nolint:errcheck
+	return string(b)
 }

test/cliconfig.go

@@ -18,7 +18,7 @@ provider_installation {
 {{ if gt (len $method.Include) 0 }}
     include = [{{ range $index, $include := $method.Include }}{{ if $index }},{{ end }}"{{ $include }}"{{ end }}]
 {{ end }}{{ if gt (len $method.Exclude) 0 }}
-    include = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
+    exclude = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
 {{ end }}
   }
 {{ end }}{{ end }}
@@ -28,7 +28,7 @@ provider_installation {
 {{ if gt (len $method.Include) 0 }}
     include = [{{ range $index, $include := $method.Include }}{{ if $index }},{{ end }}"{{ $include }}"{{ end }}]
 {{ end }}{{ if gt (len $method.Exclude) 0 }}
-    include = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
+    exclude = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
 {{ end }}
   }
 {{ end }}{{ end }}
@@ -37,7 +37,7 @@ provider_installation {
 {{ if gt (len $method.Include) 0 }}
     include = [{{ range $index, $include := $method.Include }}{{ if $index }},{{ end }}"{{ $include }}"{{ end }}]
 {{ end }}{{ if gt (len $method.Exclude) 0 }}
-    include = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
+    exclude = [{{ range $index, $exclude := $method.Exclude }}{{ if $index }},{{ end }}"{{ $exclude }}"{{ end }}]
 {{ end }}
   }
 {{ end }}{{ end }}

test/fixture-provider-cache/mirror/app/main.tf

@@ -1,8 +1,15 @@
 terraform {
   required_providers {
     null = {
-      source  = "example.com/hashicorp/null"
-      version = "3.2.2"
+      source = "registry.opentofu.org/hashicorp/null"
+    }
+    aws = {
+      source  = "example.com/hashicorp/aws"
+      version = "5.59.0"
+    }
+    azurerm = {
+      source  = "example.com/hashicorp/azurerm"
+      version = "3.113.0"
     }
   }
 }