fix: Caching the same provider with different versions from various s…

…ources (#3935)

* fix: caching the same provider from different sources

* chore: update test

* fix: addressed to coderabbitai review

cli/provider_cache.go

@@ -96,47 +96,21 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 	}
 
 	providerService := services.NewProviderService(opts.ProviderCacheDir, userProviderDir, cliCfg.CredentialsSource(), opts.Logger)
+	proxyProviderHandler := handlers.NewProxyProviderHandler(opts.Logger, cliCfg.CredentialsSource())
 
-	var (
-		providerHandlers = make([]handlers.ProviderHandler, 0, len(cliCfg.ProviderInstallation.Methods))
-		excludeAddrs     = make([]string, 0, len(cliCfg.ProviderInstallation.Methods))
-		directIsdefined  bool
-	)
-
-	for _, registryName := range opts.ProviderCacheRegistryNames {
-		excludeAddrs = append(excludeAddrs, registryName+"/*/*")
-	}
-
-	for _, method := range cliCfg.ProviderInstallation.Methods {
-		switch method := method.(type) {
-		case *cliconfig.ProviderInstallationFilesystemMirror:
-			providerHandlers = append(providerHandlers, handlers.NewProviderFilesystemMirrorHandler(providerService, CacheProviderHTTPStatusCode, method))
-		case *cliconfig.ProviderInstallationNetworkMirror:
-			networkMirrorHandler, err := handlers.NewProviderNetworkMirrorHandler(providerService, CacheProviderHTTPStatusCode, method, cliCfg.CredentialsSource())
-			if err != nil {
-				return nil, err
-			}
-
-			providerHandlers = append(providerHandlers, networkMirrorHandler)
-		case *cliconfig.ProviderInstallationDirect:
-			providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, CacheProviderHTTPStatusCode, method, cliCfg.CredentialsSource()))
-			directIsdefined = true
-		}
-
-		method.AppendExclude(excludeAddrs)
-	}
-
-	if !directIsdefined {
-		// In a case if none of direct provider installation methods `cliCfg.ProviderInstallation.Methods` are specified.
-		providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, CacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect), cliCfg.CredentialsSource()))
+	providerHandlers, err := handlers.NewProviderHandlers(cliCfg, opts.Logger, opts.ProviderCacheRegistryNames)
+	if err != nil {
+		return nil, errors.Errorf("creating provider handlers failed: %w", err)
 	}
 
 	cache := cache.NewServer(
 		cache.WithHostname(opts.ProviderCacheHostname),
 		cache.WithPort(opts.ProviderCachePort),
 		cache.WithToken(opts.ProviderCacheToken),
-		cache.WithServices(providerService),
+		cache.WithProviderService(providerService),
 		cache.WithProviderHandlers(providerHandlers...),
+		cache.WithProxyProviderHandler(proxyProviderHandler),
+		cache.WithCacheProviderHTTPStatusCode(CacheProviderHTTPStatusCode),
 		cache.WithLogger(opts.Logger),
 	)
 

cli/provider_cache_test.go

@@ -48,7 +48,7 @@ func TestProviderCache(t *testing.T) {
 	providerCacheDir := t.TempDir()
 	pluginCacheDir := t.TempDir()
 
-	opts := []cache.Option{cache.WithToken(token)}
+	opts := []cache.Option{cache.WithToken(token), cache.WithCacheProviderHTTPStatusCode(cli.CacheProviderHTTPStatusCode)}
 
 	testCases := []struct {
 		opts               []cache.Option
@@ -112,11 +112,17 @@ func TestProviderCache(t *testing.T) {
 			defer cancel()
 
 			errGroup, ctx := errgroup.WithContext(ctx)
+			logger := log.New()
 
-			providerService := services.NewProviderService(providerCacheDir, pluginCacheDir, nil, log.New())
-			providerHandler := handlers.NewProviderDirectHandler(providerService, cli.CacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect), nil)
+			providerService := services.NewProviderService(providerCacheDir, pluginCacheDir, nil, logger)
+			providerHandler := handlers.NewDirectProviderHandler(logger, new(cliconfig.ProviderInstallationDirect), nil)
+			proxyProviderHandler := handlers.NewProxyProviderHandler(logger, nil)
 
-			testCase.opts = append(testCase.opts, cache.WithServices(providerService), cache.WithProviderHandlers(providerHandler))
+			testCase.opts = append(testCase.opts,
+				cache.WithProviderService(providerService),
+				cache.WithProviderHandlers(providerHandler),
+				cache.WithProxyProviderHandler(proxyProviderHandler),
+			)
 
 			server := cache.NewServer(testCase.opts...)
 			ln, err := server.Listen()

pkg/log/format/format.go

@@ -66,6 +66,12 @@ func NewPrettyFormatPlaceholders() Placeholders {
 		Message(
 			PathFormat(RelativePath),
 		),
+		Field(CacheServerURLKeyName,
+			Prefix(" "+CacheServerURLKeyName+"="),
+		),
+		Field(CacheServerStatusKeyName,
+			Prefix(" "+CacheServerStatusKeyName+"="),
+		),
 	}
 }
 

pkg/log/format/placeholders/field.go

@@ -10,6 +10,10 @@ const (
 	TFPathKeyName      = "tf-path"
 	TFCmdArgsKeyName   = "tf-command-args"
 	TFCmdKeyName       = "tf-command"
+
+	// Terragrunt Provider Cache Server fields.
+	CacheServerURLKeyName    = "url"
+	CacheServerStatusKeyName = "status"
 )
 
 type fieldPlaceholder struct {

test/fixtures/provider-cache/network-mirror/apps/app1/main.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "example.com/hashicorp/aws"
+      version = "5.58.0"
+    }
+  }
+}

test/fixtures/provider-cache/network-mirror/apps/app1/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty

test/integration_serial_test.go

@@ -104,7 +104,7 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureProviderCacheNetworkMirror)
 	rootPath := util.JoinPath(tmpEnvPath, testFixtureProviderCacheNetworkMirror)
 
-	appPath := filepath.Join(rootPath, "app")
+	appsPath := filepath.Join(rootPath, "apps")
 	providersNetkworMirrorPath := filepath.Join(rootPath, "providers-network-mirror")
 	providersFilesystemMirrorPath := filepath.Join(rootPath, "providers-filesystem-mirror")
 
@@ -122,6 +122,16 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 	netowrkProvider.CreateMirror(t, providersNetkworMirrorPath)
 
 	filesystemProvider := FakeProvider{
+		RegistryName: "example.com",
+		Namespace:    "hashicorp",
+		Name:         "aws",
+		Version:      "5.58.0",
+		PlatformOS:   runtime.GOOS,
+		PlatformArch: runtime.GOARCH,
+	}
+	filesystemProvider.CreateMirror(t, providersFilesystemMirrorPath)
+
+	filesystemProvider = FakeProvider{
 		RegistryName: "example.com",
 		Namespace:    "hashicorp",
 		Name:         "azurerm",
@@ -169,7 +179,7 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 		FilesystemMirrorMethods: []test.CLIConfigProviderInstallationFilesystemMirror{
 			{
 				Path:    providersFilesystemMirrorPath,
-				Include: []string{"example.com/hashicorp/azurerm"},
+				Include: []string{"example.com/hashicorp/azurerm", "example.com/hashicorp/aws"},
 			},
 		},
 		NetworkMirrorMethods: []test.CLIConfigProviderInstallationNetworkMirror{
@@ -181,16 +191,18 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 	}
 	test.CreateCLIConfig(t, cliConfigFilename, cliConfigSettings)
 
-	helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath))
+	helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appsPath))
 
-	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" exclude = ["example.com/hashicorp/azurerm", "example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } }`
+	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm", "example.com/hashicorp/aws"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" exclude = ["example.com/hashicorp/azurerm", "example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } }`
 	expectedProviderInstallation = fmt.Sprintf(strings.Join(strings.Fields(expectedProviderInstallation), " "), providersFilesystemMirrorPath, networkMirrorURL.String(), providerCacheDir)
 
-	terraformrcBytes, err := os.ReadFile(filepath.Join(appPath, ".terraformrc"))
-	require.NoError(t, err)
-	terraformrc := strings.Join(strings.Fields(string(terraformrcBytes)), " ")
+	for _, filename := range []string{"app0/.terraformrc", "app1/.terraformrc"} {
+		terraformrcBytes, err := os.ReadFile(filepath.Join(appsPath, filename))
+		require.NoError(t, err)
+		terraformrc := strings.Join(strings.Fields(string(terraformrcBytes)), " ")
 
-	assert.Contains(t, terraformrc, expectedProviderInstallation, "%s\n\n%s", terraformrc, expectedProviderInstallation)
+		assert.Contains(t, terraformrc, expectedProviderInstallation, "%s\n\n%s", terraformrc, expectedProviderInstallation)
+	}
 }
 
 func TestTerragruntInputsFromDependency(t *testing.T) {

tf/cache/config.go

@@ -44,9 +44,9 @@ func WithToken(token string) Option {
 	}
 }
 
-func WithServices(services ...services.Service) Option {
+func WithProviderService(service *services.ProviderService) Option {
 	return func(cfg Config) Config {
-		cfg.services = services
+		cfg.providerService = service
 		return cfg
 	}
 }
@@ -58,6 +58,20 @@ func WithProviderHandlers(handlers ...handlers.ProviderHandler) Option {
 	}
 }
 
+func WithProxyProviderHandler(handler *handlers.ProxyProviderHandler) Option {
+	return func(cfg Config) Config {
+		cfg.proxyProviderHandler = handler
+		return cfg
+	}
+}
+
+func WithCacheProviderHTTPStatusCode(statusCode int) Option {
+	return func(cfg Config) Config {
+		cfg.cacheProviderHTTPStatusCode = statusCode
+		return cfg
+	}
+}
+
 func WithLogger(logger log.Logger) Option {
 	return func(cfg Config) Config {
 		cfg.logger = logger
@@ -71,8 +85,10 @@ type Config struct {
 	token           string
 	shutdownTimeout time.Duration
 
-	services         []services.Service
-	providerHandlers handlers.ProviderHandlers
+	providerService             *services.ProviderService
+	providerHandlers            handlers.ProviderHandlers
+	proxyProviderHandler        *handlers.ProxyProviderHandler
+	cacheProviderHTTPStatusCode int
 
 	logger log.Logger
 }

tf/cache/controllers/downloader.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/tf/cache/handlers"
 	"github.com/gruntwork-io/terragrunt/tf/cache/models"
 	"github.com/gruntwork-io/terragrunt/tf/cache/router"
+	"github.com/gruntwork-io/terragrunt/tf/cache/services"
 	"github.com/labstack/echo/v4"
 )
 
@@ -17,7 +18,8 @@ const (
 type DownloaderController struct {
 	*router.Router
 
-	ProviderHandlers []handlers.ProviderHandler
+	ProviderService      *services.ProviderService
+	ProxyProviderHandler *handlers.ProxyProviderHandler
 }
 
 // Register implements router.Controller.Register
@@ -45,13 +47,16 @@ func (controller *DownloaderController) downloadProviderAction(ctx echo.Context)
 		},
 	}
 
-	for _, handler := range controller.ProviderHandlers {
-		if handler.CanHandleProvider(provider) {
-			if err := handler.Download(ctx, provider); err == nil {
-				break
-			}
+	if cache := controller.ProviderService.GetProviderCache(provider); cache != nil {
+		if path := cache.ArchivePath(); path != "" {
+			controller.ProviderService.Logger().Debugf("Download cached provider %s", cache.Provider)
+			return ctx.File(path)
 		}
 	}
 
+	if err := controller.ProxyProviderHandler.Download(ctx, provider); err != nil {
+		return err
+	}
+
 	return ctx.NoContent(http.StatusNotFound)
 }

tf/cache/controllers/provider.go

@@ -4,9 +4,11 @@ package controllers
 import (
 	"net/http"
 
+	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/gruntwork-io/terragrunt/tf/cache/handlers"
 	"github.com/gruntwork-io/terragrunt/tf/cache/models"
 	"github.com/gruntwork-io/terragrunt/tf/cache/router"
+	"github.com/gruntwork-io/terragrunt/tf/cache/services"
 	"github.com/labstack/echo/v4"
 )
 
@@ -20,11 +22,15 @@ const (
 type ProviderController struct {
 	*router.Router
 
+	Logger               log.Logger
 	Server               http.Server
 	DownloaderController router.Controller
 
-	AuthMiddleware   echo.MiddlewareFunc
-	ProviderHandlers []handlers.ProviderHandler
+	AuthMiddleware              echo.MiddlewareFunc
+	ProviderHandlers            []handlers.ProviderHandler
+	ProxyProviderHandler        *handlers.ProxyProviderHandler
+	ProviderService             *services.ProviderService
+	CacheProviderHTTPStatusCode int
 }
 
 // Endpoints implements controllers.Endpointer.Endpoints
@@ -65,15 +71,30 @@ func (controller *ProviderController) getVersionsAction(ctx echo.Context) error
 		Name:         name,
 	}
 
+	var allVersions models.Versions
+
 	for _, handler := range controller.ProviderHandlers {
 		if handler.CanHandleProvider(provider) {
-			if err := handler.GetVersions(ctx, provider); err == nil {
-				break
+			versions, err := handler.GetVersions(ctx.Request().Context(), provider)
+			if err != nil {
+				controller.Logger.Errorf("Failed to get provider versions from %q: %s", handler, err.Error())
+			}
+
+			if versions != nil {
+				allVersions = append(allVersions, versions...)
 			}
 		}
 	}
 
-	return ctx.NoContent(http.StatusNotFound)
+	versions := struct {
+		ID       string          `json:"id"`
+		Versions models.Versions `json:"versions"`
+	}{
+		ID:       provider.Address(),
+		Versions: allVersions,
+	}
+
+	return ctx.JSON(http.StatusOK, versions)
 }
 
 func (controller *ProviderController) getPlatformsAction(ctx echo.Context) (er error) {
@@ -96,13 +117,32 @@ func (controller *ProviderController) getPlatformsAction(ctx echo.Context) (er e
 		Arch:         arch,
 	}
 
+	if cacheRequestID == "" {
+		return controller.ProxyProviderHandler.GetPlatform(ctx, provider, controller.DownloaderController)
+	}
+
+	var (
+		resp *models.ResponseBody
+		err  error
+	)
+
 	for _, handler := range controller.ProviderHandlers {
 		if handler.CanHandleProvider(provider) {
-			if err := handler.GetPlatform(ctx, provider, controller.DownloaderController, cacheRequestID); err == nil {
+			resp, err = handler.GetPlatform(ctx.Request().Context(), provider)
+			if err != nil {
+				controller.Logger.Errorf("Failed to get provider platform from %q: %s", handler, err.Error())
+			}
+
+			if resp != nil {
 				break
 			}
 		}
 	}
 
-	return ctx.NoContent(http.StatusNotFound)
+	provider.ResponseBody = resp
+
+	// start caching and return 423 status
+	controller.ProviderService.CacheProvider(ctx.Request().Context(), cacheRequestID, provider)
+
+	return ctx.NoContent(controller.CacheProviderHTTPStatusCode)
 }