Allow custom data templating in images

docs/content/docs/bundle/manifest/_index.md

@@ -545,6 +545,24 @@ A last note on `digest`.  Taking the example of the library `nginx` Docker image
 ]
 ```
 
+{{< callout type="info" >}}
+  Custom data in the `images` section is support from v1.2.0
+{{< /callout >}}
+
+It is possible to make the `images` section more dynamic by using [custom data](#custom). It is the only templating supported in the section.
+
+```yaml
+custom:
+  websvcDigest: "sha256:85b1a9b4b60a4cf73a23517dad677e64edf467107fa7d58fce9c50e6a3e4c914"
+
+images:
+  websvc:
+      description: "A simple web service"
+      imageType: "docker"
+      repository: "jeremyrickard/devops-days-msp"
+      digest: "${ bundle.custom.websvcDigest }"
+```
+
 ## Custom
 
 The Custom section of a Porter manifest is intended for bundle authors to

pkg/cnab/config-adapter/helpers.go

@@ -13,7 +13,7 @@ import (
 
 func LoadTestBundle(t *testing.T, config *config.Config, path string) cnab.ExtendedBundle {
 	ctx := context.Background()
-	m, err := manifest.ReadManifest(config.Context, path, config)
+	m, err := manifest.ReadManifest(config.Context, path, config, false)
 	require.NoError(t, err)
 	b, err := ConvertToTestBundle(ctx, config, m)
 	require.NoError(t, err)

pkg/manifest/manifest.go

@@ -1291,7 +1291,7 @@ func ReadManifestData(cxt *portercontext.Context, path string) ([]byte, error) {
 
 // ReadManifest determines if specified path is a URL or a filepath.
 // After reading the data in the path it returns a Manifest and any errors
-func ReadManifest(cxt *portercontext.Context, path string, config *config.Config) (*Manifest, error) {
+func ReadManifest(cxt *portercontext.Context, path string, config *config.Config, allowMissingVariables bool) (*Manifest, error) {
 	data, err := ReadManifestData(cxt, path)
 	if err != nil {
 		return nil, err
@@ -1302,6 +1302,26 @@ func ReadManifest(cxt *portercontext.Context, path string, config *config.Config
 		return nil, fmt.Errorf("unsupported property set or a custom action is defined incorrectly: %w", err)
 	}
 
+	// Map custom values to image section
+	imageSection, err := yaml.Marshal(m.ImageMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal image section: %w", err)
+	}
+	mustache.AllowMissingVariables = allowMissingVariables
+	imageContext := map[string]interface{}{
+		"bundle": map[string]interface{}{
+			"custom": m.Custom,
+		},
+	}
+	renderedImageSection, err := mustache.RenderRaw(m.GetTemplatePrefix()+string(imageSection), true, imageContext)
+	if err != nil {
+		return nil, fmt.Errorf("failed to render image section: %w", err)
+	}
+	err = yaml.Unmarshal([]byte(renderedImageSection), &m.ImageMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal image section: %w", err)
+	}
+
 	tmplResult, err := m.ScanManifestTemplating(data, config)
 	if err != nil {
 		return nil, err
@@ -1424,10 +1444,20 @@ func (m *Manifest) getTemplateVariables(data string) (map[string]struct{}, error
 // LoadManifestFrom reads and validates the manifest at the specified location,
 // and returns a populated Manifest structure.
 func LoadManifestFrom(ctx context.Context, config *config.Config, file string) (*Manifest, error) {
+	return loadManifestFrom(ctx, config, file, false)
+}
+
+// LoadManifestFromAllowMissingVariables reads and validates the manifest at the specified location,
+// and returns a populated Manifest structure, while allowing missing variables.
+func LoadManifestFromAllowMissingVariables(ctx context.Context, config *config.Config, file string) (*Manifest, error) {
+	return loadManifestFrom(ctx, config, file, true)
+}
+
+func loadManifestFrom(ctx context.Context, config *config.Config, file string, allowMissingVariables bool) (*Manifest, error) {
 	ctx, log := tracing.StartSpan(ctx)
 	defer log.EndSpan()
 
-	m, err := ReadManifest(config.Context, file, config)
+	m, err := ReadManifest(config.Context, file, config, allowMissingVariables)
 	if err != nil {
 		return nil, err
 	}

pkg/manifest/manifest_test.go

@@ -263,7 +263,7 @@ func TestManifest_Validate_SchemaVersion(t *testing.T) {
 		cfg.TestContext.UseFilesystem()
 		cfg.Data.SchemaCheck = string(schema.CheckStrategyExact)
 
-		m, err := ReadManifest(cfg.Context, "testdata/porter.yaml", cfg.Config)
+		m, err := ReadManifest(cfg.Context, "testdata/porter.yaml", cfg.Config, false)
 		require.NoError(t, err)
 
 		err = m.Validate(ctx, cfg.Config)
@@ -281,7 +281,7 @@ func TestManifest_Validate_SchemaVersion(t *testing.T) {
 		cfg.TestContext.EditYaml("porter.yaml", func(yq *yaml.Editor) error {
 			return yq.SetValue("schemaVersion", "1.1.0")
 		})
-		m, err := ReadManifest(cfg.Context, "porter.yaml", cfg.Config)
+		m, err := ReadManifest(cfg.Context, "porter.yaml", cfg.Config, false)
 		require.NoError(t, err)
 
 		err = m.Validate(ctx, cfg.Config)
@@ -300,7 +300,7 @@ func TestManifest_Validate_SchemaVersion(t *testing.T) {
 		defer span.EndSpan()
 		cfg.Data.SchemaCheck = string(schema.CheckStrategyNone)
 
-		m, err := ReadManifest(cfg.Context, "testdata/porter.yaml", cfg.Config)
+		m, err := ReadManifest(cfg.Context, "testdata/porter.yaml", cfg.Config, false)
 		require.NoError(t, err)
 
 		m.SchemaVersion = ""
@@ -415,15 +415,15 @@ func TestManifest_Validate_WrongSchema(t *testing.T) {
 func TestReadManifest_URL(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	url := "https://raw.githubusercontent.com/getporter/porter/v0.27.1/pkg/manifest/testdata/simple.porter.yaml"
-	m, err := ReadManifest(cxt.Context, url, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, url, config.NewTestConfig(t).Config, false)
 
 	require.NoError(t, err)
 	assert.Equal(t, "hello", m.Name)
 }
 
 func TestReadManifest_Validate_InvalidURL(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
-	_, err := ReadManifest(cxt.Context, "http://fake-example-porter", config.NewTestConfig(t).Config)
+	_, err := ReadManifest(cxt.Context, "http://fake-example-porter", config.NewTestConfig(t).Config, false)
 
 	assert.Error(t, err)
 	assert.Regexp(t, "could not reach url http://fake-example-porter", err)
@@ -432,7 +432,7 @@ func TestReadManifest_Validate_InvalidURL(t *testing.T) {
 func TestReadManifest_File(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/simple.porter.yaml", config.Name)
-	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 
 	require.NoError(t, err)
 	assert.Equal(t, "hello", m.Name)
@@ -562,15 +562,15 @@ func TestSetDefaults(t *testing.T) {
 
 func TestReadManifest_Validate_MissingFile(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
-	_, err := ReadManifest(cxt.Context, "fake-porter.yaml", config.NewTestConfig(t).Config)
+	_, err := ReadManifest(cxt.Context, "fake-porter.yaml", config.NewTestConfig(t).Config, false)
 
 	assert.EqualError(t, err, "the specified porter configuration file fake-porter.yaml does not exist")
 }
 
 func TestMixinDeclaration_UnmarshalYAML(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/mixin-with-config.yaml", config.Name)
-	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 
 	require.NoError(t, err)
 	assert.Len(t, m.Mixins, 3, "expected 3 mixins")
@@ -583,7 +583,7 @@ func TestMixinDeclaration_UnmarshalYAML(t *testing.T) {
 func TestMixinDeclaration_UnmarshalYAML_Invalid(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/mixin-with-bad-config.yaml", config.Name)
-	_, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	_, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mixin declaration contained more than one mixin")
@@ -598,7 +598,7 @@ func TestCredentialsDefinition_UnmarshalYAML(t *testing.T) {
 	t.Run("all credentials in the generated manifest file are required", func(t *testing.T) {
 		cxt := portercontext.NewTestContext(t)
 		cxt.AddTestFile("testdata/with-credentials.yaml", config.Name)
-		m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+		m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 		require.NoError(t, err)
 		assertAllCredentialsRequired(t, m.Credentials)
 
@@ -851,6 +851,24 @@ func TestLoadManifestWithCustomData(t *testing.T) {
 	require.Equal(t, "two", val10, "test5[1] value is unexpected")
 }
 
+func TestLoadManifestWithImageSectionUsingCustomData(t *testing.T) {
+	c := config.NewTestConfig(t)
+
+	c.TestContext.AddTestFile("testdata/porter-with-custom-image-section.yaml", config.Name)
+
+	m, err := LoadManifestFrom(context.Background(), c.Config, config.Name)
+	require.NoError(t, err, "could not load manifest")
+
+	require.NotNil(t, m, "manifest was nil")
+	val, ok := m.Custom["myApp"].(map[string]interface{})
+	require.True(t, ok, "Cannot cast foo value to map[string]interface{}")
+
+	myAppImage := m.ImageMap["myApp"]
+	expectedDigest, ok := val["digest"].(string)
+	require.True(t, ok, "Cannot cast digest to string")
+	require.Equal(t, expectedDigest, myAppImage.Digest, "digest in image should be taken from custom value")
+}
+
 func TestLoadManifestWithRequiredExtensions(t *testing.T) {
 	c := config.NewTestConfig(t)
 
@@ -878,7 +896,7 @@ func TestLoadManifestWithRequiredExtensions(t *testing.T) {
 func TestReadManifest_WithTemplateVariables(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/porter-with-templating.yaml", config.Name)
-	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 	require.NoError(t, err, "ReadManifest failed")
 	wantVars := []string{"bundle.dependencies.mysql.outputs.mysql-password", "bundle.outputs.msg", "bundle.outputs.name"}
 	assert.Equal(t, wantVars, m.TemplateVariables)
@@ -887,7 +905,7 @@ func TestReadManifest_WithTemplateVariables(t *testing.T) {
 func TestManifest_GetTemplatedOutputs(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/porter-with-templating.yaml", config.Name)
-	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 	require.NoError(t, err, "ReadManifest failed")
 
 	outputs := m.GetTemplatedOutputs()
@@ -899,7 +917,7 @@ func TestManifest_GetTemplatedOutputs(t *testing.T) {
 func TestManifest_GetTemplatedDependencyOutputs(t *testing.T) {
 	cxt := portercontext.NewTestContext(t)
 	cxt.AddTestFile("testdata/porter-with-templating.yaml", config.Name)
-	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config)
+	m, err := ReadManifest(cxt.Context, config.Name, config.NewTestConfig(t).Config, false)
 	require.NoError(t, err, "ReadManifest failed")
 
 	outputs := m.GetTemplatedDependencyOutputs()

pkg/manifest/testdata/porter-with-custom-image-section.yaml

@@ -0,0 +1,27 @@
+schemaVersion: 1.0.0
+name: mybundle
+version: 0.1.0
+registry: example.com
+
+mixins:
+  - exec
+
+custom:
+  myApp: 
+    digest: sha256:6b91f0ccfc945d46b8f76937cae9acb7123f210b05105ef9a4b6f0e23e9cdcb6
+
+images:
+  myApp:
+    repository: test/repo
+    digest: ${ bundle.custom.myApp.digest }
+
+install:
+  - exec:
+      description: "Install Hello World"
+      command: bash
+
+
+uninstall:
+  - exec:
+      description: "Uninstall Hello World"
+      command: bash

pkg/porter/build.go

@@ -132,7 +132,7 @@ func (p *Porter) Build(ctx context.Context, opts BuildOptions) error {
 	m.ManifestPath = opts.File
 
 	if !opts.NoLint {
-		if err := p.preLint(ctx, opts.File); err != nil {
+		if err := p.preLint(ctx, build.LOCAL_MANIFEST); err != nil {
 			return err
 		}
 	}

pkg/porter/lint.go

@@ -55,7 +55,7 @@ func (o *LintOptions) validateFile(cxt *portercontext.Context) error {
 // Lint porter.yaml for any problems and report the results.
 // This calls the mixins to analyze their sections of the manifest.
 func (p *Porter) Lint(ctx context.Context, opts LintOptions) (linter.Results, error) {
-	manifest, err := manifest.LoadManifestFrom(ctx, p.Config, opts.File)
+	manifest, err := manifest.LoadManifestFromAllowMissingVariables(ctx, p.Config, opts.File)
 	if err != nil {
 		return nil, err
 	}

pkg/porter/lint_test.go

@@ -67,6 +67,50 @@ func TestPorter_Lint(t *testing.T) {
 	assert.Len(t, results, 1, "Lint returned the wrong number of results")
 }
 
+func TestPorter_LintImageCustomDataWithDefault(t *testing.T) {
+	p := NewTestPorter(t)
+	defer p.Close()
+
+	p.TestConfig.TestContext.AddTestFile("testdata/porter-image-custom-data-default.yaml", "porter.yaml")
+
+	mixins := p.Mixins.(*mixin.TestMixinProvider)
+	mixins.LintResults = linter.Results{
+		{
+			Level: linter.LevelError,
+		},
+	}
+
+	var opts LintOptions
+	err := opts.Validate(p.Context)
+	require.NoError(t, err, "Validate failed")
+
+	results, err := p.Lint(context.Background(), opts)
+	require.NoError(t, err, "Lint failed")
+	assert.Len(t, results, 1, "Lint returned the wrong number of results")
+}
+
+func TestPorter_LintImageCustomDataWithoutDefault(t *testing.T) {
+	p := NewTestPorter(t)
+	defer p.Close()
+
+	p.TestConfig.TestContext.AddTestFile("testdata/porter-image-custom-data-no-default.yaml", "porter.yaml")
+
+	mixins := p.Mixins.(*mixin.TestMixinProvider)
+	mixins.LintResults = linter.Results{
+		{
+			Level: linter.LevelError,
+		},
+	}
+
+	var opts LintOptions
+	err := opts.Validate(p.Context)
+	require.NoError(t, err, "Validate failed")
+
+	results, err := p.Lint(context.Background(), opts)
+	require.NoError(t, err, "Lint failed")
+	assert.Len(t, results, 1, "Lint returned the wrong number of results")
+}
+
 func TestPorter_PrintLintResults(t *testing.T) {
 	lintResults := linter.Results{
 		{

pkg/porter/testdata/porter-image-custom-data-default.yaml

@@ -0,0 +1,39 @@
+schemaVersion: 1.0.0
+name: porter-hello
+version: 0.1.0
+description: "A bundle with a custom action"
+registry: "localhost:5000"
+
+custom:
+  digest: "sha256:6b5a28ccbb76f12ce771a23757880c6083234255c5ba191fca1c5db1f71c1687"
+
+images: 
+   something:
+      description: "an image"
+      imageType: "docker"
+      repository: "getporter/boo"
+      digest: "${ bundle.custom.digest }"
+
+mixins:
+  - exec
+
+install:
+  - exec:
+      description: "Install Hello World"
+      command: echo
+      arguments:
+        - "Hello world"
+
+upgrade:
+  - exec:
+      description: "World 2.0"
+      command: bash
+      flags:
+        c: echo World 2.0
+
+uninstall:
+  - exec:
+      description: "Uninstall Hello World"
+      command: bash
+      flags:
+        c: echo Goodbye World