Implement chart/manifest merging (#1975)

## Description

This implements more intelligent merging functionality based on the
uniqueness of names in `charts` and `manifests`

## Related Issue

Fixes #1976

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [X] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: razzle <[email protected]>

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -1119,13 +1119,11 @@ Must be one of:
 
 <details>
 <summary>
-<strong> <a name="components_items_charts_items_version"></a>version *</strong>
+<strong> <a name="components_items_charts_items_version"></a>version</strong>
 </summary>
 &nbsp;
 <blockquote>
 
-![Required](https://img.shields.io/badge/Required-red)
-
 **Description:** The version of the chart to deploy; for git-based charts this is also the tag of the git repo
 
 |          |          |

examples/composable-packages/README.md

@@ -18,6 +18,18 @@ You can create a skeleton package from a `zarf.yaml` by pointing `zarf package p
 zarf package publish path/containing/package/definition oci://your-registry.com
 ```
 
+## Merge Strategies
+
+When merging components together Zarf will adopt the following strategies depending on the kind of primitive (`files`, `required`, `manifests`) that it is merging:
+
+| Kind                       | Key(s)                                 | Description |
+|----------------------------|----------------------------------------|-------------|
+| Component Behavior         | `name`, `group`, `default`, `required` | These keys control how Zarf interacts with a given component and will _always_ take the value of the overriding component |
+| Component Description      | `description` | This key will only take the value of the overriding component if it is not empty |
+| Cosign Key Path            | `cosignKeyPath` | [Deprecated] This key will only take the value of the overriding component if it is not empty |
+| Un'name'd Primitive Arrays | `actions`, `dataInjections`, `files`, `images`, `repos` | These keys will append the overriding component's version of the array to the end of the base component's array |
+| 'name'd Primitive Arrays   | `charts`, `manifests` | For any given element in the overriding component, if the element matches based on `name` then its values will be merged with the base element of the same `name`. If not then the element will be appended to the end of the array |
+
 :::
 
 ## `zarf.yaml` {#zarf.yaml}
@@ -33,8 +45,8 @@ To view the example in its entirety, select the `Edit this page` link below the
 Creating this example requires a locally hosted container registry that has the `wordpress` skeleton package published and available. You can do this by running the following commands:
 
 ```bash
-docker run -d -p 5000:5000 --restart=always --name registry registry:2
-zarf package publish examples/wordpress oci://127.0.0.1:5000 --insecure
+docker run -d -p 555:5000 --restart=always --name registry registry:2
+zarf package publish examples/wordpress oci://127.0.0.1:555 --insecure
 ```
 
 You will also need to pass the `--insecure` flag to `zarf package create` to pull from the `http` registry:

examples/composable-packages/zarf.yaml

@@ -15,10 +15,12 @@ components:
       path: ../dos-games
       # Example optional custom name to point to in the imported package (default is to use this component's name)
       name: baseline
-    # Array keys are appended to the end of anything the imported component defined (in this case an extra service)
+    # 'name'd Zarf primitives will merge the arrays together on import:
+    # - 'manifests' of the same name will merge namespace, files and kustomizations
+    # - 'charts' of the same name will merge namespace, releaseName and valuesFiles
+    # Zarf primitives without matching 'name's will be appended to the end of the primitive's list for that component.
     manifests:
-      - name: additional-games
-        namespace: dos-games
+      - name: multi-games
         files:
           - quake-service.yaml
 
@@ -29,10 +31,10 @@ components:
     # default: false    # the initial value overrides the child component
     import:
       # The URL to the skeleton package containing this component's package definition
-      url: oci://localhost:5000/wordpress:16.0.4-skeleton
+      url: oci://localhost:555/wordpress:16.0.4-skeleton
       # Example optional custom name to point to in the imported package (default is to use this component's name)
       name: wordpress
-    # Array keys are appended to the end of anything the imported component defined (in this case an override of the blog name)
+    # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component.
     actions:
       onDeploy:
         before:

examples/variables/README.md

@@ -60,6 +60,12 @@ Zarf `variables` can also have additional fields that describe how Zarf will han
 
 <Properties item="ZarfPackageVariable" />
 
+:::info
+
+Variables with `type: file` will be set to the filepath in `actions` due to constraints on the size of environment variables in the shell.  This also allows for additional processing of the file by its filename.
+
+:::
+
 :::note
 
 The fields `default`, `description` and `prompt` are not available on `setVariables` since they always take the standard output of an action command and will not be interacted with directly by a deploy user.

examples/variables/zarf.yaml

@@ -54,7 +54,7 @@ components:
     actions:
       onDeploy:
         after:
-          # This command uses Zarf to return the SHASUM of the terraform file (`type: file` variables will return the filepath instead of the contents when used in actions)
+          # This command uses Zarf to return the SHASUM of the terraform file (`type: file` variables will return the filepath instead of the contents when used in actions due to constraints on env var size)
           - cmd: ./zarf prepare sha256sum ${ZARF_VAR_MODIFIED_TERRAFORM}
             # `mute` is set to exclude the command output from being shown (since we are treating it as sensitive below)
             mute: true

src/cmd/tools/common.go

@@ -16,6 +16,12 @@ var toolsCmd = &cobra.Command{
 	Aliases: []string{"t"},
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		config.SkipLogFile = true
+
+		// Skip for vendor-only commands
+		if common.CheckVendorOnlyFromPath(cmd) {
+			return
+		}
+
 		common.SetupCLI()
 	},
 	Short: lang.CmdToolsShort,

src/config/lang/english.go

@@ -568,6 +568,7 @@ const (
 	PkgValidateErrChart                   = "invalid chart definition: %w"
 	PkgValidateErrChartName               = "chart %s exceed the maximum length of %d characters"
 	PkgValidateErrChartNameMissing        = "chart %s must include a name"
+	PkgValidateErrChartNameNotUnique      = "chart name %q is not unique"
 	PkgValidateErrChartNamespaceMissing   = "chart %s must include a namespace"
 	PkgValidateErrChartURLOrPath          = "chart %s must only have a url or localPath"
 	PkgValidateErrChartVersion            = "chart %s must include a chart version"
@@ -586,6 +587,7 @@ const (
 	PkgValidateErrManifestFileOrKustomize = "manifest %s must have at least one file or kustomization"
 	PkgValidateErrManifestNameLength      = "manifest %s exceed the maximum length of %d characters"
 	PkgValidateErrManifestNameMissing     = "manifest %s must include a name"
+	PkgValidateErrManifestNameNotUnique   = "manifest name %q is not unique"
 	PkgValidateErrName                    = "invalid package name: %w"
 	PkgValidateErrPkgConstantName         = "constant name '%s' must be all uppercase and contain no special characters except _"
 	PkgValidateErrPkgName                 = "package name '%s' must be all lowercase and contain no special characters except -"

src/internal/packager/validate/validate.go

@@ -46,14 +46,14 @@ func Run(pkg types.ZarfPackage) error {
 		}
 	}
 
-	uniqueNames := make(map[string]bool)
+	uniqueComponentNames := make(map[string]bool)
 
 	for _, component := range pkg.Components {
 		// ensure component name is unique
-		if _, ok := uniqueNames[component.Name]; ok {
+		if _, ok := uniqueComponentNames[component.Name]; ok {
 			return fmt.Errorf(lang.PkgValidateErrComponentNameNotUnique, component.Name)
 		}
-		uniqueNames[component.Name] = true
+		uniqueComponentNames[component.Name] = true
 
 		if err := validateComponent(pkg, component); err != nil {
 			return fmt.Errorf(lang.PkgValidateErrComponent, err)
@@ -120,13 +120,27 @@ func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) err
 		}
 	}
 
+	uniqueChartNames := make(map[string]bool)
 	for _, chart := range component.Charts {
+		// ensure chart name is unique
+		if _, ok := uniqueChartNames[chart.Name]; ok {
+			return fmt.Errorf(lang.PkgValidateErrChartNameNotUnique, chart.Name)
+		}
+		uniqueChartNames[chart.Name] = true
+
 		if err := validateChart(chart); err != nil {
 			return fmt.Errorf(lang.PkgValidateErrChart, err)
 		}
 	}
 
+	uniqueManifestNames := make(map[string]bool)
 	for _, manifest := range component.Manifests {
+		// ensure manifest name is unique
+		if _, ok := uniqueManifestNames[manifest.Name]; ok {
+			return fmt.Errorf(lang.PkgValidateErrManifestNameNotUnique, manifest.Name)
+		}
+		uniqueManifestNames[manifest.Name] = true
+
 		if err := validateManifest(manifest); err != nil {
 			return fmt.Errorf(lang.PkgValidateErrManifest, err)
 		}

src/pkg/packager/compose.go

@@ -290,9 +290,9 @@ func (p *Packager) fixComposedActionFilepaths(pathAncestry string, actions []typ
 // Sets Name, Default, Required and Description to the original components values.
 func (p *Packager) mergeComponentOverrides(target *types.ZarfComponent, override types.ZarfComponent) {
 	target.Name = override.Name
+	target.Group = override.Group
 	target.Default = override.Default
 	target.Required = override.Required
-	target.Group = override.Group
 
 	// Override description if it was provided.
 	if override.Description != "" {
@@ -305,12 +305,52 @@ func (p *Packager) mergeComponentOverrides(target *types.ZarfComponent, override
 	}
 
 	// Append slices where they exist.
-	target.Charts = append(target.Charts, override.Charts...)
 	target.DataInjections = append(target.DataInjections, override.DataInjections...)
 	target.Files = append(target.Files, override.Files...)
 	target.Images = append(target.Images, override.Images...)
-	target.Manifests = append(target.Manifests, override.Manifests...)
 	target.Repos = append(target.Repos, override.Repos...)
+
+	// Merge charts with the same name to keep them unique
+	for _, overrideChart := range override.Charts {
+		existing := false
+		for idx := range target.Charts {
+			if target.Charts[idx].Name == overrideChart.Name {
+				if overrideChart.Namespace != "" {
+					target.Charts[idx].Namespace = overrideChart.Namespace
+				}
+				if overrideChart.ReleaseName != "" {
+					target.Charts[idx].ReleaseName = overrideChart.ReleaseName
+				}
+				target.Charts[idx].ValuesFiles = append(target.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...)
+				existing = true
+			}
+		}
+
+		if !existing {
+			target.Charts = append(target.Charts, overrideChart)
+		}
+	}
+
+	// Merge manifests with the same name to keep them unique
+	for _, overrideManifest := range override.Manifests {
+		existing := false
+		for idx := range target.Manifests {
+			if target.Manifests[idx].Name == overrideManifest.Name {
+				if overrideManifest.Namespace != "" {
+					target.Manifests[idx].Namespace = overrideManifest.Namespace
+				}
+				target.Manifests[idx].Files = append(target.Manifests[idx].Files, overrideManifest.Files...)
+				target.Manifests[idx].Kustomizations = append(target.Manifests[idx].Kustomizations, overrideManifest.Kustomizations...)
+
+				existing = true
+			}
+		}
+
+		if !existing {
+			target.Manifests = append(target.Manifests, overrideManifest)
+		}
+	}
+
 	// Check for nil array
 	if override.Extensions.BigBang != nil {
 		if override.Extensions.BigBang.ValuesFiles != nil {

src/pkg/utils/yaml.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/fatih/color"
 	goyaml "github.com/goccy/go-yaml"
@@ -85,15 +86,21 @@ func ColorPrintYAML(data any, hints map[string]string, spaceRootLists bool) {
 		}
 	}
 
-	colorizedYAML := p.PrintTokens(tokens)
+	outputYAML := p.PrintTokens(tokens)
 
 	// Inject the hints into the colorized YAML
 	for key, value := range hints {
-		colorizedYAML = strings.Replace(colorizedYAML, key, value, 1)
+		outputYAML = strings.Replace(outputYAML, key, value, 1)
+	}
+
+	if config.NoColor {
+		// If no color is specified strip any color codes from the output - https://regex101.com/r/YFyIwC/2
+		ansiRegex := regexp.MustCompile(`\x1b\[(.*?)m`)
+		outputYAML = ansiRegex.ReplaceAllString(outputYAML, "")
 	}
 
 	pterm.Println()
-	pterm.Println(colorizedYAML)
+	pterm.Println(outputYAML)
 }
 
 // AddRootListHint adds a hint string for a given root list key and value.

src/test/e2e/51_oci_compose_test.go

@@ -30,10 +30,14 @@ type SkeletonSuite struct {
 }
 
 var (
-	importEverything   = filepath.Join("src", "test", "packages", "51-import-everything")
-	importception      = filepath.Join("src", "test", "packages", "51-import-everything", "inception")
-	everythingExternal = filepath.Join("src", "test", "packages", "everything-external")
-	absNoCode          = filepath.Join("/", "tmp", "nocode")
+	composeExample       = filepath.Join("examples", "composable-packages")
+	composeExamplePath   string
+	importEverything     = filepath.Join("src", "test", "packages", "51-import-everything")
+	importEverythingPath string
+	importception        = filepath.Join("src", "test", "packages", "51-import-everything", "inception")
+	importceptionPath    string
+	everythingExternal   = filepath.Join("src", "test", "packages", "everything-external")
+	absNoCode            = filepath.Join("/", "tmp", "nocode")
 )
 
 func (suite *SkeletonSuite) SetupSuite() {
@@ -54,6 +58,11 @@ func (suite *SkeletonSuite) SetupSuite() {
 
 	e2e.SetupDockerRegistry(suite.T(), 555)
 	suite.Reference.Registry = "localhost:555"
+
+	// Setup the package paths after e2e has been initialized
+	composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch))
+	importEverythingPath = filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch))
+	importceptionPath = filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch))
 }
 
 func (suite *SkeletonSuite) TearDownSuite() {
@@ -64,16 +73,27 @@ func (suite *SkeletonSuite) TearDownSuite() {
 	suite.NoError(err)
 	err = os.RemoveAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local"))
 	suite.NoError(err)
-	err = os.RemoveAll(filepath.Join("files"))
+	err = os.RemoveAll("files")
+	suite.NoError(err)
+	err = os.RemoveAll(composeExamplePath)
+	suite.NoError(err)
+	err = os.RemoveAll(importEverythingPath)
+	suite.NoError(err)
+	err = os.RemoveAll(importceptionPath)
 	suite.NoError(err)
 }
 
 func (suite *SkeletonSuite) Test_0_Publish_Skeletons() {
 	suite.T().Log("E2E: Skeleton Package Publish oci://")
 	ref := suite.Reference.String()
 
+	wordpress := filepath.Join("examples", "wordpress")
+	_, stdErr, err := e2e.Zarf("package", "publish", wordpress, "oci://"+ref, "--insecure")
+	suite.NoError(err)
+	suite.Contains(stdErr, "Published "+ref)
+
 	helmCharts := filepath.Join("examples", "helm-charts")
-	_, stdErr, err := e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure")
+	_, stdErr, err = e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure")
 	suite.NoError(err)
 	suite.Contains(stdErr, "Published "+ref)
 
@@ -99,20 +119,48 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() {
 	suite.NoError(err)
 }
 
-func (suite *SkeletonSuite) Test_1_Compose() {
+func (suite *SkeletonSuite) Test_1_Compose_Example() {
 	suite.T().Log("E2E: Skeleton Package Compose oci://")
 
-	_, _, err := e2e.Zarf("package", "create", importEverything, "--confirm", "-o", "build", "--insecure")
+	_, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm")
 	suite.NoError(err)
 
-	_, _, err = e2e.Zarf("package", "create", importception, "--confirm", "-o", "build", "--insecure")
-	suite.NoError(err)
+	// Ensure that common names merge
+	suite.Contains(stdErr, `
+  manifests:
+  - name: multi-games
+    namespace: dos-games
+    files:
+    - ../dos-games/manifests/deployment.yaml
+    - ../dos-games/manifests/service.yaml
+    - quake-service.yaml`)
+
+	// Ensure that the action was appended
+	suite.Contains(stdErr, `
+  - docker.io/bitnami/wordpress:6.2.0-debian-11-r18
+  actions:
+    onDeploy:
+      before:
+      - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp}
+        setVariables:
+        - name: WORDPRESS_BLOG_NAME`)
+
+	// Ensure that the variables were merged
+	suite.Contains(stdErr, `
+- name: WORDPRESS_BLOG_NAME
+  description: The blog name that is used for the WordPress admin account
+  default: The Zarf Blog
+  prompt: true`)
 }
 
-func (suite *SkeletonSuite) Test_2_Component_Templates() {
-	suite.T().Log("E2E: Component Templates")
-	e2e.SetupWithCluster(suite.T())
-	importEverythingPath := fmt.Sprintf("build/zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)
+func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() {
+	suite.T().Log("E2E: Skeleton Package Compose oci://")
+
+	_, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm")
+	suite.NoError(err)
+
+	_, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--confirm")
+	suite.NoError(err)
 
 	_, stdErr, err := e2e.Zarf("package", "inspect", importEverythingPath)
 	suite.NoError(err)
@@ -133,7 +181,6 @@ func (suite *SkeletonSuite) Test_2_Component_Templates() {
 	for _, target := range targets {
 		suite.Contains(stdErr, target)
 	}
-
 }
 
 func (suite *SkeletonSuite) Test_3_FilePaths() {