feat: Allow overriding the default QA Mappings file (#1158)

feat: Allow overriding the default QA Mappings file

chore: Moved ports questions to a separate category
fix: Categories disabled in the QA mappings file were not being respected

---------

Signed-off-by: Akash Nayak <[email protected]>
Co-authored-by: Harikrishnan Balagopal <[email protected]>

Author: Akash-Nayak

assets/built-in/qa/qamappings.yaml

@@ -1,72 +1,80 @@
-categories:
-  - name: imageregistry
-    # or use the flags --enable imageregistry/--disable imageregistry
-    enabled: true
-    questions:
-      - move2kube.target.imageregistry.url
-      - move2kube.target.imageregistry.namespace
-      - move2kube.target.imageregistry.*.logintype
-      - move2kube.target.imageregistry.*.password
-  - name: sshkeys
-    enabled: true
-    questions:
-      - move2kube.repo.keys.*.key
-      - move2kube.repo.keys.pub.domain.*.pubkey
-      - move2kube.repo.keys.pub.load
-      - move2kube.repo.keys.load
-      - move2kube.repo.keys.paths
-      - move2kube.repo.keys.priv.*.password
-  - name: storage
-    enabled: true
-    questions:
-      - move2kube.storage.type.*.options
-  - name: sourceanalyzer
-    enabled: true
-    questions:
-      - move2kube.services.*.enable
-      - move2kube.services.*.statefulset
-      - move2kube.services.*.containerizationoption
-      - move2kube.services.*.childProjects.*.publishprofile
-      - move2kube.services.*.apacheconfig
-      - move2kube.services.*.pythonmainfile
-      - move2kube.services.*.pythonstartingfile
-      - move2kube.services.*.dockerfileType
-      - move2kube.services.*.childModules.*.enable
-      - move2kube.services.*.childProjects.*.enable
-      - move2kube.services.*.childModules.*.springBootProfiles
-      - move2kube.services.*.mavenProfiles
-  - name: cluster
-    enabled: true
-    questions:
-      - move2kube.target.*.clustertype
-      - move2kube.minreplicas
-  - name: network
-    enabled: true
-    questions:
-      - move2kube.services.*.*.servicetype
-      - move2kube.services.*.*.urlpath
-      - move2kube.services.*.ports
-      - move2kube.services.*.port
-      - move2kube.target.*.ingress.ingressclassname
-      - move2kube.target.*.ingress.host
-      - move2kube.target.*.ingress.tls
-  - name: git
-    enabled: true
-    questions:
-      - move2kube.vcs.git.name
-      - move2kube.vcs.git.username
-      - move2kube.vcs.git.email
-      - move2kube.vcs.git.pass
-  - name: cicd
-    enabled: true
-    questions:
-      - move2kube.target.cicd.tekton.gitreposshsecret
-      - move2kube.target.cicd.tekton.gitrepobasicauthsecret
-      - move2kube.target.cicd.tekton.registrypushsecret
-      - move2kube.transformers.kubernetes.argocd.namespace
-  - name: transformers
-    enabled: true
-    questions:
-      - move2kube.transformerselector
-      - move2kube.spawncontainers
-      - move2kube.transformers.types
+apiVersion: move2kube.konveyor.io/v1alpha1
+kind: QAMappings
+metadata:
+  name: DefaultQA-Mappings
+spec:
+  categories:
+    - name: imageregistry
+      # or use the flags --enable imageregistry/--disable imageregistry
+      enabled: true
+      questions:
+        - move2kube.target.imageregistry.url
+        - move2kube.target.imageregistry.namespace
+        - move2kube.target.imageregistry.*.logintype
+        - move2kube.target.imageregistry.*.password
+    - name: sshkeys
+      enabled: true
+      questions:
+        - move2kube.repo.keys.*.key
+        - move2kube.repo.keys.pub.domain.*.pubkey
+        - move2kube.repo.keys.pub.load
+        - move2kube.repo.keys.load
+        - move2kube.repo.keys.paths
+        - move2kube.repo.keys.priv.*.password
+    - name: storage
+      enabled: true
+      questions:
+        - move2kube.storage.type.*.options
+    - name: sourceanalyzer
+      enabled: true
+      questions:
+        - move2kube.services.*.enable
+        - move2kube.services.*.statefulset
+        - move2kube.services.*.containerizationoption
+        - move2kube.services.*.childProjects.*.publishprofile
+        - move2kube.services.*.apacheconfig
+        - move2kube.services.*.pythonmainfile
+        - move2kube.services.*.pythonstartingfile
+        - move2kube.services.*.dockerfileType
+        - move2kube.services.*.childModules.*.enable
+        - move2kube.services.*.childProjects.*.enable
+        - move2kube.services.*.childModules.*.springBootProfiles
+        - move2kube.services.*.mavenProfiles
+    - name: cluster
+      enabled: true
+      questions:
+        - move2kube.target.*.clustertype
+        - move2kube.minreplicas
+    - name: network
+      enabled: true
+      questions:
+        - move2kube.services.*.*.servicetype
+        - move2kube.services.*.*.urlpath
+        - move2kube.target.*.ingress.ingressclassname
+        - move2kube.target.*.ingress.host
+        - move2kube.target.*.ingress.tls
+    - name: ports
+      enabled: true
+      questions:
+        - move2kube.services.*.ports
+        - move2kube.services.*.port
+    - name: git
+      enabled: true
+      questions:
+        - move2kube.vcs.git.name
+        - move2kube.vcs.git.username
+        - move2kube.vcs.git.email
+        - move2kube.vcs.git.pass
+    - name: cicd
+      enabled: true
+      questions:
+        - move2kube.target.cicd.tekton.gitreposshsecret
+        - move2kube.target.cicd.tekton.gitrepobasicauthsecret
+        - move2kube.target.cicd.tekton.registrypushsecret
+        - move2kube.transformers.kubernetes.argocd.namespace
+    - name: transformers
+      enabled: true
+      questions:
+        - move2kube.transformerselector
+        - move2kube.spawncontainers
+        - move2kube.transformers.types

cmd/flags.go

@@ -77,4 +77,8 @@ type qaflags struct {
 	preSets []string
 	// persistPasswords sets whether to persist the password or not
 	persistPasswords bool
+	// qaEnabledCategories contains list of categories to be enabled
+	qaEnabledCategories []string
+	// qaDisabledCategories contains list of categories to be disabled
+	qaDisabledCategories []string
 }

cmd/transform.go

@@ -23,17 +23,14 @@ import (
 	"path/filepath"
 	"runtime/pprof"
 
-	"github.com/konveyor/move2kube/assets"
 	"github.com/konveyor/move2kube/common"
 	"github.com/konveyor/move2kube/common/download"
 	"github.com/konveyor/move2kube/common/vcs"
 	"github.com/konveyor/move2kube/lib"
 	"github.com/konveyor/move2kube/types/plan"
-	"github.com/konveyor/move2kube/types/qaengine"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"gopkg.in/yaml.v3"
 )
 
 type transformFlags struct {
@@ -59,10 +56,8 @@ type transformFlags struct {
 	// maxIterations is the maximum number of iterations to allow before aborting with an error
 	maxIterations int
 	// CustomizationsPaths contains the path to the customizations directory
-	customizationsPath   string
-	transformerSelector  string
-	qaEnabledCategories  []string
-	qaDisabledCategories []string
+	customizationsPath  string
+	transformerSelector string
 }
 
 func transformHandler(cmd *cobra.Command, flags transformFlags) {
@@ -133,46 +128,9 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) {
 		}
 	}
 
-	// qa-disable and qa=enable are mutually exclusive
-	if len(flags.qaEnabledCategories) > 0 && len(flags.qaDisabledCategories) > 0 {
-		logrus.Fatalf("--qa-enable and --qa-disable cannot be used together.\n")
-	}
-
-	// Read the QA categories from the QA mapping file
-	var qaMapping qaengine.QAMapping
-	qaMappingFilepath := filepath.Join("built-in/qa", "qamappings.yaml")
-	file, err := assets.AssetsDir.ReadFile(qaMappingFilepath)
-	if err != nil {
-		logrus.Fatalf("failed to read qa-mapping file at %s. Error: %q\n", qaMappingFilepath, err)
-	}
-
-	if err := yaml.Unmarshal(file, &qaMapping); err != nil {
-		logrus.Fatalf("failed to decode qa-mapping file. Error: %q\n", err)
-	}
-
-	for _, mapping := range qaMapping.Categories {
-		common.QACategoryMap[mapping.Name] = mapping.Questions
-	}
-	common.QACategoryMap["default"] = []string{}
-	common.QACategoryMap["external"] = []string{}
-
 	// Global settings
 	common.IgnoreEnvironment = flags.ignoreEnv
 	common.DisableLocalExecution = flags.disableLocalExecution
-	// if --qa-enable is passed, all categories are disabled by default. Otherwise, only categories passed to --qa-disable
-	// are disabled
-	if len(flags.qaEnabledCategories) > 0 {
-		for k := range common.QACategoryMap {
-			if !common.IsStringPresent(flags.qaEnabledCategories, k) {
-				common.DisabledCategories = append(common.DisabledCategories, k)
-			}
-		}
-	} else {
-		for _, cat := range flags.qaDisabledCategories {
-			common.DisabledCategories = append(common.DisabledCategories, cat)
-		}
-	}
-
 	// Parameter cleaning and curate plan
 	transformationPlan := plan.Plan{}
 	preExistingPlan := false
@@ -182,7 +140,7 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) {
 		_, err = os.Stat(flags.planfile)
 	}
 	if err != nil {
-		logrus.Debugf("No plan file found.")
+		logrus.Infof("No plan file found.")
 		if cmd.Flags().Changed(planFlag) {
 			logrus.Fatalf("Error while accessing plan file at path %s Error: %q", flags.planfile, err)
 		}
@@ -201,6 +159,11 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) {
 				logrus.Fatalf("Failed to create the output directory at path %s Error: %q", flags.outpath, err)
 			}
 		}
+		if flags.customizationsPath != "" {
+			if err := lib.CheckAndCopyCustomizations(flags.customizationsPath); err != nil {
+				logrus.Fatalf("Failed to check and copy the customizations. Error: %q", err)
+			}
+		}
 		startQA(flags.qaflags)
 		logrus.Debugf("Creating a new plan.")
 		transformationPlan, err = lib.CreatePlan(ctx, flags.srcpath, flags.outpath, flags.customizationsPath, flags.transformerSelector, flags.name)
@@ -240,7 +203,9 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) {
 		if transformationPlan.Spec.SourceDir != "" {
 			checkSourcePath(transformationPlan.Spec.SourceDir)
 		}
-		lib.CheckAndCopyCustomizations(transformationPlan.Spec.CustomizationsDir)
+		if err := lib.CheckAndCopyCustomizations(transformationPlan.Spec.CustomizationsDir); err != nil {
+			logrus.Fatalf("Failed to check and copy the customizations. Error: %q", err)
+		}
 		if !isRemoteOutPath {
 			flags.outpath = filepath.Join(flags.outpath, transformationPlan.Name)
 			checkOutputPath(flags.outpath, flags.overwrite)

cmd/utils.go

@@ -18,16 +18,20 @@ package cmd
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/gorilla/mux"
+	"github.com/konveyor/move2kube/assets"
 	"github.com/konveyor/move2kube/common"
 	"github.com/konveyor/move2kube/qaengine"
+	qaenginetypes "github.com/konveyor/move2kube/types/qaengine"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cast"
+	"gopkg.in/yaml.v3"
 )
 
 // checkSourcePath checks if the source path is an existing directory.
@@ -82,7 +86,82 @@ Exiting.`, outpath, overwriteFlag, outputFlag)
 	logrus.Infof("Output directory '%s' exists. The contents might get overwritten.", outpath)
 }
 
+func getCustomMappingFilePath() (qaenginetypes.QAMappings, error) {
+	// Read the QA categories from the QA mapping file
+	var qaMapping qaenginetypes.QAMappings
+	customizationsDir := filepath.Join(common.AssetsPath, "custom")
+	_, err := os.Stat(customizationsDir)
+	if err == nil {
+		yamlPaths, err := common.GetFilesByExt(customizationsDir, []string{".yml", ".yaml"})
+		if err != nil {
+			return qaMapping, fmt.Errorf("failed to look for yaml files in the directory '%s' . Error: %w", customizationsDir, err)
+		}
+		for _, yamlPath := range yamlPaths {
+			if err := common.ReadMove2KubeYaml(yamlPath, &qaMapping); err != nil {
+				logrus.Debugf("failed to read the mappings file metadata from the yaml file at path '%s' . Error: %q", yamlPath, err)
+				continue
+			}
+			if qaMapping.Kind != qaenginetypes.QAMappingsKind {
+				logrus.Debugf(
+					"the file at path '%s' is not a valid cluster metadata. Expected kind: '%s' Actual kind: '%s'",
+					yamlPath, qaenginetypes.QAMappingsKind, qaMapping.Kind,
+				)
+				continue
+			}
+			logrus.Infof("Found custom QA mappings file '%s' at path '%s'", qaMapping.ObjectMeta.Name, yamlPath)
+			return qaMapping, nil
+		}
+	}
+	logrus.Infof("Using the default QA mappings file")
+	qaMappingFilepath := filepath.Join("built-in/qa", "qamappings.yaml")
+	file, err := assets.AssetsDir.ReadFile(qaMappingFilepath)
+	if err != nil {
+		return qaMapping, fmt.Errorf("failed to read the mappings file metadata from the yaml file at path '%s' . Error: %w", qaMappingFilepath, err)
+	}
+	if err := yaml.Unmarshal(file, &qaMapping); err != nil {
+		return qaMapping, fmt.Errorf("failed to decode qa-mapping file. Error: %w", err)
+	}
+	return qaMapping, nil
+}
+
+func initDisabledCategories(flags qaflags) {
+	// qa-disable and qa=enable are mutually exclusive
+	if len(flags.qaEnabledCategories) > 0 && len(flags.qaDisabledCategories) > 0 {
+		logrus.Fatalf("--qa-enable and --qa-disable cannot be used together.\n")
+	}
+	// Read the QA categories from the QA mapping file
+	qaMapping, err := getCustomMappingFilePath()
+	if err != nil {
+		logrus.Fatalf("failed to read the QAMappings file. Error: %q", err)
+	}
+	for _, category := range qaMapping.Spec.Categories {
+		common.QACategoryMap[category.Name] = category.Questions
+	}
+	common.QACategoryMap["default"] = []string{}
+	common.QACategoryMap["external"] = []string{}
+	// if --qa-enable is passed, all categories are disabled by default. Otherwise, only categories passed to --qa-disable
+	// are disabled
+	for _, category := range qaMapping.Spec.Categories {
+		if !category.Enabled {
+			common.DisabledCategories = append(common.DisabledCategories, category.Name)
+		}
+	}
+	if len(flags.qaEnabledCategories) > 0 {
+		for k := range common.QACategoryMap {
+			if !common.IsStringPresent(flags.qaEnabledCategories, k) {
+				common.DisabledCategories = append(common.DisabledCategories, k)
+			}
+		}
+	} else {
+		common.DisabledCategories = append(common.DisabledCategories, flags.qaDisabledCategories...)
+	}
+	if len(common.DisabledCategories) > 0 {
+		logrus.Infof("Disabling the questions in the following categories: %v", common.DisabledCategories)
+	}
+}
+
 func startQA(flags qaflags) {
+	initDisabledCategories(flags)
 	qaengine.StartEngine(flags.qaskip, flags.qaport, flags.qadisablecli)
 	if flags.configOut == "" {
 		qaengine.SetupConfigFile("", flags.setconfigs, flags.configs, flags.preSets, flags.persistPasswords)

lib/planner.go

@@ -54,11 +54,6 @@ func CreatePlan(ctx context.Context, inputPath, outputPath string, customization
 	if remoteOutputFSPath != "" {
 		outputFSPath = remoteOutputFSPath
 	}
-	if customizationsPath != "" {
-		if err := CheckAndCopyCustomizations(customizationsPath); err != nil {
-			return plan, fmt.Errorf("failed to check and copy the customizations. Error: %w", err)
-		}
-	}
 	transformerSelectorObj, err := metav1.ParseToLabelSelector(transformerSelector)
 	if err != nil {
 		return plan, fmt.Errorf("failed to parse the string '%s' as a transformer selector. Error: %w", transformerSelector, err)

lib/utils.go

@@ -67,7 +67,7 @@ func CheckAndCopyCustomizations(customizationsPath string) error {
 	// check if the customization path has files other than YAMLs
 	yamls, err := common.GetYamlsWithTypeMeta(customizationsFSPath, TransformerTypeMeta)
 	if err == nil && len(yamls) == 0 {
-		logrus.Warnf("no manifests for external transformers found in %s, the transformers won't be loaded.", customizationsFSPath)
+		logrus.Infof("No manifests for external transformers found in the customizations directory '%s'.", customizationsFSPath)
 	}
 	if err = CopyCustomizationsAssetsData(customizationsFSPath); err != nil {
 		return fmt.Errorf("failed to copy the customizations data from the directory '%s' . Error: %w", customizationsFSPath, err)