[config] support append mdsv2 config

Author: jackblack369

.vscode/launch.json

@@ -186,7 +186,7 @@
             "showLog": true,
         },
         {
-            "name": "dingoadm upgrade -f",
+            "name": "dingoadm upgrade -f --local",
             "type": "go",
             "request": "launch",
             "mode": "auto",
@@ -198,6 +198,7 @@
             "args": [
                 "upgrade",
                 "-f",
+                "--local"
             ],
             "showLog": true,
         },

internal/common/common.go

@@ -139,6 +139,10 @@ const (
 	CLIENT_CONFIG_DELIMITER   = "="
 	TOOLS_V2_CONFIG_DELIMITER = ": "
 
+	// prefix
+	STORE_GFLAGS_PREFIX = "-"
+	MDSV2_CONFIG_PREFIX = "--"
+
 	// mdsv2
 	KEY_SKIP_MDSV2_CLI = "SKIP_MDSV2_CLI"
 

internal/task/step/file.go

@@ -66,11 +66,14 @@ type (
 
 	Mutate func(string, string, string) (string, error)
 
+	DynamicAppend func(string, string) (string, error)
+
 	Filter struct {
-		KVFieldSplit string
-		Mutate       Mutate
-		Input        *string
-		Output       *string
+		KVFieldSplit  string
+		Mutate        Mutate
+		SerivceConfig map[string]string
+		Input         *string
+		Output        *string
 	}
 
 	SyncFile struct {
@@ -80,6 +83,7 @@ type (
 		ContainerDestPath string
 		KVFieldSplit      string
 		Mutate            func(string, string, string) (string, error)
+		SerivceConfig     map[string]string
 		module.ExecOptions
 	}
 
@@ -232,6 +236,14 @@ func (s *Filter) kvSplit(line string, key, value *string) error {
 func (s *Filter) Execute(ctx *context.Context) error {
 	var key, value string
 	output := []string{}
+
+	topologySerivceConfigKeys := []string{}
+	for k := range s.SerivceConfig {
+		topologySerivceConfigKeys = append(topologySerivceConfigKeys, k)
+	}
+
+	originServiceConfigKeys := []string{}
+
 	scanner := bufio.NewScanner(strings.NewReader(string(*s.Input)))
 	for scanner.Scan() {
 		in := scanner.Text()
@@ -242,13 +254,36 @@ func (s *Filter) Execute(ctx *context.Context) error {
 			}
 		}
 
+		originServiceConfigKeys = append(originServiceConfigKeys, key)
+
 		out, err := s.Mutate(in, key, value)
 		if err != nil {
 			return err
 		}
 		output = append(output, out)
 	}
 
+	// extract the key from topologySerivceConfigKeys that are not in originServiceConfigKeys
+	extraServiceConfigKeys := utils.Filter(topologySerivceConfigKeys, func(k string) bool {
+		if !strings.HasPrefix(k, "mds_") {
+			return false
+		}
+		mdsv2_key := fmt.Sprintf("%s%s", comm.MDSV2_CONFIG_PREFIX, k)
+		return !utils.Contains(originServiceConfigKeys, mdsv2_key)
+	})
+
+	if len(extraServiceConfigKeys) > 0 {
+		output = append(output, "# dynamic config from topology")
+		for _, k := range extraServiceConfigKeys {
+			v := s.SerivceConfig[k]
+			if len(v) > 0 {
+				// dingo-mdsv2.template.conf
+				out := fmt.Sprintf("%s%s%s%s", comm.MDSV2_CONFIG_PREFIX, k, s.KVFieldSplit, v)
+				output = append(output, out)
+			}
+		}
+	}
+
 	*s.Output = strings.Join(output, "\n")
 	return nil
 }
@@ -263,10 +298,11 @@ func (s *SyncFile) Execute(ctx *context.Context) error {
 		ExecOptions:      s.ExecOptions,
 	})
 	steps = append(steps, &Filter{
-		KVFieldSplit: s.KVFieldSplit,
-		Mutate:       s.Mutate,
-		Input:        &input,
-		Output:       &output,
+		KVFieldSplit:  s.KVFieldSplit,
+		Mutate:        s.Mutate,
+		SerivceConfig: s.SerivceConfig,
+		Input:         &input,
+		Output:        &output,
 	})
 	steps = append(steps, &InstallFile{
 		ContainerId:       s.ContainerDestId,

internal/task/task/common/sync_config.go

@@ -61,13 +61,13 @@ func NewMutate(dc *topology.DeployConfig, delimiter string, forceRender bool) st
 		muteKey := strings.TrimSpace(key)
 		if dc.GetRole() == topology.ROLE_COORDINATOR || dc.GetRole() == topology.ROLE_STORE {
 			// key is like -xxx , replace  '-' to 'gflags.'
-			if strings.HasPrefix(key, "-") {
-				muteKey = fmt.Sprintf("gflags.%s", strings.TrimPrefix(key, "-"))
+			if strings.HasPrefix(key, comm.STORE_GFLAGS_PREFIX) {
+				muteKey = fmt.Sprintf("gflags.%s", strings.TrimPrefix(key, comm.STORE_GFLAGS_PREFIX))
 			}
 		} else if dc.GetRole() == topology.ROLE_MDS_V2 {
 			// key is like --xxx , trim '--'
-			if strings.HasPrefix(key, "--") {
-				muteKey = strings.TrimPrefix(key, "--")
+			if strings.HasPrefix(key, comm.MDSV2_CONFIG_PREFIX) {
+				muteKey = strings.TrimPrefix(key, comm.MDSV2_CONFIG_PREFIX)
 			}
 		}
 
@@ -151,13 +151,14 @@ func NewSyncConfigTask(dingoadm *cli.DingoAdm, dc *topology.DeployConfig) (*task
 
 	if dc.GetKind() == topology.KIND_DINGOFS {
 		for _, conf := range layout.ServiceConfFiles {
-			t.AddStep(&step.SyncFile{ // sync service config
+			t.AddStep(&step.SyncFile{ // sync service config, e.g. dingo-mdsv2.template.conf
 				ContainerSrcId:    &containerId,
 				ContainerSrcPath:  conf.SourcePath,
 				ContainerDestId:   &containerId,
 				ContainerDestPath: conf.TargetPath,
 				KVFieldSplit:      delimiter,
 				Mutate:            NewMutate(dc, delimiter, conf.Name == "nginx.conf"),
+				SerivceConfig:     dc.GetServiceConfig(),
 				ExecOptions:       dingoadm.ExecOptions(),
 			})
 		}

internal/utils/common.go

@@ -319,9 +319,19 @@ func EncryptFile(srcfile, dstfile, secret string) error {
 
 func Contains(list []string, str string) bool {
 	for _, v := range list {
-		if v == str {
+		if strings.TrimSpace(v) == strings.TrimSpace(str) {
 			return true
 		}
 	}
 	return false
 }
+
+func Filter(list []string, filterFunc func(string) bool) []string {
+	var result []string
+	for _, item := range list {
+		if filterFunc(item) {
+			result = append(result, item)
+		}
+	}
+	return result
+}