feat: support runtime configurations in workspace (#1211)

liu-hm19 · web-flow · commit 6b9c2cd6c7aa · 2024-07-10T19:15:49.000+08:00
diff --git a/pkg/apis/api.kusion.io/v1/types.go b/pkg/apis/api.kusion.io/v1/types.go
@@ -92,7 +92,7 @@ type Workspace struct {
 	// SecretStore represents a secure external location for storing secrets.
 	SecretStore *SecretStore `yaml:"secretStore,omitempty" json:"secretStore,omitempty"`
 
-	// Context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc.
+	// Context contains workspace-level configurations, such as runtimes, topologies, and metadata, etc.
 	Context GenericConfig `yaml:"context,omitempty" json:"context,omitempty"`
 }
 
@@ -475,6 +475,9 @@ const (
 	EnvAwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
 	EnvAwsDefaultRegion   = "AWS_DEFAULT_REGION"
 	EnvAwsRegion          = "AWS_REGION"
+	EnvAlicloudAccessKey  = "ALICLOUD_ACCESS_KEY"
+	EnvAlicloudSecretKey  = "ALICLOUD_SECRET_KEY"
+	EnvAlicloudRegion     = "ALICLOUD_REGION"
 )
 
 // BackendConfigs contains the configuration of multiple backends and the current backend.
@@ -861,6 +864,8 @@ type Spec struct {
 	Resources Resources `yaml:"resources" json:"resources"`
 	// SecretSore represents a external secret store location for storing secrets.
 	SecretStore *SecretStore `yaml:"secretStore" json:"secretStore"`
+	// Context contains workspace-level configurations, such as runtimes, topologies, and metadata, etc.
+	Context GenericConfig `yaml:"context" json:"context"`
 }
 
 // State is a record of an operation's result. It is a mapping between resources in KCL and the actual
diff --git a/pkg/cmd/apply/apply.go b/pkg/cmd/apply/apply.go
@@ -713,7 +713,7 @@ func Watch(
 			watchErrCh <- *err
 		}()
 		// Init the runtimes according to the resource types.
-		runtimes, s := runtimeinit.Runtimes(toBeWatched)
+		runtimes, s := runtimeinit.Runtimes(*rel.Spec)
 		if v1.IsErr(s) {
 			panic(fmt.Errorf("failed to init runtimes: %s", s.String()))
 		}
diff --git a/pkg/engine/operation/apply.go b/pkg/engine/operation/apply.go
@@ -71,9 +71,7 @@ func (ao *ApplyOperation) Apply(req *ApplyRequest) (rsp *ApplyResponse, s v1.Sta
 		stateResourceIndex[k] = v
 	}
 
-	resources := req.Release.Spec.Resources
-	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)
 	if v1.IsErr(s) {
 		return nil, s
 	}
diff --git a/pkg/engine/operation/apply_test.go b/pkg/engine/operation/apply_test.go
@@ -148,7 +148,7 @@ func TestApplyOperation_Apply(t *testing.T) {
 				return nil
 			}).Build()
 			mockey.Mock(runtimeinit.Runtimes).To(func(
-				resources apiv1.Resources,
+				spec apiv1.Spec,
 			) (map[apiv1.Type]runtime.Runtime, v1.Status) {
 				return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: &kubernetes.KubernetesRuntime{}}, nil
 			}).Build()
diff --git a/pkg/engine/operation/destory.go b/pkg/engine/operation/destory.go
@@ -48,7 +48,7 @@ func (do *DestroyOperation) Destroy(req *DestroyRequest) (rsp *DestroyResponse,
 
 	// only destroy resources we have recorded
 	resources := priorState.Resources
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)
 	if v1.IsErr(s) {
 		return nil, s
 	}
diff --git a/pkg/engine/operation/graph/resource_node.go b/pkg/engine/operation/graph/resource_node.go
@@ -106,7 +106,7 @@ func (rn *ResourceNode) replaceK8sSecretRefs(o *models.Operation) v1.Status {
 			continue
 		}
 
-		externalSecretRef, err := parseExternalSecretDataRef(ref)
+		externalSecretRef, err := ParseExternalSecretDataRef(ref)
 		if err != nil {
 			return v1.NewErrorStatus(err)
 		}
@@ -513,8 +513,8 @@ func ReplaceRef(
 	return result, v, nil
 }
 
-// parseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.
-func parseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {
+// ParseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.
+func ParseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {
 	uri, err := url.Parse(dataRefStr)
 	if err != nil {
 		return nil, err
diff --git a/pkg/engine/operation/graph/resource_node_test.go b/pkg/engine/operation/graph/resource_node_test.go
@@ -303,7 +303,7 @@ func TestParseExternalSecretDataRef(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseExternalSecretDataRef(tt.dataRefStr)
+			got, err := ParseExternalSecretDataRef(tt.dataRefStr)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("parseExternalSecretDataRef() error = %v, wantErr %v", err, tt.wantErr)
 				return
diff --git a/pkg/engine/operation/preview.go b/pkg/engine/operation/preview.go
@@ -58,9 +58,7 @@ func (po *PreviewOperation) Preview(req *PreviewRequest) (rsp *PreviewResponse,
 	priorState := req.State
 
 	// Kusion is a multi-runtime system. We initialize runtimes dynamically by resource types
-	resources := req.Spec.Resources
-	resources = append(resources, priorState.Resources...)
-	runtimesMap, s := runtimeinit.Runtimes(resources)
+	runtimesMap, s := runtimeinit.Runtimes(*req.Spec)
 	if v1.IsErr(s) {
 		return nil, s
 	}
@@ -75,7 +73,7 @@ func (po *PreviewOperation) Preview(req *PreviewRequest) (rsp *PreviewResponse,
 		priorStateResourceIndex = priorState.Resources.Index()
 		ag, s = newApplyGraph(req.Spec, priorState)
 	case models.DestroyPreview:
-		resources = req.Spec.Resources
+		resources := req.Spec.Resources
 		priorStateResourceIndex = resources.Index()
 		ag, s = newDestroyGraph(resources)
 	}
diff --git a/pkg/engine/operation/preview_test.go b/pkg/engine/operation/preview_test.go
@@ -253,7 +253,7 @@ func TestPreviewOperation_Preview(t *testing.T) {
 			}
 
 			mockey.Mock(runtimeinit.Runtimes).To(func(
-				resources apiv1.Resources,
+				spec apiv1.Spec,
 			) (map[apiv1.Type]runtime.Runtime, v1.Status) {
 				return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil
 			}).Build()
diff --git a/pkg/engine/operation/watch.go b/pkg/engine/operation/watch.go
@@ -43,7 +43,7 @@ func (wo *WatchOperation) Watch(req *WatchRequest) error {
 
 	// init runtimes
 	resources := req.Spec.Resources
-	runtimes, s := runtimeinit.Runtimes(resources)
+	runtimes, s := runtimeinit.Runtimes(*req.Spec)
 	if v1.IsErr(s) {
 		return errors.New(s.Message())
 	}
diff --git a/pkg/engine/operation/watch_test.go b/pkg/engine/operation/watch_test.go
@@ -94,7 +94,7 @@ func TestWatchOperation_Watch(t *testing.T) {
 			},
 		}
 		mockey.Mock(runtimeinit.Runtimes).To(func(
-			resources apiv1.Resources,
+			spec apiv1.Spec,
 		) (map[apiv1.Type]runtime.Runtime, v1.Status) {
 			return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: fooRuntime}, nil
 		}).Build()
diff --git a/pkg/engine/release/util.go b/pkg/engine/release/util.go
@@ -113,7 +113,19 @@ func CreateDestroyRelease(storage Storage, project, stack, workspace string) (*v
 
 	resources := make([]v1.Resource, len(lastRelease.State.Resources))
 	copy(resources, lastRelease.State.Resources)
-	spec := &v1.Spec{Resources: resources}
+
+	secretStore := &v1.SecretStore{}
+	if lastRelease.Spec != nil && lastRelease.Spec.SecretStore != nil {
+		secretStore = lastRelease.Spec.SecretStore
+	}
+
+	specContext := v1.GenericConfig{}
+	if lastRelease.Spec != nil && lastRelease.Spec.Context != nil {
+		specContext = lastRelease.Spec.Context
+	}
+
+	spec := &v1.Spec{Resources: resources, SecretStore: secretStore, Context: specContext}
+
 	// if no resource managed, set phase to Succeeded directly.
 	phase := v1.ReleasePhasePreviewing
 	if len(resources) == 0 {
diff --git a/pkg/engine/runtime/init/init.go b/pkg/engine/runtime/init/init.go
@@ -1,26 +1,45 @@
 package init
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"reflect"
+	"strings"
 
 	apiv1 "kusionstack.io/kusion/pkg/apis/api.kusion.io/v1"
 	v1 "kusionstack.io/kusion/pkg/apis/status/v1"
+	"kusionstack.io/kusion/pkg/engine/operation/graph"
 	"kusionstack.io/kusion/pkg/engine/runtime"
 	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes"
 	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform"
+	"kusionstack.io/kusion/pkg/secrets"
+	"kusionstack.io/kusion/pkg/workspace"
 )
 
 var SupportRuntimes = map[apiv1.Type]InitFn{
 	runtime.Kubernetes: kubernetes.NewKubernetesRuntime,
 	runtime.Terraform:  terraform.NewTerraformRuntime,
 }
 
+var contextKeys = []string{
+	kubeops.KubeConfigContentKey,
+	apiv1.EnvAwsAccessKeyID,
+	apiv1.EnvAwsSecretAccessKey,
+	apiv1.EnvAlicloudAccessKey,
+	apiv1.EnvAlicloudSecretKey,
+}
+
 // InitFn runtime init func
-type InitFn func(resource *apiv1.Resource) (runtime.Runtime, error)
+type InitFn func(spec apiv1.Spec) (runtime.Runtime, error)
 
-func Runtimes(resources apiv1.Resources) (map[apiv1.Type]runtime.Runtime, v1.Status) {
+func Runtimes(spec apiv1.Spec) (map[apiv1.Type]runtime.Runtime, v1.Status) {
+	// Parse the secret ref in the Context of Spec.
+	if err := parseContextSecretRef(&spec); err != nil {
+		return nil, v1.NewErrorStatus(err)
+	}
+	resources := spec.Resources
 	runtimesMap := map[apiv1.Type]runtime.Runtime{}
 	if resources == nil {
 		return runtimesMap, nil
@@ -32,7 +51,7 @@ func Runtimes(resources apiv1.Resources) (map[apiv1.Type]runtime.Runtime, v1.Sta
 	for _, resource := range resources {
 		rt := resource.Type
 		if runtimesMap[rt] == nil {
-			r, err := SupportRuntimes[rt](&resource)
+			r, err := SupportRuntimes[rt](spec)
 			if err != nil {
 				return nil, v1.NewErrorStatus(fmt.Errorf("init %s runtime failed. %w", rt, err))
 			}
@@ -65,3 +84,51 @@ func validResources(resources apiv1.Resources) v1.Status {
 	}
 	return nil
 }
+
+// parseContextSecretRef parses the external secret ref of the credentials
+// in the Context of Spec.
+func parseContextSecretRef(spec *apiv1.Spec) error {
+	// Copy the Context of Spec.
+	parsedContext := apiv1.GenericConfig{}
+	for k, v := range spec.Context {
+		parsedContext[k] = v
+	}
+
+	// Retrieve the context with the specified keys from spec and parse the external secret ref.
+	for _, key := range contextKeys {
+		contextStr, err := workspace.GetStringFromGenericConfig(spec.Context, key)
+		if err != nil {
+			return err
+		}
+
+		if contextStr != "" {
+			// Replace the secret store ref.
+			if strings.HasPrefix(contextStr, graph.SecretRefPrefix) {
+				externalSecretRef, err := graph.ParseExternalSecretDataRef(contextStr)
+				if err != nil {
+					return err
+				}
+
+				provider, exist := secrets.GetProvider(spec.SecretStore.Provider)
+				if !exist {
+					return errors.New("no matched secret store found, please check workspace yaml")
+				}
+				secretStore, err := provider.NewSecretStore(spec.SecretStore)
+				if err != nil {
+					return err
+				}
+				secretData, err := secretStore.GetSecret(context.Background(), *externalSecretRef)
+				if err != nil {
+					return err
+				}
+
+				parsedContext[key] = string(secretData)
+			}
+		}
+	}
+
+	// Reset the Context with the parsed values.
+	spec.Context = parsedContext
+
+	return nil
+}
diff --git a/pkg/engine/runtime/kubernetes/kubeops/config.go b/pkg/engine/runtime/kubernetes/kubeops/config.go
@@ -15,6 +15,11 @@ const (
 	RecommendedKubeConfigFileName = "config"
 )
 
+const (
+	KubeConfigPathKey    = "kubeconfig_path"
+	KubeConfigContentKey = "kubeconfig_content"
+)
+
 var (
 	RecommendedConfigDir      = filepath.Join(homedir.HomeDir(), RecommendedHomeDir)
 	RecommendedKubeConfigFile = filepath.Join(RecommendedConfigDir, RecommendedKubeConfigFileName)
diff --git a/pkg/engine/runtime/kubernetes/kubernetes_runtime.go b/pkg/engine/runtime/kubernetes/kubernetes_runtime.go
@@ -35,6 +35,7 @@ import (
 	"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
 	"kusionstack.io/kusion/pkg/log"
 	jsonutil "kusionstack.io/kusion/pkg/util/json"
+	"kusionstack.io/kusion/pkg/workspace"
 )
 
 var _ runtime.Runtime = (*KubernetesRuntime)(nil)
@@ -45,8 +46,8 @@ type KubernetesRuntime struct {
 }
 
 // NewKubernetesRuntime create a new KubernetesRuntime
-func NewKubernetesRuntime(resource *apiv1.Resource) (runtime.Runtime, error) {
-	client, mapper, err := getKubernetesClient(resource)
+func NewKubernetesRuntime(spec apiv1.Spec) (runtime.Runtime, error) {
+	client, mapper, err := getKubernetesClient(spec)
 	if err != nil {
 		return nil, err
 	}
@@ -394,11 +395,50 @@ func (k *KubernetesRuntime) Watch(ctx context.Context, request *runtime.WatchReq
 }
 
 // getKubernetesClient get kubernetes client
-func getKubernetesClient(resource *apiv1.Resource) (dynamic.Interface, meta.RESTMapper, error) {
+func getKubernetesClient(spec apiv1.Spec) (dynamic.Interface, meta.RESTMapper, error) {
 	// build config
-	cfg, err := clientcmd.BuildConfigFromFlags("", kubeops.GetKubeConfig(resource))
-	if err != nil {
-		return nil, nil, err
+	var err error
+	var cfg *rest.Config
+
+	if len(spec.Context) != 0 {
+		kubeConfigPath, err := workspace.GetStringFromGenericConfig(spec.Context, kubeops.KubeConfigPathKey)
+		if err != nil {
+			return nil, nil, err
+		}
+		kubeConfigContent, err := workspace.GetStringFromGenericConfig(spec.Context, kubeops.KubeConfigContentKey)
+		if err != nil {
+			return nil, nil, err
+		}
+		if kubeConfigContent != "" {
+			clientCfg, err := clientcmd.NewClientConfigFromBytes([]byte(kubeConfigContent))
+			if err != nil {
+				return nil, nil, err
+			}
+
+			cfg, err = clientCfg.ClientConfig()
+			if err != nil {
+				return nil, nil, err
+			}
+		} else if kubeConfigPath != "" {
+			cfg, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+	} else {
+		var kubeConfigFromRes string
+		for _, res := range spec.Resources {
+			if res.Type == apiv1.Kubernetes {
+				kubeConfigFromRes = kubeops.GetKubeConfig(&res)
+			}
+			if kubeConfigFromRes != "" {
+				break
+			}
+		}
+		cfg, err = clientcmd.BuildConfigFromFlags("", kubeConfigFromRes)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
 	// DynamicRESTMapper can discover resource types at runtime dynamically
diff --git a/pkg/engine/runtime/terraform/terraform_runtime.go b/pkg/engine/runtime/terraform/terraform_runtime.go
diff --git a/pkg/engine/runtime/terraform/tfops/workspace.go b/pkg/engine/runtime/terraform/tfops/workspace.go
diff --git a/pkg/modules/generators/app_configurations_generator.go b/pkg/modules/generators/app_configurations_generator.go

Original file line number	Diff line number	Diff line change
`@@ -713,7 +713,7 @@ func Watch(`
`713`	`713`	`watchErrCh <- *err`
`714`	`714`	`}()`
`715`	`715`	`// Init the runtimes according to the resource types.`
`716`		`- runtimes, s := runtimeinit.Runtimes(toBeWatched)`
	`716`	`+ runtimes, s := runtimeinit.Runtimes(*rel.Spec)`
`717`	`717`	`if v1.IsErr(s) {`
`718`	`718`	`panic(fmt.Errorf("failed to init runtimes: %s", s.String()))`
`719`	`719`	`}`
Original file line number	Diff line number	Diff line change
`@@ -71,9 +71,7 @@ func (ao ApplyOperation) Apply(req ApplyRequest) (rsp *ApplyResponse, s v1.Sta`
`71`	`71`	`stateResourceIndex[k] = v`
`72`	`72`	`}`
`73`	`73`
`74`		`- resources := req.Release.Spec.Resources`
`75`		`- resources = append(resources, priorState.Resources...)`
`76`		`- runtimesMap, s := runtimeinit.Runtimes(resources)`
	`74`	`+ runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)`
`77`	`75`	`if v1.IsErr(s) {`
`78`	`76`	`return nil, s`
`79`	`77`	`}`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ func (do DestroyOperation) Destroy(req DestroyRequest) (rsp *DestroyResponse,`
`48`	`48`
`49`	`49`	`// only destroy resources we have recorded`
`50`	`50`	`resources := priorState.Resources`
`51`		`- runtimesMap, s := runtimeinit.Runtimes(resources)`
	`51`	`+ runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)`
`52`	`52`	`if v1.IsErr(s) {`
`53`	`53`	`return nil, s`
`54`	`54`	`}`
Original file line number	Diff line number	Diff line change
`@@ -106,7 +106,7 @@ func (rn ResourceNode) replaceK8sSecretRefs(o models.Operation) v1.Status {`
`106`	`106`	`continue`
`107`	`107`	`}`
`108`	`108`
`109`		`- externalSecretRef, err := parseExternalSecretDataRef(ref)`
	`109`	`+ externalSecretRef, err := ParseExternalSecretDataRef(ref)`
`110`	`110`	`if err != nil {`
`111`	`111`	`return v1.NewErrorStatus(err)`
`112`	`112`	`}`
`@@ -513,8 +513,8 @@ func ReplaceRef(`
`513`	`513`	`return result, v, nil`
`514`	`514`	`}`
`515`	`515`
`516`		`-// parseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.`
`517`		`-func parseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {`
	`516`	`+// ParseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.`
	`517`	`+func ParseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {`
`518`	`518`	`uri, err := url.Parse(dataRefStr)`
`519`	`519`	`if err != nil {`
`520`	`520`	`return nil, err`
Original file line number	Diff line number	Diff line change
`@@ -303,7 +303,7 @@ func TestParseExternalSecretDataRef(t *testing.T) {`
`303`	`303`	`}`
`304`	`304`	`for _, tt := range tests {`
`305`	`305`	`t.Run(tt.name, func(t *testing.T) {`
`306`		`- got, err := parseExternalSecretDataRef(tt.dataRefStr)`
	`306`	`+ got, err := ParseExternalSecretDataRef(tt.dataRefStr)`
`307`	`307`	`if (err != nil) != tt.wantErr {`
`308`	`308`	`t.Errorf("parseExternalSecretDataRef() error = %v, wantErr %v", err, tt.wantErr)`
`309`	`309`	`return`
Original file line number	Diff line number	Diff line change
`@@ -253,7 +253,7 @@ func TestPreviewOperation_Preview(t *testing.T) {`
`253`	`253`	`}`
`254`	`254`
`255`	`255`	`mockey.Mock(runtimeinit.Runtimes).To(func(`
`256`		`- resources apiv1.Resources,`
	`256`	`+ spec apiv1.Spec,`
`257`	`257`	`) (map[apiv1.Type]runtime.Runtime, v1.Status) {`
`258`	`258`	`return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil`
`259`	`259`	`}).Build()`
Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ func (wo WatchOperation) Watch(req WatchRequest) error {`
`43`	`43`
`44`	`44`	`// init runtimes`
`45`	`45`	`resources := req.Spec.Resources`
`46`		`- runtimes, s := runtimeinit.Runtimes(resources)`
	`46`	`+ runtimes, s := runtimeinit.Runtimes(*req.Spec)`
`47`	`47`	`if v1.IsErr(s) {`
`48`	`48`	`return errors.New(s.Message())`
`49`	`49`	`}`