Skip to content

Commit

Permalink
multicluster: auto install and uninstall the remote helm release (cha…
Browse files Browse the repository at this point in the history
…os-mesh#3414)

* add remotecluster type

Signed-off-by: YangKeao <[email protected]>

* add the infrastructure to operate multiple clusters

Signed-off-by: YangKeao <[email protected]>

* remove clientset

Signed-off-by: YangKeao <[email protected]>

* modify changelog

Signed-off-by: YangKeao <[email protected]>

* add license to the remote-cluster.yaml

Signed-off-by: YangKeao <[email protected]>

* fix Wrapf arguments

Signed-off-by: YangKeao <[email protected]>

* make check

Signed-off-by: YangKeao <[email protected]>

* make the lock private

Signed-off-by: YangKeao <[email protected]>

* chore: helm chart integration and basic interface

Signed-off-by: STRRL <[email protected]>

* chore: complete the implementation of ReleaseService

Signed-off-by: STRRL <[email protected]>

* chore: update changelog

Signed-off-by: STRRL <[email protected]>

* install chaos mesh

Signed-off-by: YangKeao <[email protected]>

* add CHANGELOG

Signed-off-by: YangKeao <[email protected]>

* chore: helm chart integration and basic interface

Signed-off-by: STRRL <[email protected]>

* chore: complete the implementation of ReleaseService

Signed-off-by: STRRL <[email protected]>

* chore: update changelog

Signed-off-by: STRRL <[email protected]>

* chore: fix make check

Signed-off-by: STRRL <[email protected]>

* chore: update lincense checker config file

Signed-off-by: STRRL <[email protected]>

* test: mark examples of helm as integration test

Signed-off-by: STRRL <[email protected]>

* add condition initialization for remote cluster

Signed-off-by: YangKeao <[email protected]>

* upgrade chaosmesh release version to 2.4.1

Signed-off-by: YangKeao <[email protected]>

Signed-off-by: YangKeao <[email protected]>
Signed-off-by: STRRL <[email protected]>
Co-authored-by: STRRL <[email protected]>
Co-authored-by: Ti Chi Robot <[email protected]>
  • Loading branch information
3 people authored Nov 15, 2022
1 parent fc1349d commit 2617080
Show file tree
Hide file tree
Showing 5 changed files with 294 additions and 14 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ For more information and how-to, see [RFC: Keep A Changelog](https://github.com/
- Add `clusterregistry` package to help developers to develop multi-cluster reconciler [#3342](https://github.com/chaos-mesh/chaos-mesh/pull/3342)
- Add features about integration with helm to install Chaos Mesh in remote cluster [#3384](https://github.com/chaos-mesh/chaos-mesh/pull/3384)
- Add new CI "Manually Sign Container Images" to sign existing container images [#3708](https://github.com/chaos-mesh/chaos-mesh/pull/3708)
- Install and uninstall chaos mesh in remote cluster through `RemoteCluster` resource [#3414](https://github.com/chaos-mesh/chaos-mesh/pull/3414)

### Changed

Expand Down
42 changes: 42 additions & 0 deletions controllers/multicluster/remotecluster/condition.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright 2022 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package remotecluster

import (
corev1 "k8s.io/api/core/v1"

"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
)

func setRemoteClusterCondition(obj *v1alpha1.RemoteCluster, typ v1alpha1.RemoteClusterConditionType, status corev1.ConditionStatus, reason string) {
conditionMap := map[v1alpha1.RemoteClusterConditionType]v1alpha1.RemoteClusterCondition{
v1alpha1.RemoteClusterConditionInstalled: {Type: v1alpha1.RemoteClusterConditionInstalled, Status: corev1.ConditionFalse},
v1alpha1.RemoteClusterConditionReady: {Type: v1alpha1.RemoteClusterConditionReady, Status: corev1.ConditionFalse},
}

for _, condition := range obj.Status.Conditions {
conditionMap[condition.Type] = condition
}

conditionMap[typ] = v1alpha1.RemoteClusterCondition{Type: typ, Status: status, Reason: reason}

conditions := make([]v1alpha1.RemoteClusterCondition, 0, len(conditionMap))
for _, condition := range conditionMap {
conditions = append(conditions, condition)
}

obj.Status.Conditions = conditions
}
47 changes: 47 additions & 0 deletions controllers/multicluster/remotecluster/condition_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2022 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package remotecluster

import (
"testing"

. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"

"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
)

func TestSetRemoteClusterCondition(t *testing.T) {
RegisterTestingT(t)

obj := &v1alpha1.RemoteCluster{}
setRemoteClusterCondition(obj, v1alpha1.RemoteClusterConditionInstalled, corev1.ConditionTrue, "test")

Expect(len(obj.Status.Conditions)).To(Equal(2))
haveConditionInstalled := false
for _, condition := range obj.Status.Conditions {
if condition.Type == v1alpha1.RemoteClusterConditionInstalled {
Expect(condition.Status).To(Equal(corev1.ConditionTrue))
Expect(condition.Reason).To(Equal("test"))

haveConditionInstalled = true
} else {
Expect(condition.Status).To(Equal(corev1.ConditionFalse))
}
}

Expect(haveConditionInstalled).To(Equal(true))
}
147 changes: 133 additions & 14 deletions controllers/multicluster/remotecluster/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,27 @@ package remotecluster

import (
"context"
"encoding/json"

"github.com/go-logr/logr"
"github.com/pkg/errors"
"helm.sh/helm/v3/pkg/storage/driver"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
"github.com/chaos-mesh/chaos-mesh/controllers/multicluster/clusterregistry"
"github.com/chaos-mesh/chaos-mesh/pkg/helm"
)

const remoteClusterControllerFinalizer = "chaos-mesh/remotecluster-controllers"
const chaosMeshReleaseName = "chaos-mesh"
const chaosMeshReleaseVersion = "2.4.1"

type Reconciler struct {
Log logr.Logger
Expand All @@ -42,7 +46,7 @@ type Reconciler struct {
client.Client
}

func (r *Reconciler) getRestConfig(ctx context.Context, secretRef v1alpha1.RemoteClusterSecretRef) (*rest.Config, error) {
func (r *Reconciler) getRestConfig(ctx context.Context, secretRef v1alpha1.RemoteClusterSecretRef) (clientcmd.ClientConfig, error) {
var secret corev1.Secret
err := r.Client.Get(ctx, types.NamespacedName{
Namespace: secretRef.Namespace,
Expand All @@ -59,7 +63,7 @@ func (r *Reconciler) getRestConfig(ctx context.Context, secretRef v1alpha1.Remot
return nil, errors.Wrap(err, "load kubeconfig")
}

return clientcmd.NewDefaultClientConfig(*config, nil).ClientConfig()
return clientcmd.NewDefaultClientConfig(*config, nil), nil
}

func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
Expand All @@ -75,6 +79,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, nil
}

clientConfig, err := r.getRestConfig(ctx, obj.Spec.KubeConfig.SecretRef)
if err != nil {
r.Log.Error(err, "fail to get clientConfig from secret")
return ctrl.Result{Requeue: true}, nil
}

// if the remoteCluster itself is being deleted, we should remove the cluster controller manager
if !obj.DeletionTimestamp.IsZero() {
err := r.registry.Stop(ctx, obj.Name)
if err != nil {
Expand All @@ -84,9 +95,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
}
}

err = r.uninstallHelmRelease(ctx, &obj, clientConfig)
if err != nil {
r.Log.Error(err, "fail to uninstall helm release")
return ctrl.Result{Requeue: true}, nil
}

err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
obj.Finalizers = []string{}
return r.Client.Update(ctx, &obj)
var newObj v1alpha1.RemoteCluster
r.Client.Get(ctx, req.NamespacedName, &newObj)

newObj.Finalizers = []string{}
setRemoteClusterCondition(&newObj, v1alpha1.RemoteClusterConditionInstalled, corev1.ConditionFalse, "")

return r.Client.Update(ctx, &newObj)
})
if err != nil {
r.Log.Error(err, "fail to update finalizer", "name", obj.Name)
Expand All @@ -95,23 +117,33 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, nil
}

restConfig, err := r.getRestConfig(ctx, obj.Spec.KubeConfig.SecretRef)
err = r.ensureHelmRelease(ctx, &obj, clientConfig)
if err != nil {
r.Log.Error(err, "fail to get rest config")
r.Log.Error(err, "fail to list or install remote helm release")
return ctrl.Result{Requeue: true}, nil
}

err = r.registry.Spawn(obj.Name, restConfig)
err = r.ensureClusterControllerManager(ctx, &obj, clientConfig)
if err != nil {
if !errors.Is(err, clusterregistry.ErrAlreadyExist) {
r.Log.Error(err, "fail to spawn controllers", "name", obj.Name)
return ctrl.Result{Requeue: true}, nil
}
r.Log.Error(err, "fail to boot remote cluster controller manager")
return ctrl.Result{Requeue: true}, nil
}
obj.Finalizers = []string{remoteClusterControllerFinalizer}

if err != nil {
r.Log.Error(err, "fail to operate the helm release in remote cluster")
return ctrl.Result{Requeue: true}, nil
}

err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
obj.Finalizers = []string{remoteClusterControllerFinalizer}
return r.Client.Update(ctx, &obj)
var newObj v1alpha1.RemoteCluster
r.Client.Get(ctx, req.NamespacedName, &newObj)

newObj.Finalizers = obj.Finalizers
setRemoteClusterCondition(&newObj, v1alpha1.RemoteClusterConditionInstalled, corev1.ConditionTrue, "")
// TODO: do auto config migration
newObj.Status.CurrentVersion = chaosMeshReleaseVersion
return r.Client.Update(ctx, &newObj)
})
if err != nil {
r.Log.Error(err, "fail to update finalizer", "name", obj.Name)
Expand All @@ -120,3 +152,90 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu

return ctrl.Result{}, nil
}

func (r *Reconciler) ensureClusterControllerManager(ctx context.Context, obj *v1alpha1.RemoteCluster, config clientcmd.ClientConfig) error {
restConfig, err := config.ClientConfig()
if err != nil {
return errors.Wrap(err, "get rest config from client config")
}

err = r.registry.Spawn(obj.Name, restConfig)
if err != nil {
if !errors.Is(err, clusterregistry.ErrAlreadyExist) {
return err
}
}

return nil
}

func (r *Reconciler) getHelmClient(ctx context.Context, clientConfig clientcmd.ClientConfig) (*helm.HelmClient, error) {
restClientGetter := helm.NewRESTClientGetter(clientConfig)

helmClient, err := helm.NewHelmClient(restClientGetter, r.Log)
if err != nil {
return nil, err
}

return helmClient, nil
}

func (r *Reconciler) ensureHelmRelease(ctx context.Context, obj *v1alpha1.RemoteCluster, clientConfig clientcmd.ClientConfig) error {
helmClient, err := r.getHelmClient(ctx, clientConfig)
if err != nil {
return err
}

_, err = helmClient.GetRelease(obj.Spec.Namespace, chaosMeshReleaseName)
if err != nil {
if errors.Is(err, driver.ErrReleaseNotFound) {
chart, err := helm.FetchChaosMeshChart(ctx, chaosMeshReleaseVersion)
if err != nil {
return err
}

values := make(map[string]interface{})
if obj.Spec.ConfigOverride != nil {
err = json.Unmarshal(obj.Spec.ConfigOverride, &values)
if err != nil {
return err
}
}
_, err = helmClient.UpgradeOrInstall(obj.Spec.Namespace, chaosMeshReleaseName, chart, values)
if err != nil {
return err
}
} else {
return err
}
}
return nil
}

func (r *Reconciler) uninstallHelmRelease(ctx context.Context, obj *v1alpha1.RemoteCluster, clientConfig clientcmd.ClientConfig) error {
helmClient, err := r.getHelmClient(ctx, clientConfig)
if err != nil {
return err
}

_, err = helmClient.GetRelease(obj.Spec.Namespace, chaosMeshReleaseName)
if err != nil {
if errors.Is(err, driver.ErrReleaseNotFound) {
return nil
}

return err
}

// the release still exist
_, err = helmClient.UninstallRelease(obj.Spec.Namespace, chaosMeshReleaseName)
if err != nil {
if errors.Is(err, driver.ErrReleaseNotFound) {
return nil
}

return err
}

return nil
}
71 changes: 71 additions & 0 deletions pkg/helm/restclientgetter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright 2022 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package helm

import (
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/discovery"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/clientcmd"
)

var _ genericclioptions.RESTClientGetter = &restClientGetter{}

type restClientGetter struct {
clientConfig clientcmd.ClientConfig
}

func NewRESTClientGetter(clientConfig clientcmd.ClientConfig) genericclioptions.RESTClientGetter {
return &restClientGetter{
clientConfig,
}
}

func (getter *restClientGetter) ToRESTConfig() (*rest.Config, error) {
return getter.clientConfig.ClientConfig()
}

func (getter *restClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
restConfig, err := getter.clientConfig.ClientConfig()
if err != nil {
return nil, errors.Wrap(err, "get rest config from client config")
}

client, err := discovery.NewDiscoveryClientForConfig(restConfig)
if err != nil {
return nil, errors.Wrap(err, "create discovery client")
}
return memory.NewMemCacheClient(client), nil
}

func (getter *restClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
discoveryClient, err := getter.ToDiscoveryClient()
if err != nil {
return nil, err
}

mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
expander := restmapper.NewShortcutExpander(mapper, discoveryClient)
return expander, nil
}

func (getter *restClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
return getter.clientConfig
}

0 comments on commit 2617080

Please sign in to comment.