Support multicluster control plane

Signed-off-by: Tamal Saha <[email protected]>

Author: tamalsaha

cmd/addon-manager/main.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/textlogger"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
@@ -70,6 +71,7 @@ func main() {
 	var signerSecretNamespace, signerSecretName string
 	var agentInstallAll bool
 	var enableKubeApiProxy bool
+	var mcKubeconfig string
 
 	logger := textlogger.NewLogger(textlogger.NewConfig())
 	klog.SetOutput(os.Stdout)
@@ -93,37 +95,70 @@ func main() {
 		"Configure the install strategy of agent on managed clusters. "+
 			"Enabling this will automatically install agent on all managed cluster.")
 	flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver")
+	flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "",
+		"The path to multicluster-controlplane kubeconfig")
 
 	flag.Parse()
 
 	// pipe controller-runtime logs to klog
 	ctrl.SetLogger(logger)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	hostManager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "cluster-proxy-addon-manager",
 	})
 	if err != nil {
-		setupLog.Error(err, "unable to start manager")
+		setupLog.Error(err, "unable to create host manager")
 		os.Exit(1)
 	}
 
-	nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
+	hostNativeClient, err := kubernetes.NewForConfig(hostManager.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up kubernetes native client")
 		os.Exit(1)
 	}
 
-	addonClient, err := addonclient.NewForConfig(mgr.GetConfig())
+	mcMode := mcKubeconfig != ""
+	var mcManager ctrl.Manager
+	var mcNativeClient kubernetes.Interface
+	if mcMode {
+		mcConfig, err := clientcmd.BuildConfigFromFlags("", mcKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "unable to build multicluster rest config")
+			os.Exit(1)
+		}
+		mcManager, err = ctrl.NewManager(mcConfig, ctrl.Options{
+			Scheme:                 scheme,
+			Metrics:                metricsserver.Options{BindAddress: ""},
+			HealthProbeBindAddress: "",
+			LeaderElection:         enableLeaderElection,
+			LeaderElectionID:       "cluster-proxy-addon-manager",
+		})
+		if err != nil {
+			setupLog.Error(err, "unable to create mc manager")
+			os.Exit(1)
+		}
+
+		mcNativeClient, err = kubernetes.NewForConfig(mcManager.GetConfig())
+		if err != nil {
+			setupLog.Error(err, "unable to set up multicluster native client")
+			os.Exit(1)
+		}
+	} else {
+		mcManager = hostManager
+		mcNativeClient = hostNativeClient
+	}
+
+	addonClient, err := addonclient.NewForConfig(mcManager.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up ocm addon client")
 		os.Exit(1)
 	}
 
-	supportsV1CSR, supportsV1beta1CSR, err := addonutil.IsCSRSupported(nativeClient)
+	supportsV1CSR, supportsV1beta1CSR, err := addonutil.IsCSRSupported(mcNativeClient)
 	if err != nil {
 		setupLog.Error(err, "unable to detect available CSR API versions")
 		os.Exit(1)
@@ -138,43 +173,45 @@ func main() {
 		os.Exit(1)
 	}
 
-	nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
+	hostNativeInformer := informers.NewSharedInformerFactoryWithOptions(hostNativeClient, 0)
 
 	// loading self-signer
 	selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(
-		nativeClient, signerSecretNamespace, signerSecretName)
+		hostNativeClient, signerSecretNamespace, signerSecretName)
 	if err != nil {
 		setupLog.Error(err, "failed loading self-signer")
 		os.Exit(1)
 	}
 
 	if err := controllers.RegisterClusterManagementAddonReconciler(
-		mgr,
+		hostManager,
 		selfSigner,
-		nativeClient,
-		nativeInformer.Core().V1().Secrets(),
+		hostNativeClient,
+		hostNativeInformer.Core().V1().Secrets(),
+		mcManager,
+		mcMode,
 		supportsV1CSR,
 	); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
 		os.Exit(1)
 	}
 
-	if err := controllers.RegisterServiceResolverReconciler(mgr); err != nil {
+	if err := controllers.RegisterServiceResolverReconciler(mcManager); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ServiceResolverReconciler")
 		os.Exit(1)
 	}
 
 	//+kubebuilder:scaffold:builder
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+	if err := hostManager.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+	if err := hostManager.AddReadyzCheck("readyz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
 
-	addonManager, err := addonmanager.New(mgr.GetConfig())
+	addonManager, err := addonmanager.New(mcManager.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
@@ -184,8 +221,9 @@ func main() {
 		selfSigner,
 		signerSecretNamespace,
 		supportsV1CSR,
-		mgr.GetClient(),
-		nativeClient,
+		mcManager.GetClient(),
+		mcNativeClient,
+		hostNativeClient,
 		agentInstallAll,
 		enableKubeApiProxy,
 		addonClient,
@@ -202,16 +240,19 @@ func main() {
 
 	ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
 	defer cancel()
-	go nativeInformer.Start(ctx.Done())
+	go hostNativeInformer.Start(ctx.Done())
 	go func() {
 		if err := addonManager.Start(ctx); err != nil {
 			setupLog.Error(err, "unable to start addon manager")
 			os.Exit(1)
 		}
 	}()
-	setupLog.Info("starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
+	if mcMode {
+		go mcManager.Start(ctx)
+	}
+	setupLog.Info("starting host manager")
+	if err := hostManager.Start(ctx); err != nil {
+		setupLog.Error(err, "problem running host manager")
 		os.Exit(1)
 	}
 }

pkg/proxyagent/agent/agent.go

@@ -51,8 +51,9 @@ func NewAgentAddon(
 	signer selfsigned.SelfSigner,
 	signerNamespace string,
 	v1CSRSupported bool,
-	runtimeClient client.Client,
-	nativeClient kubernetes.Interface,
+	mcRtc client.Client,
+	mcNativeClient kubernetes.Interface,
+	hostNativeClient kubernetes.Interface,
 	agentInstallAll bool,
 	enableKubeApiProxy bool,
 	addonClient addonclient.Interface) (agent.AgentAddon, error) {
@@ -93,7 +94,7 @@ func NewAgentAddon(
 			CSRApproveCheck: func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn, csr *csrv1.CertificateSigningRequest) bool {
 				return cluster.Spec.HubAcceptsClient
 			},
-			PermissionConfig: utils.NewRBACPermissionConfigBuilder(nativeClient).
+			PermissionConfig: utils.NewRBACPermissionConfigBuilder(mcNativeClient).
 				WithStaticRole(&rbacv1.Role{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "cluster-proxy-addon-agent",
@@ -134,7 +135,7 @@ func NewAgentAddon(
 			utils.AddOnDeploymentConfigGVR,
 		).
 		WithGetValuesFuncs(
-			GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
+			GetClusterProxyValueFunc(mcRtc, hostNativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
 			addonfactory.GetAddOnDeploymentConfigValues(
 				utils.NewAddOnDeploymentConfigGetter(addonClient),
 				toAgentAddOnChartValues(caCertData),
@@ -149,8 +150,8 @@ func NewAgentAddon(
 }
 
 func GetClusterProxyValueFunc(
-	runtimeClient client.Client,
-	nativeClient kubernetes.Interface,
+	mcRtc client.Client,
+	hostNativeClient kubernetes.Interface,
 	signerNamespace string,
 	caCertData []byte,
 	v1CSRSupported bool,
@@ -173,7 +174,7 @@ func GetClusterProxyValueFunc(
 		}
 
 		proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{}
-		if err := runtimeClient.Get(context.TODO(), types.NamespacedName{
+		if err := mcRtc.Get(context.TODO(), types.NamespacedName{
 			Name: managedProxyConfigurations[0],
 		}, proxyConfig); err != nil {
 			return nil, err
@@ -188,7 +189,7 @@ func GetClusterProxyValueFunc(
 		// find the referenced proxy load-balancer prescribed in the proxy config if there's any
 		var proxyServerLoadBalancer *corev1.Service
 		if proxyConfig.Spec.ProxyServer.Entrypoint.Type == proxyv1alpha1.EntryPointTypeLoadBalancerService {
-			entrySvc, err := nativeClient.CoreV1().
+			entrySvc, err := hostNativeClient.CoreV1().
 				Services(proxyConfig.Spec.ProxyServer.Namespace).
 				Get(context.TODO(),
 					proxyConfig.Spec.ProxyServer.Entrypoint.LoadBalancerService.Name,
@@ -228,7 +229,7 @@ func GetClusterProxyValueFunc(
 		// "agent-client" to the managed clusters.
 		certDataBase64, keyDataBase64 := "", ""
 		if !v1CSRSupported {
-			agentClientSecret, err := nativeClient.CoreV1().
+			agentClientSecret, err := hostNativeClient.CoreV1().
 				Secrets(signerNamespace).
 				Get(context.TODO(), common.AgentClientSecretName, metav1.GetOptions{})
 			if err != nil {
@@ -249,7 +250,7 @@ func GetClusterProxyValueFunc(
 
 		// List all available managedClusterSets
 		managedClusterSetList := &clusterv1beta2.ManagedClusterSetList{}
-		err = runtimeClient.List(context.TODO(), managedClusterSetList)
+		err = mcRtc.List(context.TODO(), managedClusterSetList)
 		if err != nil {
 			return nil, err
 		}
@@ -261,7 +262,7 @@ func GetClusterProxyValueFunc(
 
 		// List all available serviceResolvers
 		serviceResolverList := &proxyv1alpha1.ManagedProxyServiceResolverList{}
-		err = runtimeClient.List(context.TODO(), serviceResolverList)
+		err = mcRtc.List(context.TODO(), serviceResolverList)
 		if err != nil {
 			return nil, err
 		}