Make cluster-proxy work with multicluster-controlplane

Signed-off-by: Tamal Saha <[email protected]>

.gitignore

@@ -12,7 +12,7 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 
 /apiserver.local.config
 /bin

Makefile

@@ -1,9 +1,9 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
 IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
 IMAGE_NAME = cluster-proxy
 IMAGE_TAG ?= latest
+IMG ?= $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG)
 E2E_TEST_CLUSTER_NAME ?= loopback
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"

cmd/Dockerfile

@@ -8,14 +8,14 @@ ARG KUBECTL_VERSION=v1.23.1
 ARG ADDON_AGENT_IMAGE_NAME
 
 # Build Apiserver-network-proxy binaries
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
-      wget https://github.com/kubernetes-sigs/apiserver-network-proxy/archive/refs/tags/v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
+RUN set -x \
+      && wget https://github.com/kubernetes-sigs/apiserver-network-proxy/archive/refs/tags/v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
       && tar xzvf v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
       && cd apiserver-network-proxy-${APISERVER_NETWORK_PROXY_VERSION} \
-      && go build -o /workspace/proxy-server ./cmd/server/ \
-      && go build -o /workspace/proxy-agent ./cmd/agent/ \
-      && go build -o /workspace/proxy-test-client ./cmd/test-client/ \
-      && go build -o /workspace/proxy-test-server ./cmd/test-server/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-server ./cmd/server/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-agent ./cmd/agent/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-test-client ./cmd/test-client/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-test-server ./cmd/test-server/ \
       && cd /workspace \
       && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
       && chmod a+x kubectl

cmd/addon-manager/main.go

@@ -31,6 +31,8 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
@@ -46,6 +48,8 @@ import (
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/controllers"
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	//+kubebuilder:scaffold:imports
 )
@@ -71,6 +75,7 @@ func main() {
 	var signerSecretNamespace, signerSecretName string
 	var agentInstallAll bool
 	var enableKubeApiProxy bool
+	var mcKubeconfig string
 
 	logger := klogr.New()
 	klog.SetOutput(os.Stdout)
@@ -94,13 +99,30 @@ func main() {
 		"Configure the install strategy of agent on managed clusters. "+
 			"Enabling this will automatically install agent on all managed cluster.")
 	flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver")
+	flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "",
+		"The path to multicluster-controlplane kubeconfig")
 
 	flag.Parse()
 
 	// pipe controller-runtime logs to klog
 	ctrl.SetLogger(logger)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	var mcConfig, hostConfig *rest.Config
+
+	if mcKubeconfig != "" {
+		var err error
+		mcConfig, err = clientcmd.BuildConfigFromFlags("", mcKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "unable to build multicluster rest config")
+			os.Exit(1)
+		}
+		hostConfig = ctrl.GetConfigOrDie()
+	} else {
+		hostConfig = ctrl.GetConfigOrDie()
+		mcConfig = hostConfig
+	}
+
+	mgr, err := ctrl.NewManager(mcConfig, ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
 		Port:                   9443,
@@ -119,6 +141,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	hostClient, err := kubernetes.NewForConfig(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "unable to set up host kubernetes native client")
+		os.Exit(1)
+	}
+
 	nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up kubernetes native client")
@@ -147,22 +175,30 @@ func main() {
 	}
 
 	informerFactory := externalversions.NewSharedInformerFactory(client, 0)
-	nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
+	hostInformer := informers.NewSharedInformerFactoryWithOptions(hostClient, 0, informers.WithNamespace(signerSecretNamespace))
 
 	// loading self-signer
 	selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(
-		nativeClient, signerSecretNamespace, signerSecretName)
+		hostClient, signerSecretNamespace, signerSecretName)
 	if err != nil {
 		setupLog.Error(err, "failed loading self-signer")
 		os.Exit(1)
 	}
 
+	hostKubeClient, err := newHostClient(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "failed create host KubeClient")
+		os.Exit(1)
+	}
+
 	if err := controllers.RegisterClusterManagementAddonReconciler(
 		mgr,
 		selfSigner,
-		nativeClient,
-		nativeInformer.Core().V1().Secrets(),
+		hostKubeClient,
+		hostClient,
+		hostInformer.Core().V1().Secrets(),
 		supportsV1CSR,
+		mcKubeconfig != "",
 	); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
 		os.Exit(1)
@@ -195,6 +231,7 @@ func main() {
 		supportsV1CSR,
 		mgr.GetClient(),
 		nativeClient,
+		hostClient,
 		agentInstallAll,
 		enableKubeApiProxy,
 		addonClient,
@@ -212,7 +249,7 @@ func main() {
 	ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
 	defer cancel()
 	go informerFactory.Start(ctx.Done())
-	go nativeInformer.Start(ctx.Done())
+	go hostInformer.Start(ctx.Done())
 	go func() {
 		if err := addonManager.Start(ctx); err != nil {
 			setupLog.Error(err, "unable to start addon manager")
@@ -225,3 +262,18 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func newHostClient(hostConfig *rest.Config) (client.Client, error) {
+	hc, err := rest.HTTPClientFor(hostConfig)
+	if err != nil {
+		return nil, err
+	}
+	mapper, err := apiutil.NewDynamicRESTMapper(hostConfig, hc)
+	if err != nil {
+		return nil, err
+	}
+	return client.New(hostConfig, client.Options{
+		Scheme: clientgoscheme.Scheme,
+		Mapper: mapper,
+	})
+}

go.mod

@@ -9,10 +9,10 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.15.1
 	github.com/stretchr/testify v1.8.1
-	google.golang.org/grpc v1.51.0
-	k8s.io/api v0.27.4
-	k8s.io/apimachinery v0.27.4
-	k8s.io/client-go v0.27.4
+	google.golang.org/grpc v1.56.3
+	k8s.io/api v0.27.9
+	k8s.io/apimachinery v0.27.9
+	k8s.io/client-go v0.27.9
 	k8s.io/klog/v2 v2.90.1
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491
 	open-cluster-management.io/addon-framework v0.8.0
@@ -79,23 +79,23 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.12.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.11.1 // indirect
-	k8s.io/apiextensions-apiserver v0.27.2 // indirect
-	k8s.io/apiserver v0.27.2 // indirect
-	k8s.io/component-base v0.27.4 // indirect
+	k8s.io/apiextensions-apiserver v0.27.9 // indirect
+	k8s.io/apiserver v0.27.9 // indirect
+	k8s.io/component-base v0.27.9 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect