Make cluster-proxy work with multicluster-controlplane

.github/workflows/go-postsubmit.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.20'
+  GO_VERSION: '1.22'
   GO_REQUIRED_MIN_VERSION: ''
 defaults:
   run:

.github/workflows/go-presubmit.yml

@@ -13,7 +13,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.20'
+  GO_VERSION: '1.22'
   GO_REQUIRED_MIN_VERSION: ''
 
 jobs:

.github/workflows/go-release.yml

@@ -6,7 +6,7 @@ on:
       - 'v*.*.*'
 env:
   # Common versions
-  GO_VERSION: '1.20'
+  GO_VERSION: '1.22'
   GO_REQUIRED_MIN_VERSION: ''
   GITHUB_REF: ${{ github.ref }}
   CHART_NAME: 'cluster-proxy'

.gitignore

@@ -12,7 +12,7 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 
 /apiserver.local.config
 /bin

Makefile

@@ -1,9 +1,9 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
 IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
 IMAGE_NAME = cluster-proxy
 IMAGE_TAG ?= latest
+IMG ?= $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG)
 E2E_TEST_CLUSTER_NAME ?= loopback
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
@@ -95,7 +95,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -116,7 +116,7 @@ rm -rf $$TMP_DIR ;\
 endef
 
 client-gen:
-	go install k8s.io/code-generator/cmd/client-gen@v0.23.0
+	go install k8s.io/code-generator/cmd/client-gen@v0.29.2
 	go install sigs.k8s.io/apiserver-runtime/tools/[email protected]
 	apiserver-runtime-gen \
  	--module open-cluster-management.io/cluster-proxy \

client/client.go

@@ -8,9 +8,9 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/rest"
 	clusterv1client "open-cluster-management.io/api/client/cluster/clientset/versioned"
-	clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta2"
 	"open-cluster-management.io/cluster-proxy/pkg/generated/clientset/versioned"
 	"open-cluster-management.io/cluster-proxy/pkg/util"
+	clusterv1beta2sdk "open-cluster-management.io/sdk-go/pkg/apis/cluster/v1beta2"
 )
 
 func GetProxyHost(ctx context.Context, kubeconfig *rest.Config, clusterName string, namespace string, serviceName string) (string, error) {
@@ -40,7 +40,7 @@ func GetProxyHost(ctx context.Context, kubeconfig *rest.Config, clusterName stri
 		if err != nil {
 			return "", err
 		}
-		selector, err := clusterv1beta2.BuildClusterSelector(set)
+		selector, err := clusterv1beta2sdk.BuildClusterSelector(set)
 		if err != nil {
 			return "", err
 		}

cmd/Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang:1.22 as builder
 
 WORKDIR /workspace
 
@@ -8,14 +8,13 @@ ARG KUBECTL_VERSION=v1.23.1
 ARG ADDON_AGENT_IMAGE_NAME
 
 # Build Apiserver-network-proxy binaries
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
-      wget https://github.com/kubernetes-sigs/apiserver-network-proxy/archive/refs/tags/v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
+RUN wget https://github.com/kubernetes-sigs/apiserver-network-proxy/archive/refs/tags/v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
       && tar xzvf v${APISERVER_NETWORK_PROXY_VERSION}.tar.gz \
       && cd apiserver-network-proxy-${APISERVER_NETWORK_PROXY_VERSION} \
-      && go build -o /workspace/proxy-server ./cmd/server/ \
-      && go build -o /workspace/proxy-agent ./cmd/agent/ \
-      && go build -o /workspace/proxy-test-client ./cmd/test-client/ \
-      && go build -o /workspace/proxy-test-server ./cmd/test-server/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-server ./cmd/server/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-agent ./cmd/agent/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-test-client ./cmd/test-client/ \
+      && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /workspace/proxy-test-server ./cmd/test-server/ \
       && cd /workspace \
       && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
       && chmod a+x kubectl

cmd/addon-manager/main.go

@@ -31,6 +31,8 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
@@ -46,7 +48,10 @@ import (
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/controllers"
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -71,6 +76,7 @@ func main() {
 	var signerSecretNamespace, signerSecretName string
 	var agentInstallAll bool
 	var enableKubeApiProxy bool
+	var mcKubeconfig string
 
 	logger := klogr.New()
 	klog.SetOutput(os.Stdout)
@@ -94,16 +100,32 @@ func main() {
 		"Configure the install strategy of agent on managed clusters. "+
 			"Enabling this will automatically install agent on all managed cluster.")
 	flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver")
+	flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "",
+		"The path to multicluster-controlplane kubeconfig")
 
 	flag.Parse()
 
 	// pipe controller-runtime logs to klog
 	ctrl.SetLogger(logger)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	var mcConfig, hostConfig *rest.Config
+
+	if mcKubeconfig != "" {
+		var err error
+		mcConfig, err = clientcmd.BuildConfigFromFlags("", mcKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "unable to build multicluster rest config")
+			os.Exit(1)
+		}
+		hostConfig = ctrl.GetConfigOrDie()
+	} else {
+		hostConfig = ctrl.GetConfigOrDie()
+		mcConfig = hostConfig
+	}
+
+	mgr, err := ctrl.NewManager(mcConfig, ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
+		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "cluster-proxy-addon-manager",
@@ -119,6 +141,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	hostClient, err := kubernetes.NewForConfig(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "unable to set up host kubernetes native client")
+		os.Exit(1)
+	}
+
 	nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up kubernetes native client")
@@ -147,22 +175,30 @@ func main() {
 	}
 
 	informerFactory := externalversions.NewSharedInformerFactory(client, 0)
-	nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
+	hostInformer := informers.NewSharedInformerFactoryWithOptions(hostClient, 0, informers.WithNamespace(signerSecretNamespace))
 
 	// loading self-signer
 	selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(
-		nativeClient, signerSecretNamespace, signerSecretName)
+		hostClient, signerSecretNamespace, signerSecretName)
 	if err != nil {
 		setupLog.Error(err, "failed loading self-signer")
 		os.Exit(1)
 	}
 
+	hostKubeClient, err := newHostClient(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "failed create host KubeClient")
+		os.Exit(1)
+	}
+
 	if err := controllers.RegisterClusterManagementAddonReconciler(
 		mgr,
 		selfSigner,
-		nativeClient,
-		nativeInformer.Core().V1().Secrets(),
+		hostKubeClient,
+		hostClient,
+		hostInformer.Core().V1().Secrets(),
 		supportsV1CSR,
+		mcKubeconfig != "",
 	); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
 		os.Exit(1)
@@ -195,6 +231,7 @@ func main() {
 		supportsV1CSR,
 		mgr.GetClient(),
 		nativeClient,
+		hostClient,
 		agentInstallAll,
 		enableKubeApiProxy,
 		addonClient,
@@ -212,7 +249,7 @@ func main() {
 	ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
 	defer cancel()
 	go informerFactory.Start(ctx.Done())
-	go nativeInformer.Start(ctx.Done())
+	go hostInformer.Start(ctx.Done())
 	go func() {
 		if err := addonManager.Start(ctx); err != nil {
 			setupLog.Error(err, "unable to start addon manager")
@@ -225,3 +262,18 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func newHostClient(hostConfig *rest.Config) (client.Client, error) {
+	hc, err := rest.HTTPClientFor(hostConfig)
+	if err != nil {
+		return nil, err
+	}
+	mapper, err := apiutil.NewDynamicRESTMapper(hostConfig, hc)
+	if err != nil {
+		return nil, err
+	}
+	return client.New(hostConfig, client.Options{
+		Scheme: clientgoscheme.Scheme,
+		Mapper: mapper,
+	})
+}