Kubernetes Config Update: Prioritize InClusterConfig function (#3584)

* Kubernetes Config Update: Prioritize InClusterConfig function * Create k8sconfig in pkg/util * Fix lint * Review changes * small change * Combine two methods * small tweak * use logrus library * review changes * small tweak
googleforgames · Jan 26, 2024 · f369bc2 · f369bc2
1 parent f8083af
commit f369bc2
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 59 deletions.
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -24,17 +24,6 @@ import (
 	"strings"
 	"time"
 
-	"agones.dev/agones/pkg"
-	"agones.dev/agones/pkg/client/clientset/versioned"
-	"agones.dev/agones/pkg/client/informers/externalversions"
-	"agones.dev/agones/pkg/cloudproduct"
-	"agones.dev/agones/pkg/fleetautoscalers"
-	"agones.dev/agones/pkg/fleets"
-	"agones.dev/agones/pkg/gameservers"
-	"agones.dev/agones/pkg/gameserversets"
-	"agones.dev/agones/pkg/metrics"
-	"agones.dev/agones/pkg/util/runtime"
-	"agones.dev/agones/pkg/util/signals"
 	"github.com/google/uuid"
 	"github.com/heptiolabs/healthcheck"
 	"github.com/pkg/errors"
@@ -49,9 +38,20 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+
+	"agones.dev/agones/pkg"
+	"agones.dev/agones/pkg/client/clientset/versioned"
+	"agones.dev/agones/pkg/client/informers/externalversions"
+	"agones.dev/agones/pkg/cloudproduct"
+	"agones.dev/agones/pkg/fleetautoscalers"
+	"agones.dev/agones/pkg/fleets"
+	"agones.dev/agones/pkg/gameservers"
+	"agones.dev/agones/pkg/gameserversets"
+	"agones.dev/agones/pkg/metrics"
+	"agones.dev/agones/pkg/util/runtime"
+	"agones.dev/agones/pkg/util/signals"
 )
 
 const (
@@ -133,8 +133,8 @@ func main() {
 		logger.Fatal("Could not create controller from environment or flags")
 	}
 
-	// if the kubeconfig fails BuildConfigFromFlags will try in cluster config
-	clientConf, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig)
+	// if the kubeconfig fails InClusterBuildConfig will try in cluster config
+	clientConf, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig)
 	if err != nil {
 		logger.WithError(err).Fatal("Could not create in cluster config")
 	}

diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go
@@ -24,6 +24,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/heptiolabs/healthcheck"
+	"github.com/pkg/errors"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+
 	"agones.dev/agones/pkg"
 	"agones.dev/agones/pkg/client/clientset/versioned"
 	"agones.dev/agones/pkg/client/informers/externalversions"
@@ -39,16 +49,6 @@ import (
 	"agones.dev/agones/pkg/util/runtime"
 	"agones.dev/agones/pkg/util/signals"
 	"agones.dev/agones/pkg/util/webhooks"
-	"github.com/heptiolabs/healthcheck"
-	"github.com/pkg/errors"
-	prom "github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-	"gopkg.in/natefinch/lumberjack.v2"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -114,8 +114,8 @@ func main() {
 	logger.WithField("version", pkg.Version).WithField("featureGates", runtime.EncodeFeatures()).
 		WithField("ctlConf", ctlConf).Info("starting extensions operator...")
 
-	// if the kubeconfig fails BuildConfigFromFlags will try in cluster config
-	clientConf, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig)
+	// if the kubeconfig fails InClusterBuildConfig will try in cluster config
+	clientConf, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig)
 	if err != nil {
 		logger.WithError(err).Fatal("Could not create in cluster config")
 	}

diff --git a/cmd/sdk-server/main.go b/cmd/sdk-server/main.go
@@ -25,14 +25,6 @@ import (
 	"strings"
 	"time"
 
-	"agones.dev/agones/pkg"
-	"agones.dev/agones/pkg/client/clientset/versioned"
-	"agones.dev/agones/pkg/sdk"
-	sdkalpha "agones.dev/agones/pkg/sdk/alpha"
-	sdkbeta "agones.dev/agones/pkg/sdk/beta"
-	"agones.dev/agones/pkg/sdkserver"
-	"agones.dev/agones/pkg/util/runtime"
-	"agones.dev/agones/pkg/util/signals"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -42,7 +34,15 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
+
+	"agones.dev/agones/pkg"
+	"agones.dev/agones/pkg/client/clientset/versioned"
+	"agones.dev/agones/pkg/sdk"
+	sdkalpha "agones.dev/agones/pkg/sdk/alpha"
+	sdkbeta "agones.dev/agones/pkg/sdk/beta"
+	"agones.dev/agones/pkg/sdkserver"
+	"agones.dev/agones/pkg/util/runtime"
+	"agones.dev/agones/pkg/util/signals"
 )
 
 const (
@@ -119,8 +119,8 @@ func main() {
 		}
 	default:
 		var config *rest.Config
-		// if the kubeconfig fails BuildConfigFromFlags will try in cluster config
-		config, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig)
+		// if the kubeconfig fails InClusterBuildConfig will try in cluster config
+		config, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig)
 		if err != nil {
 			logger.WithError(err).Fatal("Could not create in cluster config")
 		}

diff --git a/pkg/fleets/controller_test.go b/pkg/fleets/controller_test.go
@@ -704,10 +704,10 @@ func TestControllerUpdateFleetCounterStatus(t *testing.T) {
 	gsSet1.ObjectMeta.Name = "gsSet1"
 	gsSet1.Status.Counters = map[string]agonesv1.AggregatedCounterStatus{
 		"fullCounter": {
-            AllocatedCount:    9223372036854775807,
-            AllocatedCapacity: 9223372036854775807,
-            Capacity:          9223372036854775807,
-            Count:             9223372036854775807,
+			AllocatedCount:    9223372036854775807,
+			AllocatedCapacity: 9223372036854775807,
+			Capacity:          9223372036854775807,
+			Count:             9223372036854775807,
 		},
 		"anotherCounter": {
 			AllocatedCount:    11,
@@ -752,10 +752,10 @@ func TestControllerUpdateFleetCounterStatus(t *testing.T) {
 			ua := action.(k8stesting.UpdateAction)
 			fleet := ua.GetObject().(*agonesv1.Fleet)
 
-            assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCount)
-            assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCapacity)
-            assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Capacity)
-            assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Count)
+			assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCount)
+			assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCapacity)
+			assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Capacity)
+			assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Count)
 
 			assert.Equal(t, int64(11), fleet.Status.Counters["anotherCounter"].AllocatedCount)
 			assert.Equal(t, int64(20), fleet.Status.Counters["anotherCounter"].AllocatedCapacity)

diff --git a/pkg/util/runtime/runtime.go b/pkg/util/runtime/runtime.go
@@ -25,6 +25,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/encoding/protojson"
 	"k8s.io/apimachinery/pkg/util/runtime"
+	restclient "k8s.io/client-go/rest"
+	clientcmd "k8s.io/client-go/tools/clientcmd"
 )
 
 const sourceKey = "source"
@@ -110,3 +112,24 @@ func NewServerMux() *gwruntime.ServeMux {
 	)
 	return mux
 }
+
+// InClusterBuildConfig is a helper function that first attempts to build configurations
+// using InClusterConfig(). If InClusterConfig is unsuccessful, it then tries to build
+// configurations from a kubeconfigPath. This path is typically passed in as a command line
+// flag for cluster components. If neither the InClusterConfig nor the kubeconfigPath
+// are successful, the function logs a warning and falls back to a default configuration.
+func InClusterBuildConfig(logger *logrus.Entry, kubeconfigPath string) (*restclient.Config, error) {
+	kubeconfig, err := restclient.InClusterConfig()
+	if err == nil {
+		return kubeconfig, nil
+	}
+	logger.WithError(err).Warning("Error creating inClusterConfig, trying to build config from flags", err)
+
+	if kubeconfigPath == "" {
+		logrus.Warning("No kubeconfigPath provided. Attempting to use a default configuration.")
+	}
+
+	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
+		&clientcmd.ConfigOverrides{}).ClientConfig()
+}
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
@@ -30,11 +30,6 @@ import (
 	"testing"
 	"time"
 
-	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
-	allocationv1 "agones.dev/agones/pkg/apis/allocation/v1"
-	autoscaling "agones.dev/agones/pkg/apis/autoscaling/v1"
-	"agones.dev/agones/pkg/client/clientset/versioned"
-	"agones.dev/agones/pkg/util/runtime"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -53,7 +48,12 @@ import (
 
 	// required to use gcloud login see: https://github.com/kubernetes/client-go/issues/242
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
-	"k8s.io/client-go/tools/clientcmd"
+
+	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	allocationv1 "agones.dev/agones/pkg/apis/allocation/v1"
+	autoscaling "agones.dev/agones/pkg/apis/autoscaling/v1"
+	"agones.dev/agones/pkg/client/clientset/versioned"
+	"agones.dev/agones/pkg/util/runtime"
 )
 
 // special labels that can be put on pods to trigger automatic cleanup.
@@ -81,7 +81,8 @@ type Framework struct {
 }
 
 func newFramework(kubeconfig string, qps float32, burst int) (*Framework, error) {
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	logger := runtime.NewLoggerWithSource("framework")
+	config, err := runtime.InClusterBuildConfig(logger, kubeconfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "build config from flags failed")
 	}

diff --git a/test/eviction/evictpod.go b/test/eviction/evictpod.go
@@ -19,14 +19,14 @@ package main
 import (
 	"context"
 	"flag"
-	"log"
 	"path/filepath"
 
 	policy "k8s.io/api/policy/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
+
+	"agones.dev/agones/pkg/util/runtime"
 )
 
 // Borrowed from https://stackoverflow.com/questions/62803041/how-to-evict-or-delete-pods-from-kubernetes-using-golang-client
@@ -45,22 +45,23 @@ func main() {
 	namespace := flag.String("namespace", "default", "Namespace (defaults to `default`)")
 	pod := flag.String("pod", "", "Pod name (required)")
 	flag.Parse()
+	logger := runtime.NewLoggerWithSource("evictpod")
 
 	if *pod == "" {
-		log.Fatal("--pod must be non-empty")
+		logger.Fatal("--pod must be non-empty")
 	}
 
-	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	config, err := runtime.InClusterBuildConfig(logger, *kubeconfig)
 	if err != nil {
-		log.Fatalf("Could not build config: %v", err)
+		logger.WithError(err).Fatalf("Could not build config: %v", err)
 	}
 
 	kubeClient, err := kubernetes.NewForConfig(config)
 	if err != nil {
-		log.Fatalf("Could not create the kubernetes clientset: %v", err)
+		logger.WithError(err).Fatalf("Could not create the kubernetes clientset: %v", err)
 	}
 
 	if err := evictPod(ctx, kubeClient, *pod, *namespace); err != nil {
-		log.Fatalf("Pod eviction failed: %v", err)
+		logger.WithError(err).Fatalf("Pod eviction failed: %v", err)
 	}
 }