From f369bc284328e0e38d53002ec71ab998c054b78c Mon Sep 17 00:00:00 2001 From: Kalaiselvim <117940852+Kalaiselvi84@users.noreply.github.com> Date: Fri, 26 Jan 2024 02:27:37 +0000 Subject: [PATCH] Kubernetes Config Update: Prioritize InClusterConfig function (#3584) * Kubernetes Config Update: Prioritize InClusterConfig function * Create k8sconfig in pkg/util * Fix lint * Review changes * small change * Combine two methods * small tweak * use logrus library * review changes * small tweak --- cmd/controller/main.go | 28 ++++++++++++++-------------- cmd/extensions/main.go | 24 ++++++++++++------------ cmd/sdk-server/main.go | 22 +++++++++++----------- pkg/fleets/controller_test.go | 16 ++++++++-------- pkg/util/runtime/runtime.go | 23 +++++++++++++++++++++++ test/e2e/framework/framework.go | 15 ++++++++------- test/eviction/evictpod.go | 15 ++++++++------- 7 files changed, 84 insertions(+), 59 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index d25d6a80fa..67efe61a45 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -24,17 +24,6 @@ import ( "strings" "time" - "agones.dev/agones/pkg" - "agones.dev/agones/pkg/client/clientset/versioned" - "agones.dev/agones/pkg/client/informers/externalversions" - "agones.dev/agones/pkg/cloudproduct" - "agones.dev/agones/pkg/fleetautoscalers" - "agones.dev/agones/pkg/fleets" - "agones.dev/agones/pkg/gameservers" - "agones.dev/agones/pkg/gameserversets" - "agones.dev/agones/pkg/metrics" - "agones.dev/agones/pkg/util/runtime" - "agones.dev/agones/pkg/util/signals" "github.com/google/uuid" "github.com/heptiolabs/healthcheck" "github.com/pkg/errors" @@ -49,9 +38,20 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" + + "agones.dev/agones/pkg" + "agones.dev/agones/pkg/client/clientset/versioned" + "agones.dev/agones/pkg/client/informers/externalversions" + "agones.dev/agones/pkg/cloudproduct" + "agones.dev/agones/pkg/fleetautoscalers" + "agones.dev/agones/pkg/fleets" + "agones.dev/agones/pkg/gameservers" + "agones.dev/agones/pkg/gameserversets" + "agones.dev/agones/pkg/metrics" + "agones.dev/agones/pkg/util/runtime" + "agones.dev/agones/pkg/util/signals" ) const ( @@ -133,8 +133,8 @@ func main() { logger.Fatal("Could not create controller from environment or flags") } - // if the kubeconfig fails BuildConfigFromFlags will try in cluster config - clientConf, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig) + // if the kubeconfig fails InClusterBuildConfig will try in cluster config + clientConf, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig) if err != nil { logger.WithError(err).Fatal("Could not create in cluster config") } diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go index 02fcbb8847..f09e83aaea 100644 --- a/cmd/extensions/main.go +++ b/cmd/extensions/main.go @@ -24,6 +24,16 @@ import ( "strings" "time" + "github.com/heptiolabs/healthcheck" + "github.com/pkg/errors" + prom "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "gopkg.in/natefinch/lumberjack.v2" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "agones.dev/agones/pkg" "agones.dev/agones/pkg/client/clientset/versioned" "agones.dev/agones/pkg/client/informers/externalversions" @@ -39,16 +49,6 @@ import ( "agones.dev/agones/pkg/util/runtime" "agones.dev/agones/pkg/util/signals" "agones.dev/agones/pkg/util/webhooks" - "github.com/heptiolabs/healthcheck" - "github.com/pkg/errors" - prom "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "gopkg.in/natefinch/lumberjack.v2" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" ) const ( @@ -114,8 +114,8 @@ func main() { logger.WithField("version", pkg.Version).WithField("featureGates", runtime.EncodeFeatures()). WithField("ctlConf", ctlConf).Info("starting extensions operator...") - // if the kubeconfig fails BuildConfigFromFlags will try in cluster config - clientConf, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig) + // if the kubeconfig fails InClusterBuildConfig will try in cluster config + clientConf, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig) if err != nil { logger.WithError(err).Fatal("Could not create in cluster config") } diff --git a/cmd/sdk-server/main.go b/cmd/sdk-server/main.go index 76dcd3f936..3166160813 100644 --- a/cmd/sdk-server/main.go +++ b/cmd/sdk-server/main.go @@ -25,14 +25,6 @@ import ( "strings" "time" - "agones.dev/agones/pkg" - "agones.dev/agones/pkg/client/clientset/versioned" - "agones.dev/agones/pkg/sdk" - sdkalpha "agones.dev/agones/pkg/sdk/alpha" - sdkbeta "agones.dev/agones/pkg/sdk/beta" - "agones.dev/agones/pkg/sdkserver" - "agones.dev/agones/pkg/util/runtime" - "agones.dev/agones/pkg/util/signals" gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -42,7 +34,15 @@ import ( "google.golang.org/grpc/credentials/insecure" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" + + "agones.dev/agones/pkg" + "agones.dev/agones/pkg/client/clientset/versioned" + "agones.dev/agones/pkg/sdk" + sdkalpha "agones.dev/agones/pkg/sdk/alpha" + sdkbeta "agones.dev/agones/pkg/sdk/beta" + "agones.dev/agones/pkg/sdkserver" + "agones.dev/agones/pkg/util/runtime" + "agones.dev/agones/pkg/util/signals" ) const ( @@ -119,8 +119,8 @@ func main() { } default: var config *rest.Config - // if the kubeconfig fails BuildConfigFromFlags will try in cluster config - config, err := clientcmd.BuildConfigFromFlags("", ctlConf.KubeConfig) + // if the kubeconfig fails InClusterBuildConfig will try in cluster config + config, err := runtime.InClusterBuildConfig(logger, ctlConf.KubeConfig) if err != nil { logger.WithError(err).Fatal("Could not create in cluster config") } diff --git a/pkg/fleets/controller_test.go b/pkg/fleets/controller_test.go index 5b4f4909d4..ae1bfba0da 100644 --- a/pkg/fleets/controller_test.go +++ b/pkg/fleets/controller_test.go @@ -704,10 +704,10 @@ func TestControllerUpdateFleetCounterStatus(t *testing.T) { gsSet1.ObjectMeta.Name = "gsSet1" gsSet1.Status.Counters = map[string]agonesv1.AggregatedCounterStatus{ "fullCounter": { - AllocatedCount: 9223372036854775807, - AllocatedCapacity: 9223372036854775807, - Capacity: 9223372036854775807, - Count: 9223372036854775807, + AllocatedCount: 9223372036854775807, + AllocatedCapacity: 9223372036854775807, + Capacity: 9223372036854775807, + Count: 9223372036854775807, }, "anotherCounter": { AllocatedCount: 11, @@ -752,10 +752,10 @@ func TestControllerUpdateFleetCounterStatus(t *testing.T) { ua := action.(k8stesting.UpdateAction) fleet := ua.GetObject().(*agonesv1.Fleet) - assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCount) - assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCapacity) - assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Capacity) - assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Count) + assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCount) + assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].AllocatedCapacity) + assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Capacity) + assert.Equal(t, int64(9223372036854775807), fleet.Status.Counters["fullCounter"].Count) assert.Equal(t, int64(11), fleet.Status.Counters["anotherCounter"].AllocatedCount) assert.Equal(t, int64(20), fleet.Status.Counters["anotherCounter"].AllocatedCapacity) diff --git a/pkg/util/runtime/runtime.go b/pkg/util/runtime/runtime.go index d2965a809b..4c3039235a 100644 --- a/pkg/util/runtime/runtime.go +++ b/pkg/util/runtime/runtime.go @@ -25,6 +25,8 @@ import ( "github.com/sirupsen/logrus" "google.golang.org/protobuf/encoding/protojson" "k8s.io/apimachinery/pkg/util/runtime" + restclient "k8s.io/client-go/rest" + clientcmd "k8s.io/client-go/tools/clientcmd" ) const sourceKey = "source" @@ -110,3 +112,24 @@ func NewServerMux() *gwruntime.ServeMux { ) return mux } + +// InClusterBuildConfig is a helper function that first attempts to build configurations +// using InClusterConfig(). If InClusterConfig is unsuccessful, it then tries to build +// configurations from a kubeconfigPath. This path is typically passed in as a command line +// flag for cluster components. If neither the InClusterConfig nor the kubeconfigPath +// are successful, the function logs a warning and falls back to a default configuration. +func InClusterBuildConfig(logger *logrus.Entry, kubeconfigPath string) (*restclient.Config, error) { + kubeconfig, err := restclient.InClusterConfig() + if err == nil { + return kubeconfig, nil + } + logger.WithError(err).Warning("Error creating inClusterConfig, trying to build config from flags", err) + + if kubeconfigPath == "" { + logrus.Warning("No kubeconfigPath provided. Attempting to use a default configuration.") + } + + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{}).ClientConfig() +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 158c89bb25..f1acf14b0a 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -30,11 +30,6 @@ import ( "testing" "time" - agonesv1 "agones.dev/agones/pkg/apis/agones/v1" - allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" - autoscaling "agones.dev/agones/pkg/apis/autoscaling/v1" - "agones.dev/agones/pkg/client/clientset/versioned" - "agones.dev/agones/pkg/util/runtime" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/pflag" @@ -53,7 +48,12 @@ import ( // required to use gcloud login see: https://github.com/kubernetes/client-go/issues/242 _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/client-go/tools/clientcmd" + + agonesv1 "agones.dev/agones/pkg/apis/agones/v1" + allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" + autoscaling "agones.dev/agones/pkg/apis/autoscaling/v1" + "agones.dev/agones/pkg/client/clientset/versioned" + "agones.dev/agones/pkg/util/runtime" ) // special labels that can be put on pods to trigger automatic cleanup. @@ -81,7 +81,8 @@ type Framework struct { } func newFramework(kubeconfig string, qps float32, burst int) (*Framework, error) { - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + logger := runtime.NewLoggerWithSource("framework") + config, err := runtime.InClusterBuildConfig(logger, kubeconfig) if err != nil { return nil, errors.Wrap(err, "build config from flags failed") } diff --git a/test/eviction/evictpod.go b/test/eviction/evictpod.go index cb9ad1e327..fe0c3a20a6 100644 --- a/test/eviction/evictpod.go +++ b/test/eviction/evictpod.go @@ -19,14 +19,14 @@ package main import ( "context" "flag" - "log" "path/filepath" policy "k8s.io/api/policy/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" + + "agones.dev/agones/pkg/util/runtime" ) // Borrowed from https://stackoverflow.com/questions/62803041/how-to-evict-or-delete-pods-from-kubernetes-using-golang-client @@ -45,22 +45,23 @@ func main() { namespace := flag.String("namespace", "default", "Namespace (defaults to `default`)") pod := flag.String("pod", "", "Pod name (required)") flag.Parse() + logger := runtime.NewLoggerWithSource("evictpod") if *pod == "" { - log.Fatal("--pod must be non-empty") + logger.Fatal("--pod must be non-empty") } - config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + config, err := runtime.InClusterBuildConfig(logger, *kubeconfig) if err != nil { - log.Fatalf("Could not build config: %v", err) + logger.WithError(err).Fatalf("Could not build config: %v", err) } kubeClient, err := kubernetes.NewForConfig(config) if err != nil { - log.Fatalf("Could not create the kubernetes clientset: %v", err) + logger.WithError(err).Fatalf("Could not create the kubernetes clientset: %v", err) } if err := evictPod(ctx, kubeClient, *pod, *namespace); err != nil { - log.Fatalf("Pod eviction failed: %v", err) + logger.WithError(err).Fatalf("Pod eviction failed: %v", err) } }