From 2cb73c937d544c9b2907194bd21ccbb2dcd78241 Mon Sep 17 00:00:00 2001 From: wanggangqiang Date: Wed, 6 Sep 2023 10:46:41 +0800 Subject: [PATCH] update log; fix ccm deployment yaml --- Makefile | 6 +- deploy/kube-cloud-controller-manager.yaml | 10 +- go.mod | 73 +++++- go.sum | 3 - .../clusternode/cluster_node_controller.go | 7 +- .../endpoint/endpoint_controller.go | 13 +- .../service/cloud_service_controller.go | 5 +- pkg/controllers/utils/node.go | 2 +- pkg/executor/backend.go | 24 +- pkg/executor/eip.go | 28 +- pkg/executor/lb.go | 58 +++-- pkg/executor/listener.go | 12 +- pkg/executor/sg.go | 37 ++- pkg/executor/tags.go | 4 +- pkg/qingcloud/annotations.go | 2 +- pkg/qingcloud/loadbalancer_utils.go | 2 +- pkg/qingcloud/qingcloud.go | 241 +++++++++++------- pkg/qingcloud/qingcloud_utils.go | 10 +- pkg/util/util.go | 38 ++- 19 files changed, 404 insertions(+), 171 deletions(-) diff --git a/Makefile b/Makefile index c69e4c24..eda3d6a1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # GitHub viewer defaults to 8, change with ?ts=4 in URL GIT_REPOSITORY= github.com/yunify/qingcloud-cloud-controller-manager -IMG?= qingcloud/cloud-controller-manager:v1.4.18 +IMG?= qingcloud/cloud-controller-manager:v1.4.19 #Debug level: 0, 1, 2 (1 true, 2 use bash) DEBUG?= 0 DOCKERFILE?= deploy/Dockerfile @@ -31,9 +31,9 @@ endif publish: test build docker build -t ${IMG} -f ${DOCKERFILE} bin/ @echo "updating kustomize image patch file for manager resource" - sed -i'' -e 's@image: .*@image: '"${IMG}"'@' config/${TARGET}/manager_image_patch.yaml + #sed -i'' -e 's@image: .*@image: '"${IMG}"'@' config/${TARGET}/manager_image_patch.yaml docker push ${IMG} - kustomize build config/${TARGET} > ${DEPLOY} + #kustomize build config/${TARGET} > ${DEPLOY} clean: rm -rf bin/ diff --git a/deploy/kube-cloud-controller-manager.yaml b/deploy/kube-cloud-controller-manager.yaml index ba4452df..6ca3ab6a 100644 --- a/deploy/kube-cloud-controller-manager.yaml +++ b/deploy/kube-cloud-controller-manager.yaml @@ -137,7 +137,7 @@ spec: - -v=3 - --cloud-provider=qingcloud - --cloud-config=/etc/kubernetes/qingcloud.yaml - image: qingcloud/cloud-controller-manager:v1.4.18 + image: qingcloud/cloud-controller-manager:v1.4.19 imagePullPolicy: IfNotPresent name: qingcloud-cloud-controller-manager resources: @@ -167,8 +167,10 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: + defaultMode: 420 name: lbconfig name: lbconfig - - name: qingcloud - secret: - secretName: qcsecret + - configMap: + defaultMode: 420 + name: ccm-qingcloud + name: qingcloud diff --git a/go.mod b/go.mod index ac94fd36..4096369c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/yunify/qingcloud-cloud-controller-manager -go 1.16 +go 1.19 require ( github.com/davecgh/go-spew v1.1.1 @@ -13,11 +13,80 @@ require ( k8s.io/cloud-provider v0.21.1 k8s.io/component-base v0.21.1 k8s.io/controller-manager v0.21.1 - k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.9.0 ) +require ( + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-openapi/jsonpointer v0.19.3 // indirect + github.com/go-openapi/jsonreference v0.19.3 // indirect + github.com/go-openapi/spec v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.2 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/googleapis/gnostic v0.4.1 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/hashicorp/golang-lru v0.5.1 // indirect + github.com/imdario/mergo v0.3.5 // indirect; indirectsd + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/json-iterator/go v1.1.10 // indirect + github.com/mailru/easyjson v0.7.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.7.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.10.0 // indirect + github.com/prometheus/procfs v0.2.0 // indirect + github.com/sirupsen/logrus v1.7.0 // indirect + github.com/spf13/cobra v1.1.1 // indirect + go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 // indirect + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.10.0 // indirect + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect + golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect + golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.4 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect + google.golang.org/grpc v1.27.1 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + k8s.io/apiserver v0.21.1 // indirect + k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) + replace ( + // github.com/yunify/qingcloud-sdk-go => ../qingcloud-sdk-go k8s.io/api => k8s.io/api v0.21.1 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.1 k8s.io/apimachinery => k8s.io/apimachinery v0.21.1 diff --git a/go.sum b/go.sum index f206f989..67f98e99 100644 --- a/go.sum +++ b/go.sum @@ -59,7 +59,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -685,8 +684,6 @@ k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkII k8s.io/controller-manager v0.21.1 h1:IFbukN4M0xl3OHEasNQ91h2MLEAMk3uQrBU4+Edka8w= k8s.io/controller-manager v0.21.1/go.mod h1:8ugs8DCcHqybiwdVERhnnyGoS5Ksq/ea1p2B0CosHyc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= diff --git a/pkg/controllers/clusternode/cluster_node_controller.go b/pkg/controllers/clusternode/cluster_node_controller.go index ef2668d2..4a8b129e 100644 --- a/pkg/controllers/clusternode/cluster_node_controller.go +++ b/pkg/controllers/clusternode/cluster_node_controller.go @@ -19,7 +19,7 @@ import ( cloudproviderapp "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/controllers/utils" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud" @@ -167,7 +167,7 @@ func (cnc *ClusterNodeController) processNextWorkItem() bool { return true } -// handleNodesUpdate handle service backend according to node lables +// handleNodesUpdate handle node labels change for service which has node lable annotation func (cnc *ClusterNodeController) handleNodesUpdate(key string) error { startTime := time.Now() defer func() { @@ -197,7 +197,8 @@ func (cnc *ClusterNodeController) handleNodesUpdate(key string) error { _, ok := svc.Annotations[qingcloud.ServiceAnnotationBackendLabel] if ok && svc.Spec.Type == corev1.ServiceTypeLoadBalancer && svc.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeCluster { - klog.Infof("service %s serviceType = %s, externalTrafficPolicy = %s, also has backend label annotation , going to update loadbalancer", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) + klog.Infof("node lable changed and service has backend label annotation, going to update loadbalancer for service %s/%s serviceType = %s, externalTrafficPolicy = %s, ", + svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) // 4. update lb lbInterface, _ := cnc.cloud.LoadBalancer() diff --git a/pkg/controllers/endpoint/endpoint_controller.go b/pkg/controllers/endpoint/endpoint_controller.go index a1b745ea..c5e751b8 100644 --- a/pkg/controllers/endpoint/endpoint_controller.go +++ b/pkg/controllers/endpoint/endpoint_controller.go @@ -20,7 +20,7 @@ import ( cloudproviderapp "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/controllers/utils" ) @@ -160,6 +160,7 @@ func (epc *EndpointController) processNextWorkItem() bool { return true } +// handleEndpointsUpdate handle endpoint change for loadlanbacer service which externalTrafficPolicy is local func (epc *EndpointController) handleEndpointsUpdate(key string) error { startTime := time.Now() defer func() { @@ -182,10 +183,12 @@ func (epc *EndpointController) handleEndpointsUpdate(key string) error { } // ignore service which service type != loadbalancer or externalTrafficPolicy != Local if svc.Spec.Type != corev1.ServiceTypeLoadBalancer || svc.Spec.ExternalTrafficPolicy != corev1.ServiceExternalTrafficPolicyTypeLocal { - klog.V(4).Infof("service %s serviceType = %s, externalTrafficPolicy = %s, skip handle endpoint update", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) + klog.V(4).Infof("skip handle endpoint update for service %s/%s serviceType = %s, externalTrafficPolicy = %s", + svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) return nil } - klog.Infof("service %s serviceType = %s, externalTrafficPolicy = %s, going to handle endpoint update", svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) + klog.Infof("going to handle endpoint update for service %s/%s serviceType = %s, externalTrafficPolicy = %s", + svc.Namespace, svc.Name, svc.Spec.Type, svc.Spec.ExternalTrafficPolicy) // 2. get node list var nodes []*corev1.Node @@ -203,9 +206,9 @@ func (epc *EndpointController) handleEndpointsUpdate(key string) error { cloudLbIntf, _ := epc.cloud.LoadBalancer() err = cloudLbIntf.UpdateLoadBalancer(context.TODO(), "", svc, nodes) if err != nil { - return fmt.Errorf("update loadbalancer for service %s/%s error: %v", svc.Namespace, svc.Name, err) + return fmt.Errorf("update loadbalancer for service %s/%s error: %v", namespace, name, err) } - klog.Infof("update loadbalancer for service %s/%s success", svc.Namespace, svc.Name) + klog.Infof("update loadbalancer for service %s/%s success", namespace, name) return nil } diff --git a/pkg/controllers/service/cloud_service_controller.go b/pkg/controllers/service/cloud_service_controller.go index 636a375c..c38b507d 100644 --- a/pkg/controllers/service/cloud_service_controller.go +++ b/pkg/controllers/service/cloud_service_controller.go @@ -18,7 +18,7 @@ import ( cloudproviderapp "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud" ) @@ -182,10 +182,11 @@ func (sc *ServiceController) processNextWorkItem() bool { return true } +// handleServiceUpdate clean lb listener for service which new svc del the lb annotation func (sc *ServiceController) handleServiceUpdate(svc *corev1.Service) error { startTime := time.Now() defer func() { - klog.V(4).Infof("Finished handleEndpointsUpdate %s/%s (%v)", svc.Namespace, svc.Name, time.Since(startTime)) + klog.V(4).Infof("Finished handleServiceUpdate %s/%s (%v)", svc.Namespace, svc.Name, time.Since(startTime)) }() cloudLbIntf, _ := sc.cloud.LoadBalancer() diff --git a/pkg/controllers/utils/node.go b/pkg/controllers/utils/node.go index ddeb7fa2..803d3e5f 100644 --- a/pkg/controllers/utils/node.go +++ b/pkg/controllers/utils/node.go @@ -2,7 +2,7 @@ package utils import ( corev1 "k8s.io/api/core/v1" - "k8s.io/klog" + "k8s.io/klog/v2" ) // check if the node should be backend or not diff --git a/pkg/executor/backend.go b/pkg/executor/backend.go index cb5e6475..2627a3ca 100644 --- a/pkg/executor/backend.go +++ b/pkg/executor/backend.go @@ -5,7 +5,7 @@ import ( "github.com/davecgh/go-spew/spew" qcservice "github.com/yunify/qingcloud-sdk-go/service" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/errors" @@ -20,13 +20,18 @@ func (q *QingCloudClient) DeleteBackends(ids []*string) error { klog.Warningln("No backends to delete, pls check the inputs") return nil } - output, err := q.LBService.DeleteLoadBalancerBackends(&qcservice.DeleteLoadBalancerBackendsInput{ + input := &qcservice.DeleteLoadBalancerBackendsInput{ LoadBalancerBackends: ids, - }) - if *output.RetCode != 0 { - return errors.NewCommonServerError(ResourceNameBackend, spew.Sdump(ids), "DeleteBackends", *output.Message) } - return err + output, err := q.LBService.DeleteLoadBalancerBackends(input) + if err != nil || *output.RetCode != 0 { + klog.V(4).Infof("failed to delete backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return fmt.Errorf("failed to delete backends,err=%v", err) + } + return fmt.Errorf("failed to delete backends, code=%d, msg=%s", *output.RetCode, *output.Message) + } + return nil } // need update lb @@ -48,8 +53,13 @@ func (q *QingCloudClient) CreateBackends(backends []*apis.LoadBalancerBackend) ( } output, err := q.LBService.AddLoadBalancerBackends(input) + if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to create backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("failed to create backends, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to create backends,err=%v", err) + } + return nil, fmt.Errorf("failed to create backends, code=%d, msg=%s", *output.RetCode, *output.Message) } return output.LoadBalancerBackends, nil diff --git a/pkg/executor/eip.go b/pkg/executor/eip.go index 88ff255c..e5d505b3 100644 --- a/pkg/executor/eip.go +++ b/pkg/executor/eip.go @@ -9,7 +9,7 @@ import ( qcservice "github.com/yunify/qingcloud-sdk-go/service" "github.com/yunify/qingcloud-sdk-go/utils" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" ) @@ -25,7 +25,11 @@ func (q *QingCloudClient) getEIPByID(id string) (*qcservice.EIP, error) { EIPs: []*string{&id}, }) if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to get eip by id %s, err=%s, output=%s", id, spew.Sdump(err), spew.Sdump(output)) + klog.V(4).Infof("failed to get eip by id %s, err=%s, output=%s", id, spew.Sdump(err), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to get eip by id %s, err=%v", id, err) + } + return nil, fmt.Errorf("failed to get eip by id %s, code=%d, msg=%s", id, *output.RetCode, *output.Message) } return output.EIPSet[0], nil @@ -50,7 +54,8 @@ func (q *QingCloudClient) ReleaseEIP(ids []*string) error { //QingCloud Error: Code (1400), Message (PermissionDenied, resource [eip-5ywkioa5] lease info not ready yet, please try later) if (err != nil && !strings.Contains(err.Error(), "QingCloud Error: Code (1400)")) || (output != nil && *output.RetCode != 0 && *output.RetCode != 1400) { - return false, fmt.Errorf("failed to release eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("failed to release eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + return false, fmt.Errorf("failed to release eip, err=%v", err) } else if err == nil && *output.RetCode == 0 { return true, nil } @@ -89,7 +94,11 @@ func (q *QingCloudClient) AllocateEIP(eip *apis.EIP) (*apis.EIP, error) { output, err := q.EIPService.AllocateEIPs(input) if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to allocate eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("failed to allocate eip, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to allocate eip, err=%v", err) + } + return nil, fmt.Errorf("failed to allocate eip, code=%d, msg=%s", *output.RetCode, *output.Message) } id := output.EIPs[0] @@ -120,13 +129,18 @@ func convertEIP(eip *qcservice.EIP) *apis.EIP { } func (q *QingCloudClient) GetAvaliableEIPs() ([]*apis.EIP, error) { - output, err := q.EIPService.DescribeEIPs(&qcservice.DescribeEIPsInput{ + input := &qcservice.DescribeEIPsInput{ Owner: &q.Config.UserID, Status: []*string{qcservice.String(EIPStatusAvailable)}, - }) + } + output, err := q.EIPService.DescribeEIPs(input) if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to get avaliable eips, err=%s, output=%s", spew.Sdump(err), spew.Sdump(output)) + klog.V(4).Infof("failed to get avaliable eips, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to get avaliable eips, err=%v", err) + } + return nil, fmt.Errorf("failed to get avaliable eips, code=%d, msg=%s", *output.RetCode, *output.Message) } result := make([]*apis.EIP, 0) diff --git a/pkg/executor/lb.go b/pkg/executor/lb.go index 78a85ecc..66e11d13 100644 --- a/pkg/executor/lb.go +++ b/pkg/executor/lb.go @@ -8,7 +8,7 @@ import ( qcclient "github.com/yunify/qingcloud-sdk-go/client" qcservice "github.com/yunify/qingcloud-sdk-go/service" "github.com/yunify/qingcloud-sdk-go/utils" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/errors" @@ -101,10 +101,11 @@ func (q *QingCloudClient) GetLoadBalancerByName(name string) (*apis.LoadBalancer } output, err := q.LBService.DescribeLoadBalancers(input) if err != nil && strings.Contains(err.Error(), "QingCloud Error: Code (1300)") { - klog.Warningf("cannot found lb by name, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("cannot found lb by name, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) return nil, errors.NewResourceNotFoundError(ResourceNameLoadBalancer, name) } else if err != nil { - return nil, fmt.Errorf("cannot found lb by name, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("get lb by name error, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + return nil, fmt.Errorf("get lb by name error, err=%s", err) } for _, lb := range output.LoadBalancerSet { if lb.LoadBalancerName != nil && *lb.LoadBalancerName == name { @@ -121,10 +122,11 @@ func (q *QingCloudClient) GetLoadBalancerByID(id string) (*apis.LoadBalancer, er } output, err := q.LBService.DescribeLoadBalancers(input) if err != nil && strings.Contains(err.Error(), "QingCloud Error: Code (1300)") { - klog.Warningf("cannot found lb by id, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("cannot found lb by id, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) return nil, errors.NewResourceNotFoundError(ResourceNameLoadBalancer, id) } else if err != nil { - return nil, fmt.Errorf("cannot found lb by id, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("get lb by id error, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + return nil, fmt.Errorf("get lb by id error, err=%s", err) } if len(output.LoadBalancerSet) > 0 { @@ -155,7 +157,8 @@ func (q *QingCloudClient) fillLBDefaultFileds(input *qcservice.CreateLoadBalance func (q *QingCloudClient) CreateLB(input *apis.LoadBalancer) (*apis.LoadBalancer, error) { if input.Spec.VxNetID == nil && len(input.Spec.EIPs) <= 0 { - return nil, fmt.Errorf("need vxnet or eip, input=%s", spew.Sdump(input)) + klog.V(4).Infof("need vxnet or eip, input=%s", spew.Sdump(input)) + return nil, fmt.Errorf("create lb input invalid, need vxnet or eip") } inputLB := &qcservice.CreateLoadBalancerInput{ @@ -172,7 +175,11 @@ func (q *QingCloudClient) CreateLB(input *apis.LoadBalancer) (*apis.LoadBalancer q.fillLBDefaultFileds(inputLB) output, err := q.LBService.CreateLoadBalancer(inputLB) if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to create lb, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(inputLB), spew.Sdump(output)) + klog.V(4).Infof("failed to create lb, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(inputLB), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to create lb, err=%v", err) + } + return nil, fmt.Errorf("failed to create lb, code=%d, msg=%s", *output.RetCode, *output.Message) } var lbID = *output.LoadBalancerID @@ -200,7 +207,11 @@ func (q *QingCloudClient) ModifyLB(conf *apis.LoadBalancer) error { } output, err := q.LBService.ModifyLoadBalancerAttributes(input) if err != nil || *output.RetCode != 0 { - return fmt.Errorf("failed to modify lb attr, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("failed to modify lb attr, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return fmt.Errorf("failed to modify lb attr, err=%v", err) + } + return fmt.Errorf("failed to modify lb attr, code=%d, msg=%s", *output.RetCode, *output.Message) } //need apply modify @@ -219,11 +230,15 @@ func (q *QingCloudClient) UpdateLB(id *string) error { updateInput := &qcservice.UpdateLoadBalancersInput{ LoadBalancers: []*string{id}, } - updateOutput, err := q.LBService.UpdateLoadBalancers(updateInput) - if err != nil || *updateOutput.RetCode != 0 { - return fmt.Errorf("failed to update lb attr, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(updateInput), spew.Sdump(updateOutput)) + output, err := q.LBService.UpdateLoadBalancers(updateInput) + if err != nil || *output.RetCode != 0 { + klog.V(4).Infof("failed to update lb attr, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(updateInput), spew.Sdump(output)) + if err != nil { + return fmt.Errorf("failed to update lb attr, err=%v", err) + } + return fmt.Errorf("failed to update lb attr, code=%d, msg=%s", *output.RetCode, *output.Message) } - err = qcclient.WaitJob(q.jobService, *updateOutput.JobID, operationWaitTimeout, waitInterval) + err = qcclient.WaitJob(q.jobService, *output.JobID, operationWaitTimeout, waitInterval) if err != nil { return fmt.Errorf("lb %s delete job not completed", *id) } @@ -251,7 +266,8 @@ func (q *QingCloudClient) DeleteLB(id *string) error { } if !strings.Contains(err.Error(), "QingCloud Error: Code (1400)") { - return false, fmt.Errorf("failed to delete lb %s, err=%s, output=%s", *id, spew.Sdump(err), spew.Sdump(output)) + klog.V(4).Infof("failed to delete lb %s, err=%s, output=%s", *id, spew.Sdump(err), spew.Sdump(output)) + return false, fmt.Errorf("failed to delete lb %s, err=%v", *id, err) } return false, nil @@ -285,23 +301,23 @@ func (q *QingCloudClient) AssociateEIPsToLB(id *string, eips []*string) error { if len(eips) == 0 { return nil } - + eipValueSlice := qcservice.StringValueSlice(eips) output, err = q.LBService.AssociateEIPsToLoadBalancer(&qcservice.AssociateEIPsToLoadBalancerInput{ EIPs: eips, LoadBalancer: id, }) if err != nil { - return fmt.Errorf("associate eip %s to lb %s error: %v", spew.Sdump(eips), *id, err) + return fmt.Errorf("associate eip %v to lb %s error: %v", eipValueSlice, *id, err) } if output != nil { if *output.RetCode != 0 { - return fmt.Errorf("associate eip %s to lb %s failed, code=%d, message=%s", spew.Sdump(eips), *id, *output.RetCode, *output.Message) + return fmt.Errorf("associate eip %v to lb %s failed, code=%d, message=%s", eipValueSlice, *id, *output.RetCode, *output.Message) } err = qcclient.WaitJob(q.jobService, *output.JobID, operationWaitTimeout, waitInterval) if err != nil { - return fmt.Errorf("associate eip %s to lb %s job not completed, err %v", spew.Sdump(eips), *id, err) + return fmt.Errorf("associate eip %v to lb %s job not completed, err %v", eipValueSlice, *id, err) } } @@ -314,23 +330,23 @@ func (q *QingCloudClient) DissociateEIPsFromLB(id *string, eips []*string) error if len(eips) == 0 { return nil } - + eipValueSlice := qcservice.StringValueSlice(eips) output, err = q.LBService.DissociateEIPsFromLoadBalancer(&qcservice.DissociateEIPsFromLoadBalancerInput{ EIPs: eips, LoadBalancer: id, }) if err != nil { - return fmt.Errorf("dissociate eips %s from lb %s error: %v", spew.Sdump(eips), *id, err) + return fmt.Errorf("dissociate eips %v from lb %s error: %v", eipValueSlice, *id, err) } if output != nil { if *output.RetCode != 0 { - return fmt.Errorf("dissociate eip %s from lb %s failed, code=%d, message=%s", spew.Sdump(eips), *id, *output.RetCode, *output.Message) + return fmt.Errorf("dissociate eip %v from lb %s failed, code=%d, message=%s", eipValueSlice, *id, *output.RetCode, *output.Message) } err = qcclient.WaitJob(q.jobService, *output.JobID, operationWaitTimeout, waitInterval) if err != nil { - return fmt.Errorf("dissociate eip %s from lb %s job not completed, err %v", spew.Sdump(eips), *id, err) + return fmt.Errorf("dissociate eip %v from lb %s job not completed, err %v", eipValueSlice, *id, err) } } diff --git a/pkg/executor/listener.go b/pkg/executor/listener.go index fd845be4..deec7bb7 100644 --- a/pkg/executor/listener.go +++ b/pkg/executor/listener.go @@ -3,7 +3,6 @@ package executor import ( "fmt" - "github.com/davecgh/go-spew/spew" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" qcservice "github.com/yunify/qingcloud-sdk-go/service" ) @@ -26,10 +25,10 @@ func (q *QingCloudClient) DeleteListener(lsnid []*string) error { LoadBalancerListeners: lsnid, }) if err != nil { - return fmt.Errorf("failed to delete listener %v err=%v", spew.Sdump(lsnid), err) + return fmt.Errorf("failed to delete listener %v, err=%v", qcservice.StringValueSlice(lsnid), err) } if *output.RetCode != 0 { - return fmt.Errorf("failed to delete listener %v output=%v", spew.Sdump(lsnid), spew.Sdump(output)) + return fmt.Errorf("failed to delete listener %v, code=%d, msg=%s", qcservice.StringValueSlice(lsnid), *output.RetCode, *output.Message) } return nil } @@ -89,8 +88,11 @@ func (q *QingCloudClient) CreateListener(inputs []*apis.LoadBalancerListener) ([ Listeners: convertFromLoadBalancerListener(inputs), LoadBalancer: id, }) - if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("failed to create listerner, err=%+v, input=%s, output=%s", spew.Sdump(err), spew.Sdump(inputs), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("failed to create listener, err=%v", err) + } + if *output.RetCode != 0 { + return nil, fmt.Errorf("failed to create listener, code=%d, msg=%s", *output.RetCode, *output.Message) } return q.GetListeners(output.LoadBalancerListeners) diff --git a/pkg/executor/sg.go b/pkg/executor/sg.go index baee71c6..f80cc15c 100644 --- a/pkg/executor/sg.go +++ b/pkg/executor/sg.go @@ -6,7 +6,7 @@ import ( "github.com/davecgh/go-spew/spew" qcclient "github.com/yunify/qingcloud-sdk-go/client" qcservice "github.com/yunify/qingcloud-sdk-go/service" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/errors" @@ -62,14 +62,18 @@ func (q *QingCloudClient) CreateSecurityGroup(input *apis.SecurityGroup) (*apis. } createOutput, err := q.securityGroupService.CreateSecurityGroup(createInput) if err != nil || *createOutput.RetCode != 0 { - return nil, fmt.Errorf("failed to create sg, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(createInput), spew.Sdump(createOutput)) + klog.V(4).Infof("failed to create sg, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(createInput), spew.Sdump(createOutput)) + if err != nil { + return nil, fmt.Errorf("failed to create sg, err=%v", err) + } + return nil, fmt.Errorf("failed to create sg, code=%d, msg=%s", *createOutput.RetCode, *createOutput.Message) } input.Status.SecurityGroupID = createOutput.SecurityGroupID err = q.attachTagsToResources([]*string{createOutput.SecurityGroupID}, SGTagResourceType) if err != nil { - klog.Errorf("Failed to attach tag to security group %s, err: %s", spew.Sdump(input), err.Error()) + klog.Errorf("Failed to attach tag to security group %s, err: %v", *createOutput.SecurityGroupID, err) } return input, nil @@ -83,7 +87,11 @@ func (q *QingCloudClient) addSecurityGroupRules(sg *apis.SecurityGroup, rules [] } addRuleOutput, err := q.securityGroupService.AddSecurityGroupRules(addRuleInput) if err != nil || *addRuleOutput.RetCode != 0 { - return nil, fmt.Errorf("failed to add sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(addRuleInput), spew.Sdump(addRuleOutput)) + klog.V(4).Infof("failed to add sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(addRuleInput), spew.Sdump(addRuleOutput)) + if err != nil { + return nil, fmt.Errorf("failed to add sg rules, err=%v", err) + } + return nil, fmt.Errorf("failed to add sg rules, code=%d, msg=%s", *addRuleOutput.RetCode, *addRuleOutput.Message) } sg.Status.SecurityGroupRuleIDs = addRuleOutput.SecurityGroupRules @@ -95,12 +103,17 @@ func (q *QingCloudClient) addSecurityGroupRules(sg *apis.SecurityGroup, rules [] } applyRuleOutput, err := q.securityGroupService.ApplySecurityGroup(applyRuleInput) if err != nil || *applyRuleOutput.RetCode != 0 { - return nil, fmt.Errorf("failed to apply sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(applyRuleInput), spew.Sdump(applyRuleOutput)) + klog.V(4).Infof("failed to apply sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(applyRuleInput), spew.Sdump(applyRuleOutput)) + if err != nil { + return nil, fmt.Errorf("failed to apply sg rules, err=%v", err) + } + return nil, fmt.Errorf("failed to apply sg rules, code=%d, msg=%s", *applyRuleOutput.RetCode, *applyRuleOutput.Message) } err = qcclient.WaitJob(q.jobService, *applyRuleOutput.JobID, operationWaitTimeout, sgWaitInterval) if err != nil { - return nil, fmt.Errorf("failed to apply sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(applyRuleInput), spew.Sdump(applyRuleOutput)) + klog.V(4).Infof("failed to apply sg rules, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(applyRuleInput), spew.Sdump(applyRuleOutput)) + return nil, fmt.Errorf("failed to apply sg rules, err=%v", err) } return sg, nil @@ -112,7 +125,11 @@ func (q *QingCloudClient) DeleteSG(sg *string) error { } output, err := q.securityGroupService.DeleteSecurityGroups(input) if err != nil || *output.RetCode != 0 { - return fmt.Errorf("failed to delete sg %s, err=%s, output=%s", *sg, spew.Sdump(err), spew.Sdump(output)) + klog.V(4).Infof("failed to delete sg %s, err=%s, output=%s", *sg, spew.Sdump(err), spew.Sdump(output)) + if err != nil { + return fmt.Errorf("failed to delete sg %s, err=%v", *sg, err) + } + return fmt.Errorf("failed to delete sg %s, code=%d, msg=%s", *sg, *output.RetCode, *output.Message) } return nil @@ -151,7 +168,11 @@ func (q *QingCloudClient) GetSecurityGroupByName(name string) (*apis.SecurityGro } output, err := q.securityGroupService.DescribeSecurityGroups(input) if err != nil || *output.RetCode != 0 { - return nil, fmt.Errorf("cannot get sg by name, err=%s, input=%s, output=%s", spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + klog.V(4).Infof("get sg by name %s failed, err=%s, input=%s, output=%s", name, spew.Sdump(err), spew.Sdump(input), spew.Sdump(output)) + if err != nil { + return nil, fmt.Errorf("get sg by name %s failed, err=%v", name, err) + } + return nil, fmt.Errorf("get sg by name %s failed, code=%d, msg=%s", name, *output.RetCode, *output.Message) } if len(output.SecurityGroupSet) > 1 { diff --git a/pkg/executor/tags.go b/pkg/executor/tags.go index 3b12a177..2ec6fd9c 100644 --- a/pkg/executor/tags.go +++ b/pkg/executor/tags.go @@ -26,10 +26,10 @@ func (i *QingCloudClient) attachTagsToResources(resourceIDs []*string, resourceT } output, err := i.tagService.AttachTags(input) if err != nil { - return errors.NewCommonServerError("tag", fmt.Sprintf("%v", resourceIDs), "attachTagsToResources", err.Error()) + return errors.NewCommonServerError("tag", fmt.Sprintf("%v", service.StringValueSlice(resourceIDs)), "attachTagsToResources", err.Error()) } if *output.RetCode != 0 { - return errors.NewCommonServerError("tag", fmt.Sprintf("%v", resourceIDs), "attachTagsToResources", *output.Message) + return errors.NewCommonServerError("tag", fmt.Sprintf("%v", service.StringValueSlice(resourceIDs)), "attachTagsToResources", *output.Message) } return nil } diff --git a/pkg/qingcloud/annotations.go b/pkg/qingcloud/annotations.go index c58f2b1a..2b73898c 100644 --- a/pkg/qingcloud/annotations.go +++ b/pkg/qingcloud/annotations.go @@ -201,7 +201,7 @@ func (qc *QingCloud) ParseServiceLBConfig(cluster string, service *v1.Service) ( } if timeoutConf, ok := annotation[ServiceAnnotationListenerTimeout]; ok { if err := validListenerTimeout(timeoutConf); err != nil { - return nil, err + return nil, fmt.Errorf("valid listener timeout error: %v", err) } config.Timeout = &timeoutConf } diff --git a/pkg/qingcloud/loadbalancer_utils.go b/pkg/qingcloud/loadbalancer_utils.go index 95942dea..d5472e64 100644 --- a/pkg/qingcloud/loadbalancer_utils.go +++ b/pkg/qingcloud/loadbalancer_utils.go @@ -586,7 +586,7 @@ func getRandomNodes(nodes []*v1.Node, count int) (result []*v1.Node) { return } -func getBackendCount(nodes []*v1.Node) (backendCountResult int) { +func getDefaultBackendCount(nodes []*v1.Node) (backendCountResult int) { if len(nodes) > 3 { backendCountResult = len(nodes) / 3 if backendCountResult < 3 { diff --git a/pkg/qingcloud/qingcloud.go b/pkg/qingcloud/qingcloud.go index ed8d123a..9b9b17dd 100644 --- a/pkg/qingcloud/qingcloud.go +++ b/pkg/qingcloud/qingcloud.go @@ -20,11 +20,12 @@ import ( corev1informer "k8s.io/client-go/informers/core/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" cloudprovider "k8s.io/cloud-provider" - "k8s.io/klog" + klog "k8s.io/klog/v2" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/errors" "github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor" + "github.com/yunify/qingcloud-cloud-controller-manager/pkg/util" ) const ( @@ -194,30 +195,46 @@ func (qc *QingCloud) getLoadBalancer(service *v1.Service) (*LoadBalancerConfig, // parameters as read-only and not modify them. // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { + var err error + klog.Infof("====== EnsureLoadBalancer start handle service %s/%s ======", service.Namespace, service.Name) + defer func() { + if err != nil { + klog.Errorf("EnsureLoadBalancer handle service %s/%s error: %v", service.Namespace, service.Name, err) + } + klog.Infof("====== EnsureLoadBalancer finished handle service %s/%s ======", service.Namespace, service.Name) + }() + lb, err := qc.ensureLoadBalancer(ctx, "", service, nodes) + return lb, err +} +func (qc *QingCloud) ensureLoadBalancer(ctx context.Context, _ string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { conf, lb, err := qc.getLoadBalancer(service) - klog.V(4).Infof("==== EnsureLoadBalancer %s config %s ====", spew.Sdump(lb), spew.Sdump(conf)) - if err != nil && !errors.IsResourceNotFound(err) { - klog.Errorf("getLoadBalancer for service %s error: %v", service.Name, err) - return nil, err + klog.V(4).Infof("EnsureLoadBalancer lb %s config %s", spew.Sdump(lb), spew.Sdump(conf)) + if err != nil { + if errors.IsResourceNotFound(err) && conf.Policy != ReuseExistingLB && conf.Policy != Shared { + // will auto create lb with assigned eip + klog.Infof("lb not found for service %s/%s, going to create lb with assigned eip ", service.Namespace, service.Name) + } else { + return nil, fmt.Errorf("getLoadBalancer error: %v", err) + } } + var lbID string // filter nodes by service externalTrafficPolicy nodes, e := qc.filterNodes(ctx, service, nodes, conf) if e != nil { - klog.Errorf("filterNodes for service %s/%s with externalTrafficPolicy %s error: %v", service.Namespace, service.Name, service.Spec.ExternalTrafficPolicy, e) - return nil, e + return nil, fmt.Errorf("filterNodes with externalTrafficPolicy %s error: %v", service.Spec.ExternalTrafficPolicy, e) } //1. ensure & update lb if err == nil { //The configuration of the load balancer will be independent, so we'll just create //it for now, not update it. - + lbID = *lb.Status.LoadBalancerID //need modify attribute modify := false if result := needUpdateAttr(conf, lb); result != nil { if err = qc.Client.ModifyLB(result); err != nil { - return nil, err + return nil, fmt.Errorf("modify lb %s error: %v", lbID, err) } modify = true } @@ -225,99 +242,95 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service * // update eips err = qc.updateLBEip(conf, lb) if err != nil { - klog.Errorf("update eip for lb %s error %v", *lb.Status.LoadBalancerID, err) - return nil, err + return nil, fmt.Errorf("update eip for lb %s error %v", lbID, err) } //update listener listenerIDs := filterListeners(lb.Status.LoadBalancerListeners, conf.listenerName) - klog.Infof("The loadbalancer %s has the following listeners %s", *lb.Status.LoadBalancerID, spew.Sdump(listenerIDs)) + if len(listenerIDs) <= 0 { - klog.Infof("creating listeners for loadbalancers %s, service ports %s", *lb.Status.LoadBalancerID, spew.Sdump(service.Spec.Ports)) + klog.Infof("listener is not exists for service %s/%s, creating listeners for lb %s with service ports %v", service.Namespace, service.Name, lbID, util.GetPortsSlice(service.Spec.Ports)) if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes, service); err != nil { - klog.Errorf("createListenersAndBackends for loadbalancer %s error: %v", *lb.Status.LoadBalancerID, err) - return nil, err + return nil, fmt.Errorf("createListenersAndBackends for lb %s error: %v", lbID, err) } modify = true } else { + klog.Infof("the lb %s has the following listeners %s for service %s/%s", lbID, util.CoverPointSliceToStr(listenerIDs), service.Namespace, service.Name) listeners, err := qc.Client.GetListeners(listenerIDs) if err != nil { - return nil, err + return nil, fmt.Errorf("get listeners %v error: %v", util.CoverPointSliceToStr(listenerIDs), err) } //update listerner toDelete, toAdd, toKeep := diffListeners(listeners, conf, service.Spec.Ports) if len(toDelete) > 0 { - klog.Infof("listeners %s will be deleted for lb %s", spew.Sdump(toDelete), *lb.Status.LoadBalancerID) + klog.Infof("listeners %s will be deleted for lb %s ", util.CoverPointSliceToStr(toDelete), lbID) err = qc.Client.DeleteListener(toDelete) if err != nil { - return nil, err + return nil, fmt.Errorf("delete listeners %v error: %v", util.CoverPointSliceToStr(toDelete), err) } modify = true } if len(toAdd) > 0 { - klog.Infof("listeners %s will be added for lb %s", spew.Sdump(toAdd), *lb.Status.LoadBalancerID) + klog.Infof("listeners on ports %v will be added for lb %s", util.GetPortsSlice(toAdd), lbID) err = qc.createListenersAndBackends(conf, lb, toAdd, nodes, service) if err != nil { - return nil, err + return nil, fmt.Errorf("createListenersAndBackends error: %v", err) } modify = true } //update backend; for example, service annotation for backend label changed for _, listener := range toKeep { - // toDelete, toAdd := diffBackend(listener, nodes) + listenerID := *listener.Spec.LoadBalancerListenerID + listenerName := *listener.Spec.LoadBalancerListenerName toDelete, toAdd := qc.diffBackend(listener, nodes, conf, service) if len(toDelete) > 0 { - klog.Infof("backends %s will be deleted for listener %s(%s) of lb %s", - spew.Sdump(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID) + klog.Infof("%d backends will be deleted for listener %s(id:%s,port:%d) of lb %s", len(toDelete), listenerName, listenerID, *listener.Spec.ListenerPort, lbID) + klog.V(4).Infof("backends(count %d) %v will be deleted for listener %s(id:%s,port:%d) of lb %s", len(toDelete), util.CoverPointSliceToStr(toDelete), listenerName, listenerID, *listener.Spec.ListenerPort, lbID) err = qc.Client.DeleteBackends(toDelete) if err != nil { - return nil, err + return nil, fmt.Errorf("delete backends %v error: %v", util.CoverPointSliceToStr(toDelete), err) } modify = true } - toAddBackends := generateLoadBalancerBackends(toAdd, listener, service.Spec.Ports) - if len(toAddBackends) > 0 { - klog.Infof("backends %s will be added for listener %s(%s) of lb %s", - spew.Sdump(toAddBackends), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID) + if len(toAdd) > 0 { + toAddBackends := generateLoadBalancerBackends(toAdd, listener, service.Spec.Ports) + klog.Infof("%d backends will be added for listener %s(id:%s,port:%d) of lb %s", len(toAddBackends), listenerName, listenerID, *listener.Spec.ListenerPort, lbID) + klog.V(4).Infof("backends(count %d) %v will be added for listener %s(id:%s,port:%d) of lb %s", len(toAdd), util.GetNodesName(toAdd), listenerName, listenerID, *listener.Spec.ListenerPort, lbID) + _, err = qc.Client.CreateBackends(toAddBackends) if err != nil { - return nil, err + return nil, fmt.Errorf("add backends error: %v", err) } modify = true } + } if !modify { - klog.Infof("no change for listeners %v of lb %s, skip UpdateLB", spew.Sdump(listenerIDs), *lb.Status.LoadBalancerID) + klog.Infof("no change for listeners %v of service %s/%s, skip UpdateLB %s", util.CoverPointSliceToStr(listenerIDs), service.Namespace, service.Name, lbID) return convertLoadBalancerStatus(&lb.Status), nil } } } else if errors.IsResourceNotFound(err) { - if conf.Policy == ReuseExistingLB || conf.Policy == Shared { - return nil, err - } else { - err = nil - } - //1. create lb //1.1 prepare eip if len(conf.EipIDs) <= 0 && conf.EipSource != nil { eip, err := qc.prepareEip(conf.EipSource) if err != nil { - return nil, err + return nil, fmt.Errorf("prepare eip error: %v", err) } conf.EipIDs = []*string{eip.Status.EIPID} } //1.2 prepare sg //default sg set by Client auto //1.3 create lb - klog.Infof("creating loadbalance for service %s/%s", service.Namespace, service.Name) + klog.Infof("creating lb for service %s/%s", service.Namespace, service.Name) lb, err = qc.Client.CreateLB(&apis.LoadBalancer{ Spec: apis.LoadBalancerSpec{ LoadBalancerName: &conf.LoadBalancerName, @@ -329,13 +342,14 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service * }, }) if err != nil { - return nil, fmt.Errorf("create loadbalance for service %s/%s error: %v", service.Namespace, service.Name, err) + return nil, fmt.Errorf("create lb error: %v", err) } + lbID = *lb.Status.LoadBalancerID //create listener - klog.Infof("creating listener %s for loadbalance %s", conf.listenerName, *lb.Status.LoadBalancerID) + klog.Infof("creating listener %s on ports %v for lb %s", conf.listenerName, util.GetPortsSlice(service.Spec.Ports), lbID) if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes, service); err != nil { - return nil, fmt.Errorf("create listener %s for loadbalance %s error: %v", conf.listenerName, *lb.Status.LoadBalancerID, err) + return nil, fmt.Errorf("create listener %s on ports %v for lb %s error: %v", conf.listenerName, util.GetPortsSlice(service.Spec.Ports), lbID, err) } } else { return nil, err @@ -347,8 +361,9 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service * err = qc.Client.UpdateLB(lb.Status.LoadBalancerID) if err != nil { - return nil, err + return nil, fmt.Errorf("update lb %s error: %v", lbID, err) } + klog.Infof("update lb %s success for service %s/%s", lbID, service.Namespace, service.Name) return convertLoadBalancerStatus(&lb.Status), nil } @@ -358,30 +373,43 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service * // parameters as read-only and not modify them. // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service *v1.Service, nodes []*v1.Node) error { + var err error + klog.Infof("====== UpdateLoadBalancer start handle service %s/%s ======", service.Namespace, service.Name) + defer func() { + if err != nil { + klog.Errorf("UpdateLoadBalancer handle service %s/%s error: %v", service.Namespace, service.Name, err) + } + klog.Infof("====== UpdateLoadBalancer finished handle service %s/%s ======", service.Namespace, service.Name) + }() + err = qc.updateLoadBalancer(ctx, "", service, nodes) + return err +} +func (qc *QingCloud) updateLoadBalancer(ctx context.Context, _ string, service *v1.Service, nodes []*v1.Node) error { + var modify bool conf, lb, err := qc.getLoadBalancer(service) - klog.V(4).Infof("==== UpdateLoadBalancer %s config %s ====", spew.Sdump(lb), spew.Sdump(conf)) + klog.V(4).Infof("UpdateLoadBalancer lb %s config %s", spew.Sdump(lb), spew.Sdump(conf)) if err != nil { - klog.Errorf("getLoadBalancer for service %s error: %v", service.Name, err) - return err + return fmt.Errorf("getLoadBalancer error: %v", err) } + lbID := *lb.Status.LoadBalancerID nodes, err = qc.filterNodes(ctx, service, nodes, conf) if err != nil { - klog.Errorf("filterNodes for service %s with externalTrafficPolicy %s error: %v", service.Name, service.Spec.ExternalTrafficPolicy, err) - return err + return fmt.Errorf("filterNodes with externalTrafficPolicy %s error: %v", service.Spec.ExternalTrafficPolicy, err) } //update backend listenerIDs := filterListeners(lb.Status.LoadBalancerListeners, conf.listenerName) if len(listenerIDs) <= 0 { + klog.Infof("listener is not exists for service %s/%s, skip update lb %s", service.Namespace, service.Name, lbID) return nil } listeners, err := qc.Client.GetListeners(listenerIDs) if err != nil { - return err + return fmt.Errorf("get listeners %v error: %v", util.CoverPointSliceToStr(listenerIDs), err) } for _, listener := range listeners { @@ -389,57 +417,60 @@ func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service * toDelete, toAdd := qc.diffBackend(listener, nodes, conf, service) if len(toDelete) > 0 { - klog.Infof("backends %s will be deleted for listener %s(%s) of lb %s", - spew.Sdump(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID) + klog.Infof("%d backends will be deleted for listener %s(id:%s,port:%d) of lb %s", len(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *listener.Spec.ListenerPort, lbID) + klog.V(4).Infof("backends(count %d) %v will be deleted for listener %s(id:%s,port:%d) of lb %s", len(toDelete), util.CoverPointSliceToStr(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *listener.Spec.ListenerPort, lbID) err = qc.Client.DeleteBackends(toDelete) if err != nil { - return err + return fmt.Errorf("delete backends %v error: %v", util.CoverPointSliceToStr(toDelete), err) } modify = true } toAddBackends := generateLoadBalancerBackends(toAdd, listener, service.Spec.Ports) if len(toAddBackends) > 0 { - klog.Infof("backends %s will be added for listener %s(%s) of lb %s", - spew.Sdump(toAddBackends), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID) + klog.Infof("%d backends will be added for listener %s(id:%s,port:%d) of lb %s", len(toAdd), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *listener.Spec.ListenerPort, lbID) + klog.V(4).Infof("backends(count %d) %s will be added for listener %s(id:%s,port:%d) of lb %s", len(toAdd), util.GetNodesName(toAdd), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *listener.Spec.ListenerPort, lbID) _, err = qc.Client.CreateBackends(toAddBackends) if err != nil { - return err + return fmt.Errorf("add backends error: %v", err) } modify = true } } if !modify { - klog.Infof("no backend change for listeners %v of lb %s, skip UpdateLB", spew.Sdump(listenerIDs), *lb.Status.LoadBalancerID) + klog.Infof("no backend change for listeners %v of service %s/%s, skip UpdateLB %s", util.CoverPointSliceToStr(listenerIDs), service.Namespace, service.Name, lbID) return nil } - return qc.Client.UpdateLB(lb.Status.LoadBalancerID) + err = qc.Client.UpdateLB(lb.Status.LoadBalancerID) + if err != nil { + return fmt.Errorf("update lb %s error: %v", lbID, err) + } + klog.Infof("update lb %s success for service %s/%s", lbID, service.Namespace, service.Name) + return nil } func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status *apis.LoadBalancer, ports []v1.ServicePort, nodes []*v1.Node, svc *v1.Service) error { listeners, err := generateLoadBalancerListeners(conf, status, ports) if err != nil { - klog.Errorf("generateLoadBalancerListeners for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) - return err + return fmt.Errorf("generateLoadBalancerListeners for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) } listeners, err = qc.Client.CreateListener(listeners) if err != nil { - klog.Errorf("CreateListener for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) - return err + return fmt.Errorf("CreateListener for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) } // filter backend nodes by count config if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster && conf.BackendCountResult != 0 { - klog.Infof("try to get %d random nodes as backend for service %s/%s", conf.BackendCountResult, svc.Namespace, svc.Name) + klog.Infof("try to get %d random nodes as backend for service %s/%s on ports %v", conf.BackendCountResult, svc.Namespace, svc.Name, util.GetPortsSlice(ports)) nodes = getRandomNodes(nodes, conf.BackendCountResult) var resultNames []string for _, node := range nodes { resultNames = append(resultNames, node.Name) } - klog.Infof("get random nodes result for service %s/%s: %v", svc.Namespace, svc.Name, resultNames) + klog.V(4).Infof("get random nodes result for service %s/%s: %v", svc.Namespace, svc.Name, resultNames) } // create backend @@ -447,8 +478,7 @@ func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status backends := generateLoadBalancerBackends(nodes, listener, ports) _, err = qc.Client.CreateBackends(backends) if err != nil { - klog.Errorf("CreateBackends for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) - return err + return fmt.Errorf("CreateBackends for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err) } } @@ -465,46 +495,74 @@ func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status // Implementations must treat the *v1.Service parameter as read-only and not modify it. // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager func (qc *QingCloud) EnsureLoadBalancerDeleted(ctx context.Context, _ string, service *v1.Service) error { + var err error + klog.Infof("====== EnsureLoadBalancerDeleted start handle service %s/%s ======", service.Namespace, service.Name) + defer func() { + if err != nil { + klog.Errorf("EnsureLoadBalancerDeleted handle service %s/%s error: %v", service.Namespace, service.Name, err) + } + klog.Infof("====== EnsureLoadBalancerDeleted finished handle service %s/%s ======", service.Namespace, service.Name) + }() + + err = qc.ensureLoadBalancerDeleted(ctx, "", service) + return err +} +func (qc *QingCloud) ensureLoadBalancerDeleted(ctx context.Context, _ string, service *v1.Service) error { lbConfig, lb, err := qc.getLoadBalancerBeforeDelete(service) - klog.V(4).Infof("==== EnsureLoadBalancerDeleted %s config %s ====", spew.Sdump(lb), spew.Sdump(lbConfig)) - if errors.IsResourceNotFound(err) || lb == nil { - return nil + klog.V(4).Infof("EnsureLoadBalancerDeleted lb %s config %s", spew.Sdump(lb), spew.Sdump(lbConfig)) + if err != nil { + if errors.IsResourceNotFound(err) { + klog.Infof("not found lb for service %s/%s, skip delete lb", service.Namespace, service.Name) + return nil + } else { + return fmt.Errorf("get lb error: %v", err) + } } - if err != nil { - return err + if lb == nil { + klog.Infof("there is no lb or eip annotation of service %s/%s, skip delete lb", service.Namespace, service.Name) + return nil } + lbID := *lb.Status.LoadBalancerID listeners := filterListeners(lb.Status.LoadBalancerListeners, lbConfig.listenerName) + //reuse lb or auto create lb but has other listeners, only delete listener for this service if lbConfig.ReuseLBID != "" || len(lb.Status.LoadBalancerListeners)-len(listeners) > 0 { - klog.Infof("service %s/%s reuse lb or lb %s has other listeners, try to delete listener %s", - service.Namespace, service.Name, *lb.Status.LoadBalancerID, lbConfig.listenerName) - + klog.Infof("service %s/%s reuse lb %s or lb has other listeners, try to delete listener %s", service.Namespace, service.Name, lbID, lbConfig.listenerName) if len(listeners) <= 0 { + klog.Infof("listener is not exists for service %s/%s on lb %s, skip delete listener", service.Namespace, service.Name, lbID) return nil } + err = qc.Client.DeleteListener(listeners) if err != nil { - return err + return fmt.Errorf("delete listener %v error: %v", util.CoverPointSliceToStr(listeners), err) } - return qc.Client.UpdateLB(lb.Status.LoadBalancerID) - } + klog.Infof("delete listener %v success for service %s/%s ", util.CoverPointSliceToStr(listeners), service.Namespace, service.Name) - //delete lb - if len(lb.Status.LoadBalancerListeners)-len(listeners) > 0 { + err = qc.Client.UpdateLB(lb.Status.LoadBalancerID) + if err != nil { + return fmt.Errorf("update lb %s error: %v", lbID, err) + } + klog.Infof("update lb %s success for service %s/%s", lbID, service.Namespace, service.Name) return nil } + + //delete lb err = qc.Client.DeleteLB(lb.Status.LoadBalancerID) if err != nil { - return err + return fmt.Errorf("delete lb %s error: %v", *lb.Status.LoadBalancerID, err) + } + klog.Infof("delete lb %s success for service %s/%s", lbID, service.Namespace, service.Name) //delete eip if len(lb.Status.CreatedEIPs) > 0 { err = qc.Client.ReleaseEIP(lb.Status.CreatedEIPs) if err != nil { - return err + return fmt.Errorf("release eip %v error: %v", util.CoverPointSliceToStr(lb.Status.CreatedEIPs), err) } + klog.Infof("delete eips %v success for service %s/%s", util.CoverPointSliceToStr(lb.Status.CreatedEIPs), service.Namespace, service.Name) } //delete sg @@ -563,25 +621,28 @@ func (qc *QingCloud) filterNodes(ctx context.Context, svc *v1.Service, nodes []* } // if there are no available nodes , use default backend count value if len(newNodes) == 0 { - klog.Infof("there are no available nodes filter by label %s for service %s/%s, use default backend count!", - lbconfog.BackendLabel, svc.Namespace, svc.Name) - backendCountResult = getBackendCount(nodes) - newNodes = nodes + backendCountResult = getDefaultBackendCount(nodes) + klog.Infof("there are no available nodes filter by label %s for service %s/%s, use default backend count: %d", + lbconfog.BackendLabel, svc.Namespace, svc.Name, backendCountResult) } } else if lbconfog.BackendCountConfig != "" { //filter by backend count config + klog.Infof("filter nodes for service %s/%s by backend count config: %s", svc.Namespace, svc.Name, lbconfog.BackendCountConfig) + backendCountConfig, _ := strconv.Atoi(lbconfog.BackendCountConfig) if backendCountConfig > 0 && backendCountConfig <= len(nodes) { backendCountResult = backendCountConfig } else { - klog.Infof("invalid backend count config %d for service %s/%s, use default backend count!", - backendCountConfig, svc.Namespace, svc.Name) - backendCountResult = getBackendCount(nodes) - + backendCountResult = getDefaultBackendCount(nodes) + klog.Infof("invalid backend count config %s for service %s/%s, use default backend count: %d", + lbconfog.BackendCountConfig, svc.Namespace, svc.Name, backendCountResult) } - newNodes = nodes } else { - // no need to filter + // no need to filter or for other filter policy in the future + } + + if len(newNodes) == 0 { + //no need to filter, use all nodes in params newNodes = nodes } lbconfog.BackendCountResult = backendCountResult @@ -591,7 +652,7 @@ func (qc *QingCloud) filterNodes(ctx context.Context, svc *v1.Service, nodes []* for _, node := range newNodes { resultNames = append(resultNames, node.Name) } - klog.Infof("filter node result for service %s/%s: %v", svc.Namespace, svc.Name, resultNames) + klog.V(4).Infof("filter node result for service %s/%s: %v", svc.Namespace, svc.Name, resultNames) return newNodes, nil } diff --git a/pkg/qingcloud/qingcloud_utils.go b/pkg/qingcloud/qingcloud_utils.go index 403f7f56..efe9cd15 100644 --- a/pkg/qingcloud/qingcloud_utils.go +++ b/pkg/qingcloud/qingcloud_utils.go @@ -3,7 +3,6 @@ package qingcloud import ( "fmt" - "github.com/davecgh/go-spew/spew" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -84,7 +83,7 @@ func (qc *QingCloud) diffLBEip(config *LoadBalancerConfig, lb *apis.LoadBalancer case AllocateOnly, UseAvailableOnly, UseAvailableOrAllocateOne: if len(lb.Spec.EIPs) > 0 { // lb already has eip, do nothing - klog.Infof("lb %s already has eip %s, do nothing", *lb.Status.LoadBalancerID, spew.Sdump(lb.Spec.EIPs)) + klog.Infof("lb %s already has eip %v, do nothing", *lb.Status.LoadBalancerID, util.CoverPointSliceToStr(lb.Spec.EIPs)) } else { // get or create an available eip from qingcloud and associate this eip to lb eip, err := qc.prepareEip(config.EipSource) @@ -120,7 +119,7 @@ func (qc *QingCloud) updateLBEip(config *LoadBalancerConfig, lb *apis.LoadBalanc // update lb eip if len(eipsToAdd) > 0 { - klog.Infof("associating eips %s to lb %s", spew.Sdump(eipsToAdd), *lb.Status.LoadBalancerID) + klog.Infof("associating eips %v to lb %s", util.CoverPointSliceToStr(eipsToAdd), *lb.Status.LoadBalancerID) err = qc.Client.AssociateEIPsToLB(lb.Status.LoadBalancerID, eipsToAdd) if err != nil { return err @@ -128,7 +127,7 @@ func (qc *QingCloud) updateLBEip(config *LoadBalancerConfig, lb *apis.LoadBalanc updated = true } if len(eipsToDel) > 0 { - klog.Infof("dissociating eips %s from lb %s", spew.Sdump(eipsToDel), *lb.Status.LoadBalancerID) + klog.Infof("dissociating eips %v from lb %s", util.CoverPointSliceToStr(eipsToDel), *lb.Status.LoadBalancerID) err = qc.Client.DissociateEIPsFromLB(lb.Status.LoadBalancerID, eipsToDel) if err != nil { return err @@ -150,6 +149,8 @@ func (qc *QingCloud) updateLBEip(config *LoadBalancerConfig, lb *apis.LoadBalanc func (qc *QingCloud) diffBackend(listener *apis.LoadBalancerListener, nodes []*v1.Node, conf *LoadBalancerConfig, svc *v1.Service) (toDelete []*string, toAdd []*v1.Node) { var backendLeftID []*string + + // deleted cluster node sync to lb backend for _, backend := range listener.Status.LoadBalancerBackends { if !nodesHasBackend(*backend.Spec.LoadBalancerBackendName, nodes) { toDelete = append(toDelete, backend.Status.LoadBalancerBackendID) @@ -175,6 +176,7 @@ func (qc *QingCloud) diffBackend(listener *apis.LoadBalancerListener, nodes []*v toAdd = append(toAdd, getRandomNodes(nodeLeft, conf.BackendCountResult-backendLeftCount)...) } } else { + // added cluster node sync to lb backend for _, node := range nodes { if !backendsHasNode(node, listener.Status.LoadBalancerBackends) { toAdd = append(toAdd, node) diff --git a/pkg/util/util.go b/pkg/util/util.go index 9e930c3e..09341b3a 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -19,8 +19,8 @@ const ( pageLimt = 100 ) -// GetPortsOfService return service ports and nodeports -func GetPortsOfService(service *v1.Service) ([]int, []int) { +// GetSvcPortsAndNodePorts return service ports and nodeports +func GetSvcPortsAndNodePorts(service *v1.Service) ([]int, []int) { k8sPorts := []int{} k8sNodePorts := []int{} for _, port := range service.Spec.Ports { @@ -30,6 +30,22 @@ func GetPortsOfService(service *v1.Service) ([]int, []int) { return k8sPorts, k8sNodePorts } +func GetSvcPorts(service *v1.Service) []int { + k8sPorts := []int{} + for _, port := range service.Spec.Ports { + k8sPorts = append(k8sPorts, int(port.Port)) + } + return k8sPorts +} + +func GetPortsSlice(ports []v1.ServicePort) []int { + k8sPorts := []int{} + for _, port := range ports { + k8sPorts = append(k8sPorts, int(port.Port)) + } + return k8sPorts +} + func GetNodePort(service *v1.Service, port int32, protocol v1.Protocol) (nodePort int32, found bool) { if service == nil { return @@ -98,3 +114,21 @@ func GetRandomItems(items []*string, count int) (result []*string) { } return } + +func CoverPointSliceToStr[T string | int](ps []*T) (result []T) { + for _, v := range ps { + if v != nil { + result = append(result, *v) + } + } + return +} + +func GetNodesName(nodes []*v1.Node) (names []string) { + for _, node := range nodes { + if node != nil { + names = append(names, node.Name) + } + } + return +}