From 88a4f6bc0bcdf688804041d83a27af448d4ad912 Mon Sep 17 00:00:00 2001 From: Ammar Yasser Date: Mon, 9 Sep 2024 09:28:11 +0300 Subject: [PATCH] fix: Wait until istio cni files appear before copying them (#1262) * fix: Wait until istio cni files appear before copying them Signed-off-by: aerosouund * fix: Use CNAO v0.94 Previous version of CNAO introduced a problem with passt binding Signed-off-by: aerosouund * fix: Change CNAO custom resource to not have multus in case multus is enabled This logic is controlled by the cluster-up variable KUBEVIRT_WITH_MULTUS_V3. If this variable is passed then it will get parsed as a gocli flag, which then gets added to the node config using the nodesconfig.WithMultus(deployMultus) line in run.go In this case multus is omitted from the CNAO CR and deployed using its actual manifests Signed-off-by: aerosouund * fix: Remove CNAO patch from istio operator This is the same configuration that existed prior to #1217 getting merged. Ths addition of this lead to troubles with istio passt binding, as passt wasn't able to bind to port 22 due to insufficient permissions Signed-off-by: aerosouund * test: Create test cases to assert CNAO is creating the addons config with and without Multus properly when passing the multusEnabled argument Signed-off-by: aerosouund --------- Signed-off-by: aerosouund --- cluster-provision/gocli/cmd/run.go | 2 +- cluster-provision/gocli/opts/cnao/cnao.go | 19 ++-- .../gocli/opts/cnao/cnao_test.go | 32 ++++++- .../gocli/opts/cnao/manifests/crd.yaml | 36 ++------ .../gocli/opts/cnao/manifests/namespace.yaml | 3 +- .../network-addons-config-example.cr.yaml | 2 +- .../gocli/opts/cnao/manifests/operator.yaml | 90 ++++++++++++++----- .../istio-operator-with-cnao.cr.yaml | 4 - cluster-up/cluster/k8s-provider-common.sh | 16 +++- 9 files changed, 139 insertions(+), 65 deletions(-) diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index 2931a5e611..8337a59abe 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -812,7 +812,7 @@ func provisionK8sOptions(sshClient libssh.Client, k8sClient k8s.K8sDynamicClient } if n.CNAO { - cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient) + cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient, n.Multus) opts = append(opts, cnaoOpt) } diff --git a/cluster-provision/gocli/opts/cnao/cnao.go b/cluster-provision/gocli/opts/cnao/cnao.go index 2b1a053cfd..81a6f89b39 100644 --- a/cluster-provision/gocli/opts/cnao/cnao.go +++ b/cluster-provision/gocli/opts/cnao/cnao.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "path/filepath" + "regexp" "github.com/sirupsen/logrus" k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" @@ -16,14 +17,16 @@ import ( var f embed.FS type cnaoOpt struct { - client k8s.K8sDynamicClient - sshClient libssh.Client + client k8s.K8sDynamicClient + sshClient libssh.Client + multusEnabled bool } -func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client) *cnaoOpt { +func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client, multusEnabled bool) *cnaoOpt { return &cnaoOpt{ - client: c, - sshClient: sshClient, + client: c, + sshClient: sshClient, + multusEnabled: multusEnabled, } } @@ -43,6 +46,12 @@ func (o *cnaoOpt) Exec() error { continue } + if path == "manifests/network-addons-config-example.cr.yaml" && o.multusEnabled { + re := regexp.MustCompile("(?m)[\r\n]+^.*multus.*$") + res := re.ReplaceAllString(string(yamlDoc), "") + yamlDoc = []byte(res) + } + obj, err := k8s.SerializeIntoObject(yamlDoc) if err != nil { logrus.Info(err.Error()) diff --git a/cluster-provision/gocli/opts/cnao/cnao_test.go b/cluster-provision/gocli/opts/cnao/cnao_test.go index f90eb02baa..4b2609f1d0 100644 --- a/cluster-provision/gocli/opts/cnao/cnao_test.go +++ b/cluster-provision/gocli/opts/cnao/cnao_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" + "k8s.io/apimachinery/pkg/runtime/schema" k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" ) @@ -27,16 +28,41 @@ var _ = Describe("CnaoOpt", func() { mockCtrl = gomock.NewController(GinkgoT()) client = k8s.NewTestClient() sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) - opt = NewCnaoOpt(client, sshClient) }) AfterEach(func() { mockCtrl.Finish() }) - It("should execute CnaoOpt successfully", func() { + It("should execute create CNAO with Multus", func() { + opt = NewCnaoOpt(client, sshClient, false) sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s") - err := opt.Exec() + opt.Exec() + + obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io", + Version: "v1", + Kind: "NetworkAddonsConfig"}, "cluster", "") + Expect(err).NotTo(HaveOccurred()) + + spec, ok := obj.Object["spec"].(map[string]interface{}) + Expect(ok).To(Equal(true)) + Expect(spec).To(HaveKey("multus")) + Expect(spec).To(HaveKey("multusDynamicNetworks")) + }) + + It("should execute create CNAO without Multus", func() { + opt = NewCnaoOpt(client, sshClient, true) + sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s") + opt.Exec() + + obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io", + Version: "v1", + Kind: "NetworkAddonsConfig"}, "cluster", "") Expect(err).NotTo(HaveOccurred()) + + spec, ok := obj.Object["spec"].(map[string]interface{}) + Expect(ok).To(Equal(true)) + Expect(spec).NotTo(HaveKey("multus")) + Expect(spec).NotTo(HaveKey("multusDynamicNetworks")) }) }) diff --git a/cluster-provision/gocli/opts/cnao/manifests/crd.yaml b/cluster-provision/gocli/opts/cnao/manifests/crd.yaml index 9f84c50f09..e173349389 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/crd.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/crd.yaml @@ -59,6 +59,10 @@ spec: description: NameServerIp defines the name server IP type: string type: object + kubevirtIpamController: + description: KubevirtIpamController plugin allows to support IPAM + for secondary networks + type: object linuxBridge: description: LinuxBridge plugin allows users to create a bridge and add the host and the container to it @@ -1521,20 +1525,6 @@ spec: - ECDHE-RSA-CHACHA20-POLY1305 - DHE-RSA-AES128-GCM-SHA256 - DHE-RSA-AES256-GCM-SHA384 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - ECDHE-ECDSA-AES128-GCM-SHA256 - - ECDHE-RSA-AES128-GCM-SHA256 - - ECDHE-ECDSA-AES256-GCM-SHA384 - - ECDHE-RSA-AES256-GCM-SHA384 - - ECDHE-ECDSA-CHACHA20-POLY1305 - - ECDHE-RSA-CHACHA20-POLY1305 - - DHE-RSA-AES128-GCM-SHA256 - - DHE-RSA-AES256-GCM-SHA384 - DHE-RSA-CHACHA20-POLY1305 - ECDHE-ECDSA-AES128-SHA256 - ECDHE-RSA-AES128-SHA256 @@ -1706,6 +1696,10 @@ spec: description: NameServerIp defines the name server IP type: string type: object + kubevirtIpamController: + description: KubevirtIpamController plugin allows to support IPAM + for secondary networks + type: object linuxBridge: description: LinuxBridge plugin allows users to create a bridge and add the host and the container to it @@ -3168,20 +3162,6 @@ spec: - ECDHE-RSA-CHACHA20-POLY1305 - DHE-RSA-AES128-GCM-SHA256 - DHE-RSA-AES256-GCM-SHA384 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - ECDHE-ECDSA-AES128-GCM-SHA256 - - ECDHE-RSA-AES128-GCM-SHA256 - - ECDHE-ECDSA-AES256-GCM-SHA384 - - ECDHE-RSA-AES256-GCM-SHA384 - - ECDHE-ECDSA-CHACHA20-POLY1305 - - ECDHE-RSA-CHACHA20-POLY1305 - - DHE-RSA-AES128-GCM-SHA256 - - DHE-RSA-AES256-GCM-SHA384 - DHE-RSA-CHACHA20-POLY1305 - ECDHE-ECDSA-AES128-SHA256 - ECDHE-RSA-AES128-SHA256 diff --git a/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml b/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml index ebe63fda20..972335a37f 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml @@ -1,7 +1,8 @@ +--- apiVersion: v1 kind: Namespace metadata: name: cluster-network-addons labels: name: cluster-network-addons - openshift.io/cluster-monitoring: "true" \ No newline at end of file + openshift.io/cluster-monitoring: "true" diff --git a/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml b/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml index 0c5662837e..718853d3a6 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml @@ -1,3 +1,4 @@ +--- apiVersion: networkaddonsoperator.network.kubevirt.io/v1 kind: NetworkAddonsConfig metadata: @@ -10,4 +11,3 @@ spec: macvtap: {} multus: {} multusDynamicNetworks: {} - ovs: {} \ No newline at end of file diff --git a/cluster-provision/gocli/opts/cnao/manifests/operator.yaml b/cluster-provision/gocli/opts/cnao/manifests/operator.yaml index fd7bb41929..6ee0724822 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/operator.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/operator.yaml @@ -120,6 +120,24 @@ rules: - services verbs: - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - ipamclaims + verbs: + - get + - list + - watch + - create + - update +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -144,12 +162,6 @@ rules: - create - patch - update -- apiGroups: - - k8s.cni.cncf.io - resources: - - '*' - verbs: - - '*' - apiGroups: - "" resources: @@ -238,6 +250,12 @@ rules: - get - list - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - '*' + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 @@ -329,6 +347,34 @@ rules: verbs: - get - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - patch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - get + - create + - update + - delete --- apiVersion: rbac.authorization.k8s.io/v1 @@ -349,7 +395,7 @@ apiVersion: apps/v1 kind: Deployment metadata: annotations: - networkaddonsoperator.network.kubevirt.io/version: 0.87.0 + networkaddonsoperator.network.kubevirt.io/version: 0.94.1 labels: prometheus.cnao.io: "true" name: cluster-network-addons-operator @@ -375,29 +421,31 @@ spec: - name: MULTUS_IMAGE value: ghcr.io/k8snetworkplumbingwg/multus-cni@sha256:3fbcc32bd4e4d15bd93c96def784a229cd84cca27942bf4858b581f31c97ee02 - name: MULTUS_DYNAMIC_NETWORKS_CONTROLLER_IMAGE - value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:dee1979d92f0a31598a6e3569ac7004be7d29e7ca9e31db23753ef263110dc04 + value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:83b460502671fb4f34116363a1a39b2ddfc9d14a920ee0a6413bfc3bd0580404 - name: LINUX_BRIDGE_IMAGE - value: quay.io/kubevirt/cni-default-plugins@sha256:825e3f9fec1996c54a52cec806154945b38f76476b160d554c36e38dfffe5e61 + value: quay.io/kubevirt/cni-default-plugins@sha256:0c354fa9d695b8cab97b459e8afea2f7662407a987e83f6f6f1a8af4b45726be - name: LINUX_BRIDGE_MARKER_IMAGE - value: quay.io/kubevirt/bridge-marker@sha256:5d24c6d1ecb0556896b7b81c7e5260b54173858425777b7a84df8a706c07e6d2 + value: quay.io/kubevirt/bridge-marker@sha256:bba066e3b5ff3fb8c5e20861fe8abe51e3c9b50ad6ce3b2616af9cb5479a06d0 - name: OVS_CNI_IMAGE - value: quay.io/kubevirt/ovs-cni-plugin@sha256:5f7290e2294255ab2547c3b4bf48cc2d75531ec5a43e600366e9b2719bef983f + value: quay.io/kubevirt/ovs-cni-plugin@sha256:e16ac74343da21abb8fb668ce71e728053d00503a992dae2164b9e94a280113e - name: KUBEMACPOOL_IMAGE - value: quay.io/kubevirt/kubemacpool@sha256:afba7d0c4a95d2d4924f6ee6ef16bbe59117877383819057f01809150829cb0c + value: quay.io/kubevirt/kubemacpool@sha256:20e156be33e6d3692c456081acbb91a4349f94de448f7d1f1cddd0228931b31f - name: MACVTAP_CNI_IMAGE - value: quay.io/kubevirt/macvtap-cni@sha256:434420511e09b2b5ede785a2c9062b6658ffbc26fbdd4629ce06110f9039c600 + value: quay.io/kubevirt/macvtap-cni@sha256:850b89343ace7c7ea6b18dd8e11964613974e9d1f7377af03854d407fb15230a - name: KUBE_RBAC_PROXY_IMAGE - value: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901 + value: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8 - name: KUBE_SECONDARY_DNS_IMAGE - value: ghcr.io/kubevirt/kubesecondarydns@sha256:77132adb5f840ceb0aadd408731a5c8b01a4b427a78084ab5e4e9b961195cb02 + value: ghcr.io/kubevirt/kubesecondarydns@sha256:6268d84154e2483fbce8c1adacbdaf6f0839117b2d48d9fa4687cc8f76bd5130 - name: CORE_DNS_IMAGE value: registry.k8s.io/coredns/coredns@sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e + - name: KUBEVIRT_IPAM_CONTROLLER_IMAGE + value: ghcr.io/kubevirt/ipam-controller@sha256:f272eaf82c9e4fcd7fdee5d9031afba8019cff2dc842ca3e36be49de28083fee - name: OPERATOR_IMAGE - value: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0 + value: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1 - name: OPERATOR_NAME value: cluster-network-addons-operator - name: OPERATOR_VERSION - value: 0.87.0 + value: 0.94.1 - name: OPERATOR_NAMESPACE valueFrom: fieldRef: @@ -416,8 +464,8 @@ spec: - name: MONITORING_SERVICE_ACCOUNT value: prometheus-k8s - name: RUNBOOK_URL_TEMPLATE - value: https://kubevirt.io/monitoring/runbooks/ - image: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0 + value: https://kubevirt.io/monitoring/runbooks/%s + image: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1 imagePullPolicy: Always name: cluster-network-addons-operator resources: @@ -433,7 +481,7 @@ spec: - --logtostderr - --secure-listen-address=:8443 - --upstream=http://127.0.0.1:8080 - image: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901 + image: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8 imagePullPolicy: Always name: kube-rbac-proxy ports: @@ -455,4 +503,4 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - serviceAccountName: cluster-network-addons-operator \ No newline at end of file + serviceAccountName: cluster-network-addons-operator diff --git a/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml b/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml index d6bad14670..7e50e2bc85 100644 --- a/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml +++ b/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml @@ -22,7 +22,3 @@ spec: - istio-system - kube-system logLevel: debug - cniConfFileName: "istio-cni.conf" - sidecarInjectorWebhook: - injectedAnnotations: - "k8s.v1.cni.cncf.io/networks": istio-cni \ No newline at end of file diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 8125112ae8..32c4ede425 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -14,7 +14,16 @@ function deploy_kwok() { fi } - +# copy_istio_cni_conf_files copy the generated Istio CNI net conf file +# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/') +function copy_istio_cni_conf_files() { + if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then + for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do + $ssh node${nodeNum} -- "until ls /etc/cni/multus > /dev/null 2>&1; do sleep 1; done" + $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/ + done + fi +} # configure Prometheus to select kubevirt prometheusrules function configure_prometheus() { @@ -95,4 +104,9 @@ function up() { echo "Waiting for cluster components..." sleep 5 done + + # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved + # [1] https://github.com/kubevirt/kubevirtci/issues/906 + # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982 + copy_istio_cni_conf_files }