Support SecondaryNetwork of SR-IOV type for VM Nodes (#6881)

VM Nodes don't have access to Physical Function, the patch removes the code that checks whether the Physical Function exists or not, which is not really necessary. It also fixes two issues: 1. The usage of grpc.NewClient was wrong. 2. The kubelet socket was not accessible in the Pod. Signed-off-by: Quan Tian <[email protected]>
antrea-io · Jan 8, 2025 · bf41a62 · bf41a62
1 parent 503239f
commit bf41a62
Show file tree

Hide file tree

Showing 14 changed files with 48 additions and 72 deletions.
diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md
@@ -46,6 +46,7 @@ Kubernetes: `>= 1.19.0-0`
 | agent.installCNI.resources | object | `{"requests":{"cpu":"100m"}}` | Resource requests and limits for the install-cni initContainer. |
 | agent.installCNI.securityContext.capabilities | list | `["SYS_MODULE"]` | Capabilities for the install-cni initContainer. |
 | agent.installCNI.securityContext.privileged | bool | `false` | Run the install-cni container as privileged. |
+| agent.kubeletRootDir | string | `"/var/lib/kubelet"` | The root directory where kubelet stores its files. This is required to access the pod resources API, which is used to retrieve SR-IOV device allocation details for Pods. By default, the subdirectory containing the pod resources socket is mounted into antrea-agent Pods. Setting it to an empty value disables the mounting. |
 | agent.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for the antrea-agent Pods. |
 | agent.podAnnotations | object | `{}` | Annotations to be added to antrea-agent Pods. |
 | agent.podLabels | object | `{}` | Labels to be added to antrea-agent Pods. |

diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml
@@ -260,6 +260,10 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          {{- if .Values.agent.kubeletRootDir }}
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
+          {{- end }}
           {{- with .Values.agent.antreaAgent.extraVolumeMounts }}
           {{- toYaml . | trim | nindent 10 }}
           {{- end }}
@@ -397,6 +401,12 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        {{- if .Values.agent.kubeletRootDir }}
+        - name: host-pod-resources
+          hostPath:
+            path: {{ .Values.agent.kubeletRootDir }}/pod-resources
+            type: Directory
+        {{- end }}
         {{- with .Values.agent.extraVolumes }}
         {{- toYaml . | trim | nindent 8 }}
         {{- end }}
diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -264,6 +264,12 @@ agent:
   # Note that we will never try to load a module if we can detect that it is
   # "built-in", regardless of this value.
   dontLoadKernelModules: false
+  # -- The root directory where kubelet stores its files. This is required to
+  # access the pod resources API, which is used to retrieve SR-IOV device
+  # allocation details for Pods. By default, the subdirectory containing the pod
+  # resources socket is mounted into antrea-agent Pods. Setting it to an empty
+  # value disables the mounting.
+  kubeletRootDir: "/var/lib/kubelet"
   installCNI:
     # -- Extra environment variables to be injected into install-cni.
     extraEnv: {}

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -5550,6 +5550,8 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
         - name: antrea-ovs
           image: "antrea/antrea-agent-ubuntu:latest"
           imagePullPolicy: IfNotPresent
@@ -5620,6 +5622,10 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        - name: host-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 # Source: antrea/templates/controller/deployment.yaml
 apiVersion: apps/v1

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -5551,6 +5551,8 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
         - name: antrea-ovs
           image: "antrea/antrea-agent-ubuntu:latest"
           imagePullPolicy: IfNotPresent
@@ -5621,6 +5623,10 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        - name: host-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 # Source: antrea/templates/controller/deployment.yaml
 apiVersion: apps/v1

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -5548,6 +5548,8 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
         - name: antrea-ovs
           image: "antrea/antrea-agent-ubuntu:latest"
           imagePullPolicy: IfNotPresent
@@ -5618,6 +5620,10 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        - name: host-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 # Source: antrea/templates/controller/deployment.yaml
 apiVersion: apps/v1

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -5571,6 +5571,8 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
         - name: antrea-ovs
           image: "antrea/antrea-agent-ubuntu:latest"
           imagePullPolicy: IfNotPresent
@@ -5677,6 +5679,10 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        - name: host-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 # Source: antrea/templates/controller/deployment.yaml
 apiVersion: apps/v1

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -5548,6 +5548,8 @@ spec:
             mountPropagation: HostToContainer
           - name: xtables-lock
             mountPath: /run/xtables.lock
+          - name: host-pod-resources
+            mountPath: /var/lib/kubelet/pod-resources
         - name: antrea-ovs
           image: "antrea/antrea-agent-ubuntu:latest"
           imagePullPolicy: IfNotPresent
@@ -5618,6 +5620,10 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+        - name: host-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 # Source: antrea/templates/controller/deployment.yaml
 apiVersion: apps/v1

diff --git a/pkg/agent/cniserver/interface_configuration_linux.go b/pkg/agent/cniserver/interface_configuration_linux.go
@@ -213,13 +213,6 @@ func (ic *ifConfigurator) configureContainerSriovLink(
 	containerIface := &current.Interface{Name: containerIfaceName, Sandbox: containerNetNS}
 	result.Interfaces = []*current.Interface{hostIface, containerIface}
 
-	// Get rest of the VF information
-	pfName, vfID, err := ic.getVFInfo(pciAddress)
-	klog.V(2).InfoS("Get pfName and vfID of pciAddress", "pfName", pfName, "vfID", vfID, "pciAddress", pciAddress)
-	if err != nil {
-		return fmt.Errorf("failed to get VF information: %v", err)
-	}
-
 	vfIFName, err := ic.getVFLinkName(pciAddress)
 	if err != nil || vfIFName == "" {
 		return fmt.Errorf("VF interface not found for pciAddress %s: %v", pciAddress, err)

diff --git a/pkg/agent/cniserver/interface_configuration_linux_test.go b/pkg/agent/cniserver/interface_configuration_linux_test.go
@@ -146,7 +146,6 @@ func TestConfigureContainerLink(t *testing.T) {
 	fakeSriovNet := cniservertest.NewMockSriovNet(controller)
 	fakeNetlink := netlinktest.NewMockInterface(controller)
 
-	sriovPfName := "pf"
 	sriovVfNetdeviceName := "vfDevice"
 	vfDeviceLink := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Index: 2, MTU: mtu, HardwareAddr: containerVethMac, Name: sriovVfNetdeviceName, Flags: net.FlagUp}}
 
@@ -237,8 +236,6 @@ func TestConfigureContainerLink(t *testing.T) {
 				}
 			}
 			if tc.podSriovVFDeviceID != "" {
-				fakeSriovNet.EXPECT().GetPfName(tc.podSriovVFDeviceID).Return(sriovPfName, nil).Times(1)
-				fakeSriovNet.EXPECT().GetVfid(tc.podSriovVFDeviceID, sriovPfName).Return(sriovVfIndex, nil).Times(1)
 				fakeSriovNet.EXPECT().GetVFLinkNames(tc.podSriovVFDeviceID).Return(sriovVfNetdeviceName, nil).Times(1)
 				fakeNetlink.EXPECT().LinkByName(sriovVfNetdeviceName).Return(vfDeviceLink, nil).Times(1)
 				moveVFtoNS = true

diff --git a/pkg/agent/cniserver/interfaces.go b/pkg/agent/cniserver/interfaces.go
@@ -39,7 +39,5 @@ type SriovNet interface {
 	GetUplinkRepresentor(pciAddress string) (string, error)
 	GetVfIndexByPciAddress(vfPciAddress string) (int, error)
 	GetVfRepresentor(uplink string, vfIndex int) (string, error)
-	GetPfName(vf string) (string, error)
-	GetVfid(addr string, pfName string) (int, error)
 	GetVFLinkNames(pciAddr string) (string, error)
 }
diff --git a/pkg/agent/cniserver/sriov_linux.go b/pkg/agent/cniserver/sriov_linux.go
@@ -22,23 +22,6 @@ import (
 	sriovcniutils "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils"
 )
 
-// getVFInfo takes in a VF's PCI device ID and returns its PF and VF ID.
-func (ic *ifConfigurator) getVFInfo(vfPCI string) (string, int, error) {
-	var vfID int
-
-	pf, err := ic.sriovnet.GetPfName(vfPCI)
-	if err != nil {
-		return "", vfID, err
-	}
-
-	vfID, err = ic.sriovnet.GetVfid(vfPCI, pf)
-	if err != nil {
-		return "", vfID, err
-	}
-
-	return pf, vfID, nil
-}
-
 // getVFLinkName returns a VF's network interface name given its PCI address.
 func (ic *ifConfigurator) getVFLinkName(pciAddress string) (string, error) {
 	return ic.sriovnet.GetVFLinkNames(pciAddress)
@@ -62,14 +45,6 @@ func (n *sriovNet) GetVfRepresentor(uplink string, vfIndex int) (string, error)
 	return sriovnet.GetVfRepresentor(uplink, vfIndex)
 }
 
-func (n *sriovNet) GetPfName(vf string) (string, error) {
-	return sriovcniutils.GetPfName(vf)
-}
-
-func (n *sriovNet) GetVfid(addr string, pfName string) (int, error) {
-	return sriovcniutils.GetVfid(addr, pfName)
-}
-
 func (n *sriovNet) GetVFLinkNames(pciAddr string) (string, error) {
 	return sriovcniutils.GetVFLinkNames(pciAddr)
 }

diff --git a/pkg/agent/cniserver/testing/mock_cniserver.go b/pkg/agent/cniserver/testing/mock_cniserver.go
diff --git a/pkg/agent/secondarynetwork/podwatch/sriov.go b/pkg/agent/secondarynetwork/podwatch/sriov.go
@@ -17,7 +17,6 @@ package podwatch
 import (
 	"context"
 	"fmt"
-	"net"
 	"path"
 	"time"
 
@@ -60,11 +59,8 @@ type podSriovVFDeviceIDInfo struct {
 // getPodContainerDeviceIDs returns the device IDs assigned to a Pod's containers.
 func getPodContainerDeviceIDs(podName string, podNamespace string) ([]string, error) {
 	conn, err := grpc.NewClient(
-		path.Join(kubeletPodResourcesPath, kubeletSocket),
+		"unix:///"+path.Join(kubeletPodResourcesPath, kubeletSocket),
 		grpc.WithTransportCredentials(grpcinsecure.NewCredentials()),
-		grpc.WithContextDialer(func(ctx context.Context, addr string) (conn net.Conn, e error) {
-			return net.Dial("unix", addr)
-		}),
 	)
 	if err != nil {
 		return []string{}, fmt.Errorf("error getting the gRPC client for Pod resources: %v", err)