wip: get working with flux operator + slingshot

Signed-off-by: vsoch <[email protected]>

Author: vsoch

Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /go/src/github.com/HPE/cxi-k8s-device-plugin
 RUN make build
 RUN cp bin/cxi-k8s-device-plugin /go/bin/
 
-FROM alpine:latest
-WORKDIR /root/
-COPY --from=0 /go/bin/cxi-k8s-device-plugin .
-CMD ["./cxi-k8s-device-plugin", "-logtostderr=true", "-stderrthreshold=INFO", "-v=5"]
+# FROM alpine:latest
+# WORKDIR /root/
+# COPY --from=0 /go/bin/cxi-k8s-device-plugin .
+CMD ["/go/bin/cxi-k8s-device-plugin", "-logtostderr=true", "-stderrthreshold=INFO", "-v=5"]

README.md

@@ -60,4 +60,12 @@ kubectl apply \
     -f ./deploy/hpecxi-device-plugin-ds.yaml
 ```
 
+## Pod Resources
+
+```yaml
+resources:
+  requests: 
+    beta.hpe.com/cxi: 1
+```
+
 > #### Make sure the IPAM definitions in the `./deploy/NetworkAttachmentDefinition` are follwoing your cluster network requirements. 

cmd/main.go

@@ -29,7 +29,9 @@ func main() {
 		flag.PrintDefaults()
 	}
 	var pulse int
+	devicePrefix := "hsi"
 	flag.IntVar(&pulse, "pulse", 0, "time between health check polling in seconds.  Set to 0 to disable.")
+	flag.StringVar(&devicePrefix, "prefix", "hsi", "Device prefix to search for in net")
 	flag.Parse()
 
 	for _, v := range versions {

deploy/hpecxi-device-plugin-ds.yaml

@@ -12,10 +12,16 @@ spec:
       labels:
         name: hpecxi-dp-ds
     spec:
+      nodeSelector:
+        "cray.nnf.node": "true"
       priorityClassName: system-node-critical
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
+      - key: "cray.nnf.node"
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
       containers:
       - image: ghcr.io/hewlettpackard/cxi-k8s-device-plugin:0.0.1-beta
         imagePullPolicy: Always
@@ -45,4 +51,5 @@ spec:
             path: /opt/cray
         - name: libfabric
           hostPath:
-            path: /usr/lib64
+            path: /usr/lib64
+

pkg/hpecxi/hpecxi.go

@@ -11,6 +11,8 @@ import (
 
 const HPEvendorID string = "0x17db"
 
+var cxiDriverRoot = "/sys/module/cxi_ss1/drivers"
+
 // In the future, we may want to include lib paths into the helm.
 var LibPaths = map[string]string{
 	"libfabric":   "/opt/cray/lib64",
@@ -61,22 +63,29 @@ func GetLibs() ([]string, error) {
 // GetHPECXIs return a map of HPE Cassini on a node identified by the part of the pci address
 // This may be changed to use cxilib calls instead of sysfs.
 func GetHPECXIs() map[string]int {
-	if _, err := os.Stat("/sys/module/cxi_core/drivers/"); err != nil {
+	if _, err := os.Stat(cxiDriverRoot); err != nil {
 		klog.Warningf("HPE CXI driver unavailable: %s", err)
 		return make(map[string]int)
 	}
 
-	matches, _ := filepath.Glob("/sys/module/cxi_core/drivers/pci:cxi_core/[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]:*")
+	cxiRegex := filepath.Join(cxiDriverRoot, "pci:cxi_ss1/[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]:*")
+	klog.Info(cxiRegex)
+	matches, _ := filepath.Glob(cxiRegex)
 
+	klog.Info("Found matches:")
+	klog.Info(matches)
 	devices := make(map[string]int)
 
 	for _, path := range matches {
+		// This is a directory with "net"
+		// /sys/module/cxi_ss1/drivers/pci:cxi_ss1/0000:01:00.0/
+
 		klog.Info(path)
 		devPaths, _ := filepath.Glob(path + "/net/*")
 
 		for _, devPath := range devPaths {
 			name := filepath.Base(devPath)
-			if name[0:3] == "hsn" {
+			if name[0:3] == "hsi" {
 				nic_id, _ := strconv.Atoi(name[len(name)-1:])
 				devices[name] = nic_id
 			}