From 5a137c997f17579fa7b1e95c3af3cf563d334c3d Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Tue, 18 Aug 2026 14:36:04 +0200 Subject: [PATCH 01/11] feat(nvml-mock): serve the rendered PCI sysfs tree and a DMI identity to Go consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rendered PCI tree was reachable only through MOCK_PCI_ROOT and the libpcimocksys.so LD_PRELOAD shim, which works for libc consumers such as lspci but not for Go: os.Open issues openat directly, the shim never sees it, and the process reads the node's real /sys. GPU Feature Discovery and the DRA driver are both Go, so GFD failed to read a mock GPU's vendor and class and labelled the node nvidia.com/gpu.mode=unknown. Nothing mocked the node's DMI identity either, leaving nvidia.com/gpu.machine unknown as well. Bind-mount sys/devices and sys/bus/pci/devices from the overlay onto the kernel paths, through both channels that serve containers: the CDI spec setup.sh generates and the NRI plugin's adjustment. Both mounts go together because the PCI entries are relative symlinks into ../../../devices, so one alone leaves every attribute read at ENOENT. /sys/devices is mounted whole: narrowing it to the profile's root complexes would need mountpoints the runtime cannot create on a read-only sysfs, and container creation fails outright. That hides the host's other device classes from served containers, so nri.pciSysfsMounts=false opts out. Profiles may now declare a dmi.product_name, which the renderer writes to sys/devices/virtual/dmi/id/product_name — where /sys/class/dmi/id points, and inside the only subtree a container can be handed. The NVIDIA platform profiles set it; l40s and t4 do not, since a commodity server's machine type belongs to the chassis rather than the GPU. The PCI render moves ahead of CDI spec generation: a bind mount whose source is missing fails container creation for the whole pod. Fixes #673 Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 26 ++++ cmd/nvml-mock-nri/main.go | 19 +++ cmd/nvml-mock-nri/main_test.go | 18 +++ cmd/render-pci-sysfs/main.go | 7 +- .../nvml-mock/helm/nvml-mock/README.md | 83 +++++++++++-- .../helm/nvml-mock/profiles/a100.yaml | 17 ++- .../helm/nvml-mock/profiles/b200.yaml | 17 ++- .../helm/nvml-mock/profiles/gb200.yaml | 17 ++- .../helm/nvml-mock/profiles/gb300.yaml | 9 ++ .../helm/nvml-mock/profiles/h100.yaml | 17 ++- .../helm/nvml-mock/profiles/l40s.yaml | 19 ++- .../nvml-mock/helm/nvml-mock/profiles/t4.yaml | 19 ++- .../nvml-mock/templates/nri-daemonset.yaml | 6 + .../__snapshot__/configmap_test.yaml.snap | 115 ++++++++++++++---- .../__snapshot__/daemonset_test.yaml.snap | 6 +- .../nvml-mock/tests/nri_daemonset_test.yaml | 22 ++++ .../nvml-mock/helm/nvml-mock/values.yaml | 11 ++ deployments/nvml-mock/scripts/setup.sh | 81 ++++++++---- local/gpu-operator/gpu-operator.values.yaml | 10 ++ .../configs/mock-nvml-config-a100.yaml | 8 +- .../configs/mock-nvml-config-b200.yaml | 8 +- .../configs/mock-nvml-config-gb200.yaml | 8 +- .../configs/mock-nvml-config-h100.yaml | 8 +- .../configs/mock-nvml-config-l40s.yaml | 8 +- .../mocknvml/configs/mock-nvml-config-t4.yaml | 8 +- pkg/nri/nvmlmock/adjust.go | 73 +++++++++++ pkg/nri/nvmlmock/adjust_test.go | 104 ++++++++++++++++ pkg/system/mockpcisysfs/config/types.go | 30 ++++- pkg/system/mockpcisysfs/render/render.go | 39 +++++- pkg/system/mockpcisysfs/render/render_test.go | 71 +++++++++++ tests/e2e/go/assertions/gfd_labels.go | 14 +++ tests/e2e/go/assertions/gfd_labels_test.go | 19 +++ tests/e2e/go/assertions/pcisysfs.go | 54 ++++++++ tests/e2e/go/profile/gfd_test.go | 20 +++ tests/e2e/go/profile/profile.go | 29 +++++ tests/e2e/go/scenario_gpu_operator_test.go | 20 +++ tests/e2e/gpu-operator-values.yaml | 7 ++ tests/mocknvml/util-test-config.yaml | 8 +- 38 files changed, 936 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f02e2dbd9..8c6e9e590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 than 560, keep reporting `N/A`. The `GPU Fabric GUID` row of the same block is not modelled and now renders `0x0000000000000000` where it used to read `N/A`. (#642) +- The rendered PCI sysfs tree now reaches consumers written in Go. `lspci` and + other libc consumers found it through the `libpcimocksys.so` `LD_PRELOAD` + shim, but Go reads sysfs with direct `openat` syscalls no shim can intercept, + so GPU Feature Discovery and the NVIDIA DRA driver read the node's real + `/sys` and saw no mock GPUs — GFD logged `unable to read PCI device vendor id` + and labelled the node `nvidia.com/gpu.mode=unknown`. The staged + `sys/devices` and `sys/bus/pci/devices` directories are now bind-mounted + read-only onto the kernel paths, through both the CDI spec the DaemonSet + generates and the NRI plugin's container adjustment. Both mounts go together: + the PCI entries are relative symlinks into `../../../devices/pciDDDD:BB`, so + mounting one alone leaves every attribute read failing with `ENOENT`. + `/sys/devices` is necessarily mounted whole — it cannot be narrowed to the + profile's root complexes, because a bind mount at a path sysfs lacks needs a + mountpoint the runtime cannot create on a read-only `/sys` — which hides the + host's other device classes from served containers. `nri.pciSysfsMounts=false` + opts out. (#673) +- Profiles may declare the machine type a node of that platform reports, as a + `dmi:` block with `product_name`, which the renderer writes to + `sys/devices/virtual/dmi/id/product_name` in the mock overlay — the path + `/sys/class/dmi/id` points at, and the only one a container can be handed. + Pointing GPU Feature Discovery at it with + `GFD_MACHINE_TYPE_FILE=/sys/devices/virtual/dmi/id/product_name` makes + `nvidia.com/gpu.machine` report the profile's platform (`NVIDIA-GB200-NVL72`, + `DGXA100`, …) instead of the host's or `unknown`. The NVIDIA platform + profiles set it; `l40s` and `t4` deliberately do not, since a commodity + server's machine type is a property of the chassis, not the GPU. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/cmd/nvml-mock-nri/main.go b/cmd/nvml-mock-nri/main.go index aa4c41ddd..00604bfc0 100644 --- a/cmd/nvml-mock-nri/main.go +++ b/cmd/nvml-mock-nri/main.go @@ -15,6 +15,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -49,6 +50,7 @@ func main() { flag.StringVar(&cfg.DeviceAnnotation, "device-annotation", envOr("NVML_MOCK_DEVICE_ANNOTATION", cfg.DeviceAnnotation), "pod annotation key; value true adds /dev/nvidia* device nodes") flag.StringVar(&cfg.ImexChannelAnnotation, "imex-channel-annotation", envOr("NVML_MOCK_IMEX_CHANNEL_ANNOTATION", cfg.ImexChannelAnnotation), "pod annotation key; value true adds /dev/nvidia-caps-imex-channels/* nodes") flag.StringVar(&cfg.ImexChannelHostPath, "imex-channel-host-path", envOr("NVML_MOCK_IMEX_CHANNEL_HOST_PATH", cfg.ImexChannelHostPath), "host path containing the mock IMEX channel nodes staged by imex.mockChannels (defaults to /driver/dev/nvidia-caps-imex-channels)") + flag.BoolVar(&cfg.DisablePCISysfsMounts, "disable-pci-sysfs-mounts", boolEnvOr("NVML_MOCK_DISABLE_PCI_SYSFS_MOUNTS", cfg.DisablePCISysfsMounts), "do not mount the rendered PCI sysfs tree at /sys/bus/pci/devices and /sys/devices; consumers that read sysfs directly (GPU Feature Discovery, the DRA driver) then see no GPUs") excludedNamespaces := flag.String("excluded-namespaces", envOr("NVML_MOCK_EXCLUDED_NAMESPACES", strings.Join(cfg.ExcludedNamespaces, ",")), "comma-separated namespaces to skip") shims := flag.String("ld-preload-shims", envOr("NVML_MOCK_LD_PRELOAD_SHIMS", strings.Join(cfg.Shims, ",")), "comma-separated LD_PRELOAD shim paths relative to the overlay mount or absolute paths") flag.Parse() @@ -269,6 +271,23 @@ func envOr(key, fallback string) string { return fallback } +// boolEnvOr reads a boolean env override. An unparseable value falls back +// rather than aborting: this only ever gates an optimisation of the injection, +// and a DaemonSet that refused to start over a typo'd "yes" would take the +// whole node's mock GPUs with it. +func boolEnvOr(key string, fallback bool) bool { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseBool(value) + if err != nil { + log.Printf("nvml-mock-nri: %s=%q is not a boolean; using %t", key, value, fallback) + return fallback + } + return parsed +} + // major and minor decode a Linux dev_t the way glibc encodes it // (MMMM Mmmm mmmM MMmm): the major occupies bits 8-19 and 44-63, the minor // bits 0-7 and 20-43. diff --git a/cmd/nvml-mock-nri/main_test.go b/cmd/nvml-mock-nri/main_test.go index 339547a00..1c2b1b5b2 100644 --- a/cmd/nvml-mock-nri/main_test.go +++ b/cmd/nvml-mock-nri/main_test.go @@ -411,3 +411,21 @@ func TestEnvOr(t *testing.T) { require.Equal(t, "/fallback", envOr("NVML_MOCK_TEST_KEY", "/fallback")) }) } + +func TestBoolEnvOr(t *testing.T) { + t.Run("parses the environment value", func(t *testing.T) { + t.Setenv("NVML_MOCK_TEST_BOOL", "true") + require.True(t, boolEnvOr("NVML_MOCK_TEST_BOOL", false)) + }) + + t.Run("returns the fallback when unset", func(t *testing.T) { + require.True(t, boolEnvOr("NVML_MOCK_TEST_BOOL_UNSET", true)) + }) + + // A garbled value must not flip the default: the fallback is what keeps the + // node's mock GPUs visible to sysfs consumers. + t.Run("falls back on an unparseable value", func(t *testing.T) { + t.Setenv("NVML_MOCK_TEST_BOOL", "yes-please") + require.False(t, boolEnvOr("NVML_MOCK_TEST_BOOL", false)) + }) +} diff --git a/cmd/render-pci-sysfs/main.go b/cmd/render-pci-sysfs/main.go index 0ee9d7e64..7ce9e5f8a 100644 --- a/cmd/render-pci-sysfs/main.go +++ b/cmd/render-pci-sysfs/main.go @@ -76,9 +76,10 @@ func main() { } if err := render.Render(render.Options{ - Topology: topo, - Identities: prof.DeviceIdentities(), - Output: *outDir, + Topology: topo, + Identities: prof.DeviceIdentities(), + Output: *outDir, + DMIProductName: prof.DMIProductName(), }); err != nil { fatalf("render: %v", err) } diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index f9814729a..bc010168b 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -21,10 +21,14 @@ Deploys a DaemonSet that creates on every node: - A fake PCI sysfs tree at `/var/lib/nvml-mock/sys/bus/pci/devices/...` (symlinks into `/var/lib/nvml-mock/sys/devices/pciDDDD:BB/...`) so C consumers of the PCI sysfs — `lspci` and anything else reaching it through libc — resolve the - PCIe root complex via a standard `readlink()`. The NVIDIA DRA driver is a Go - binary and does not see this tree, so `dra.k8s.io/pcieRoot` is still absent - from its ResourceSlices; see [Known Limitations](#known-limitations) and - issue [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) + PCIe root complex via a standard `readlink()`. Consumers written in Go read + sysfs with direct syscalls no `LD_PRELOAD` shim can intercept, so the two + directories are additionally bind-mounted onto `/sys/bus/pci/devices` and + `/sys/devices` in served containers (see + [PCI sysfs in containers](#pci-sysfs-in-containers)) +- A fake DMI identity at `/var/lib/nvml-mock/sys/devices/virtual/dmi/id/product_name` + when the profile declares a `dmi:` block, which GPU Feature Discovery turns + into `nvidia.com/gpu.machine` (see [PCI sysfs in containers](#pci-sysfs-in-containers)) Consumers (DRA driver, device plugin) point at `/var/lib/nvml-mock/driver` as the NVIDIA driver root and discover GPUs through standard NVML APIs. @@ -708,6 +712,64 @@ DaemonSet under `set -e` if it finds a typo: If a profile omits `pcie_topology:` entirely the renderer falls back to a flat single-root layout (every device under `pci0000:00`, NUMA 0). +### PCI sysfs in containers + +Reaching the tree through `MOCK_PCI_ROOT` requires the `libpcimocksys.so` +`LD_PRELOAD` shim, which only works for consumers that go through libc. A Go +program does not: `os.Open` issues `openat` directly, the shim never sees it, +and the process reads the node's real `/sys` — where the mock GPUs do not +exist. GPU Feature Discovery and the NVIDIA DRA driver are both Go. + +So the rendered directories are bind-mounted read-only onto the kernel paths +in containers the mock serves, through the CDI spec the DaemonSet generates at +`/var/run/cdi/nvidia.yaml` and, when `nri.enabled=true`, through the NRI +plugin's container adjustment: + +| Host | Container | +|---|---| +| `/var/lib/nvml-mock/sys/devices` | `/sys/devices` | +| `/var/lib/nvml-mock/sys/bus/pci/devices` | `/sys/bus/pci/devices` | + +Both are needed together: the entries under `/sys/bus/pci/devices` are +relative symlinks into `../../../devices/pciDDDD:BB`, so mounting only that +directory yields entries that list but whose every attribute read fails with +`ENOENT`. + +**Trade-off:** `/sys/devices` is mounted whole, which hides the host's other +device classes (CPU topology among them) from those containers. It cannot be +narrowed to the profile's root complexes — a bind mount at a path sysfs does +not already have needs a mountpoint, and the runtime cannot create one on a +read-only `/sys`. Set `nri.pciSysfsMounts=false` to drop the mounts from the +NRI path on a cluster that cannot accept this; sysfs-reading consumers then +see no GPUs there. + +### Machine type (`dmi:`) + +A profile may declare the SMBIOS product name a node of that platform +reports: + +```yaml +dmi: + product_name: "NVIDIA GB200 NVL72" +``` + +The renderer writes it to +`/var/lib/nvml-mock/sys/devices/virtual/dmi/id/product_name` — inside the +subtree above, which is where `/sys/class/dmi/id` points on a real node and +the only place a container can be given it. Point GPU Feature Discovery at it +to have `nvidia.com/gpu.machine` reflect the profile instead of the host: + +```yaml +gfd: + env: + - name: GFD_MACHINE_TYPE_FILE + value: "/sys/devices/virtual/dmi/id/product_name" +``` + +Profiles for commodity servers (`l40s`, `t4`) ship no `dmi:` block, since +their machine type is a property of the chassis rather than the GPU. Nothing +is rendered then, and the label stays `unknown`. + ### Cross-node `ibping` Sysfs mocking alone lets `ibstat` / `iblinkinfo` work, but real `ibping` @@ -1362,7 +1424,7 @@ discovery and monitoring. Some host-level subsystems are not mocked: | What's Missing | Affected Consumer | Impact | |----------------|-------------------|--------| -| `/sys/bus/pci/devices/{busID}` sysfs entries **as a Go program reads them** | DRA driver | The tree is rendered and `lspci` reads it, but the driver is a Go binary: Go's `os` package issues raw syscalls that the `LD_PRELOAD` shim cannot intercept, so it reads the host's real sysfs instead. `dra.k8s.io/pcieRoot` stays absent from ResourceSlices — **blocks topology-aware scheduling demos** (e.g., GPU + SR-IOV VF alignment). Tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) | +| `/sys/bus/pci/devices/{busID}` sysfs entries in containers the mock does **not** serve | DRA driver | The tree is now bind-mounted onto the kernel paths for CDI- and NRI-served containers, which is what Go consumers need (see [PCI sysfs in containers](#pci-sysfs-in-containers)). A consumer deployed outside those channels still reads the host's real sysfs; whether `dra.k8s.io/pcieRoot` reaches ResourceSlices is tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) | | `/sys/bus/pci/devices/{busID}/numa_node` | Device plugin | NUMA-aware topology hints unavailable; scheduling works but NUMA affinity not enforced | | `/sys/bus/pci/devices/*/vendor,device,class` **as NFD reads them** (`/host-sys/…`, fixed at link time) | NFD (Node Feature Discovery) | PCI feature labels not auto-detected. `nvidia.com/gpu.present` is written directly by nvml-mock; `pci-10de.present` is created by NFD from a feature file nvml-mock drops in `nodeLabels.featuresDir` — see [Node Labels](#node-labels) | @@ -1377,11 +1439,12 @@ W0319 11:41:21.314205 1 nvlib.go:491] error getting PCIe root for device 0 readlink /sys/bus/pci/devices/0000:07:00.0: no such file or directory ``` -**This warning is expected** but has real impact. The DRA driver resolves PCIe -root complex topology by reading sysfs symlinks. Since nvml-mock provides a mock -NVML library (not a full kernel driver), these sysfs entries don't exist. GPUs -appear in ResourceSlices and are fully allocatable, but the -`dra.k8s.io/pcieRoot` topology attribute is absent. +The DRA driver resolves PCIe root complex topology by reading sysfs symlinks. +The rendered tree now reaches served containers at `/sys/bus/pci/devices` (see +[PCI sysfs in containers](#pci-sysfs-in-containers)), so a driver the mock +serves resolves the root complex; one deployed outside the CDI and NRI paths +still reads the host's sysfs and logs the warning above, with GPUs allocatable +but `dra.k8s.io/pcieRoot` absent. **What this blocks:** DRA topology-aware scheduling that uses `pcieRoot` to align devices on the same PCIe root complex — for example, co-scheduling a GPU diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml index 210b69cef..97388cfaa 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml @@ -434,10 +434,10 @@ infiniband: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -455,3 +455,12 @@ pcie_topology: - "0000:90:00.0" - "0000:B7:00.0" - "0000:BD:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name +# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery +# derives nvidia.com/gpu.machine instead of labelling the node "unknown". +# The string is representative of the platform, not read off real hardware. +# ============================================================================= +dmi: + product_name: "DGXA100" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml index a92c03c0a..b3278ff62 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml @@ -431,10 +431,10 @@ infiniband: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -452,3 +452,12 @@ pcie_topology: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name +# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery +# derives nvidia.com/gpu.machine instead of labelling the node "unknown". +# The string is representative of the platform, not read off real hardware. +# ============================================================================= +dmi: + product_name: "NVIDIA DGX B200" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml index f1975350d..58a608599 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml @@ -513,10 +513,10 @@ infiniband: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -540,3 +540,12 @@ pcie_topology: devices: - "0000:CA:00.0" - "0000:CB:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name +# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery +# derives nvidia.com/gpu.machine instead of labelling the node "unknown". +# The string is representative of the platform, not read off real hardware. +# ============================================================================= +dmi: + product_name: "NVIDIA GB200 NVL72" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml index 4167388af..90a831919 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml @@ -540,3 +540,12 @@ pcie_topology: devices: - "0000:CA:00.0" - "0000:CB:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name +# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery +# derives nvidia.com/gpu.machine instead of labelling the node "unknown". +# The string is representative of the platform, not read off real hardware. +# ============================================================================= +dmi: + product_name: "NVIDIA GB300 NVL72" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml index a74180be1..3d16f6e6a 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml @@ -444,10 +444,10 @@ infiniband: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -465,3 +465,12 @@ pcie_topology: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name +# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery +# derives nvidia.com/gpu.machine instead of labelling the node "unknown". +# The string is representative of the platform, not read off real hardware. +# ============================================================================= +dmi: + product_name: "DGXH100" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml index b38ad30c3..b75e2437d 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml @@ -391,10 +391,10 @@ infiniband: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -412,3 +412,14 @@ pcie_topology: - "0000:B2:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity, deliberately unset: an L40S ships in commodity OEM +# servers whose product_name is vendor-specific, so any value here would be +# invented. GPU Feature Discovery therefore labels nvidia.com/gpu.machine +# "unknown" — the same answer a real node gives when DMI is unreadable. Set +# one to pin a machine type: +# +# dmi: +# product_name: "PowerEdge R760xa" +# ============================================================================= diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml index 44a826903..e76aa66b2 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml @@ -369,10 +369,10 @@ infiniband: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -383,3 +383,14 @@ pcie_topology: - "0000:86:00.0" - "0000:AF:00.0" - "0000:D8:00.0" + +# ============================================================================= +# DMI (SMBIOS) identity, deliberately unset: T4s ship in commodity OEM +# servers and cloud VMs whose product_name is vendor-specific, so any value +# here would be invented. GPU Feature Discovery therefore labels +# nvidia.com/gpu.machine "unknown" — the same answer a real node gives when +# DMI is unreadable. Set one to pin a machine type: +# +# dmi: +# product_name: "PowerEdge R740" +# ============================================================================= diff --git a/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml b/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml index e952569e8..5c57508f5 100644 --- a/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml @@ -60,6 +60,12 @@ spec: # warns and injects everything else. - --imex-channel-annotation={{ .Values.nri.imexChannelAnnotation }} - --imex-channel-host-path={{ .Values.nri.overlay.hostPath }}/driver/dev/nvidia-caps-imex-channels + {{- if not .Values.nri.pciSysfsMounts }} + # nri.pciSysfsMounts=false: keep the host's /sys/devices visible in + # served containers, at the cost of sysfs-reading consumers (GFD, + # the DRA driver) finding no mock GPUs. + - --disable-pci-sysfs-mounts + {{- end }} - --excluded-namespaces={{ .Release.Namespace }},kube-system{{- range .Values.nri.excludedNamespaces }},{{ . }}{{- end }} # NODE_NAME (below) gates ComputeDomain topology injection: when # topology.enabled stages a topology document into the overlay, diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap index 6d58c6d6f..0e412040e 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap @@ -436,10 +436,10 @@ should match snapshot with b200 profile: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -457,6 +457,15 @@ should match snapshot with b200 profile: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name + # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery + # derives nvidia.com/gpu.machine instead of labelling the node "unknown". + # The string is representative of the platform, not read off real hardware. + # ============================================================================= + dmi: + product_name: "NVIDIA DGX B200" kind: ConfigMap metadata: labels: @@ -907,10 +916,10 @@ should match snapshot with default a100 profile: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -928,6 +937,15 @@ should match snapshot with default a100 profile: - "0000:90:00.0" - "0000:B7:00.0" - "0000:BD:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name + # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery + # derives nvidia.com/gpu.machine instead of labelling the node "unknown". + # The string is representative of the platform, not read off real hardware. + # ============================================================================= + dmi: + product_name: "DGXA100" kind: ConfigMap metadata: labels: @@ -1457,10 +1475,10 @@ should match snapshot with gb200 profile: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -1484,6 +1502,15 @@ should match snapshot with gb200 profile: devices: - "0000:CA:00.0" - "0000:CB:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name + # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery + # derives nvidia.com/gpu.machine instead of labelling the node "unknown". + # The string is representative of the platform, not read off real hardware. + # ============================================================================= + dmi: + product_name: "NVIDIA GB200 NVL72" kind: ConfigMap metadata: labels: @@ -2040,6 +2067,15 @@ should match snapshot with gb300 profile: devices: - "0000:CA:00.0" - "0000:CB:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name + # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery + # derives nvidia.com/gpu.machine instead of labelling the node "unknown". + # The string is representative of the platform, not read off real hardware. + # ============================================================================= + dmi: + product_name: "NVIDIA GB300 NVL72" kind: ConfigMap metadata: labels: @@ -2500,10 +2536,10 @@ should match snapshot with h100 profile: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -2521,6 +2557,15 @@ should match snapshot with h100 profile: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name + # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery + # derives nvidia.com/gpu.machine instead of labelling the node "unknown". + # The string is representative of the platform, not read off real hardware. + # ============================================================================= + dmi: + product_name: "DGXH100" kind: ConfigMap metadata: labels: @@ -2928,10 +2973,10 @@ should match snapshot with l40s profile: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -2949,6 +2994,17 @@ should match snapshot with l40s profile: - "0000:B2:00.0" - "0000:CA:00.0" - "0000:CB:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity, deliberately unset: an L40S ships in commodity OEM + # servers whose product_name is vendor-specific, so any value here would be + # invented. GPU Feature Discovery therefore labels nvidia.com/gpu.machine + # "unknown" — the same answer a real node gives when DMI is unreadable. Set + # one to pin a machine type: + # + # dmi: + # product_name: "PowerEdge R760xa" + # ============================================================================= kind: ConfigMap metadata: labels: @@ -3334,10 +3390,10 @@ should match snapshot with t4 profile: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -3348,6 +3404,17 @@ should match snapshot with t4 profile: - "0000:86:00.0" - "0000:AF:00.0" - "0000:D8:00.0" + + # ============================================================================= + # DMI (SMBIOS) identity, deliberately unset: T4s ship in commodity OEM + # servers and cloud VMs whose product_name is vendor-specific, so any value + # here would be invented. GPU Feature Discovery therefore labels + # nvidia.com/gpu.machine "unknown" — the same answer a real node gives when + # DMI is unreadable. Set one to pin a machine type: + # + # dmi: + # product_name: "PowerEdge R740" + # ============================================================================= kind: ConfigMap metadata: labels: diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap index 98f9fba4f..52c85b29a 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap @@ -19,7 +19,7 @@ should match snapshot with all overrides: template: metadata: annotations: - checksum/config: 1893be873c970041a080b07af3cc840fb218a12192a6f0e5c305ba70a12ce620 + checksum/config: 13fe7d8201b73a6ab5a79ba31e4808f7061fd6c0e394cdcb6483c98c14413fa9 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: custom @@ -166,7 +166,7 @@ should match snapshot with b200 profile: template: metadata: annotations: - checksum/config: 25749368d74f6ae7ce538ddfa371e30c93ec11a4968b155c683b914e05f76b81 + checksum/config: 76e0a95c4f16f54fd41bcd170108cb3f5793fd9e48247f17899663db076dd946 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME @@ -292,7 +292,7 @@ should match snapshot with default values: template: metadata: annotations: - checksum/config: 4a1427c36fb9a24923ec692eb82cd2ba449715be5e194942ae26549a1d1f81b3 + checksum/config: 2e3e92a6183a58b6d99935e91924bf340cdbf32b5118700cbbf74a305bd50e92 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml b/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml index f895fbad6..62bd944e1 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml @@ -285,6 +285,28 @@ tests: path: /var/run/cdi type: DirectoryOrCreate + - it: should mount the PCI sysfs tree by default and allow opting out + set: + nri: + enabled: true + asserts: + # The flag is an opt-OUT: absent means the tree is mounted, which is what + # sysfs-reading consumers need. Its presence by default would silently + # leave GFD labelling gpu.mode=unknown. + - notContains: + path: spec.template.spec.containers[0].args + content: --disable-pci-sysfs-mounts + + - it: should drop the PCI sysfs mounts when pciSysfsMounts is false + set: + nri: + enabled: true + pciSysfsMounts: false + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --disable-pci-sysfs-mounts + - it: should switch to CDI injection and follow a relocated CDI spec dir set: nri: diff --git a/deployments/nvml-mock/helm/nvml-mock/values.yaml b/deployments/nvml-mock/helm/nvml-mock/values.yaml index 482336158..30c3affd6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/values.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/values.yaml @@ -191,6 +191,17 @@ nri: # imex.mockChannels.channelCount adds real container-start cost: creating the # default 2048 nodes measures around one second. imexChannelAnnotation: nvml-mock.nvidia.com/imex-channels + # Mount the rendered PCI sysfs tree at /sys/bus/pci/devices and /sys/devices + # in every container the plugin serves. Consumers written in Go read sysfs + # with direct syscalls, which the LD_PRELOAD redirector cannot intercept, so + # without these mounts they see no GPUs (gpu.mode / pcieRoot go unknown). + # + # The cost: /sys/devices is mounted whole, hiding the host's other device + # classes (CPU topology among them) from those containers. It cannot be + # narrowed to the profile's root complexes, because a bind mount at a path + # sysfs lacks needs a mountpoint the runtime cannot create on a read-only + # /sys. Set to false on a cluster that cannot accept that. + pciSysfsMounts: true excludedNamespaces: [] resources: {} # Port serving the plugin's /healthz and /readyz endpoints. The NRI DaemonSet diff --git a/deployments/nvml-mock/scripts/setup.sh b/deployments/nvml-mock/scripts/setup.sh index 4e5394167..f7d70bae4 100644 --- a/deployments/nvml-mock/scripts/setup.sh +++ b/deployments/nvml-mock/scripts/setup.sh @@ -114,14 +114,41 @@ CAPS_EOF echo "Mock IMEX surface ready: $IMEX_CHANNELS channels, major $IMEX_MAJOR, proc-devices at $IMEX_DIR/proc-devices" fi -# 3b. Generate CDI spec for nvidia-container-runtime CDI mode. +# 3b. Render fake PCI sysfs tree (consumed by topology-aware DRA / device +# plugins that resolve PCIe root complex via a readlink on +# /sys/bus/pci/devices/, and by GPU Feature Discovery, which reads +# each device's `vendor` / `class` to derive nvidia.com/gpu.mode). The +# renderer parses the profile's `pcie_topology:` block; profiles without +# one get a flat default covering every device under a single root +# complex (`pci0000:00`, NUMA 0). It also writes the profile's `dmi:` +# product name, which GFD turns into nvidia.com/gpu.machine. Failures +# are fatal under `set -e` for the same reason as the IB render below — +# a topology typo otherwise yields silently malformed sysfs that +# downstream `dra.k8s.io/pcieRoot` attributes would inherit. +# +# This runs before the CDI spec below because that spec bind-mounts the +# rendered directories into consumers: a bind mount whose source is +# missing fails container creation for the whole pod. +PCI_ROOT="$HOST" +mkdir -p "$PCI_ROOT" +PCI_SYSFS_RENDERED=off +if [ -x /usr/local/bin/render-pci-sysfs ]; then + /usr/local/bin/render-pci-sysfs \ + --config /etc/nvml-mock/config.yaml \ + --output "$PCI_ROOT" + if [ -d "$PCI_ROOT/sys/bus/pci/devices" ] && [ -d "$PCI_ROOT/sys/devices" ]; then + PCI_SYSFS_RENDERED=on + fi +fi + +# 3c. Generate CDI spec for nvidia-container-runtime CDI mode. # This allows the toolkit to inject our mock libs into containers without # needing libnvidia-container or kernel modules. CDI_DIR=/host/var/run/cdi mkdir -p "$CDI_DIR" # Resolve fabricmanager enablement once, here, because it influences both the -# CDI spec (below) and the daemon launch (step 11). Validate early so a typo +# CDI spec (below) and the daemon launch (step 10). Validate early so a typo # fails the pod with a clear message rather than silently disabling the gate. MOCK_FM_MODE=$(printf '%s' "${MOCK_FABRICMANAGER:-off}" | tr '[:upper:]' '[:lower:]') case "$MOCK_FM_MODE" in @@ -164,6 +191,33 @@ containerEdits: options: [ro, nosuid, nodev, bind] CDI_HEADER +# Fake PCI sysfs, mounted at the kernel paths. Consumers written in Go +# (GPU Feature Discovery, the DRA driver) read sysfs with direct syscalls, +# so the LD_PRELOAD redirector never sees their opens and MOCK_PCI_ROOT +# does nothing for them — only a real mount at /sys/bus/pci/devices works. +# Without this, GFD resolves a mock GPU's BDF from NVML, fails to read +# /sys/bus/pci/devices//vendor, and labels the node +# nvidia.com/gpu.mode=unknown. +# +# Both mounts are needed: the entries under sys/bus/pci/devices are +# relative symlinks into ../../../devices/pciDDDD:BB, which only resolve +# when the rendered sys/devices is mounted too. That second mount hides the +# host's other device classes (CPU topology among them) from the container. +# The alternative — mounting only the root complexes the profile declares — +# is not available: sysfs is read-only inside the container, so the runtime +# cannot create a mountpoint like /sys/devices/pci0000:80 that the host +# does not already have, and container creation fails outright. +if [ "$PCI_SYSFS_RENDERED" = "on" ]; then + cat >> "$CDI_DIR/nvidia.yaml" << PCI_SYSFS_MOUNT_EOF + - hostPath: /var/lib/nvml-mock/sys/devices + containerPath: /sys/devices + options: [ro, nosuid, nodev, bind] + - hostPath: /var/lib/nvml-mock/sys/bus/pci/devices + containerPath: /sys/bus/pci/devices + options: [ro, nosuid, nodev, bind] +PCI_SYSFS_MOUNT_EOF +fi + # When fabricmanager is enabled, bind-mount the node-local readiness marker # directory into CDI-injected workloads and point the mock NVML library at it. # Without this, the mock .so loaded inside user pods sees an empty @@ -252,7 +306,7 @@ done echo "CDI spec generated at $CDI_DIR/nvidia.yaml ($GPU_COUNT devices, index + UUID keyed)" -# 3c. Generate the CDI spec the NRI plugin injects (issue #436). +# 3d. Generate the CDI spec the NRI plugin injects (issue #436). # # This is deliberately a SECOND spec, not a reuse of nvidia.yaml above: # @@ -547,7 +601,7 @@ if [ "$PCI_LABEL_MODE" = "on" ]; then # step 8's /host/run/nvidia/driver symlink, crash-looping the whole mock for # an optional, gated feature. When the write fails no label appears, which is # the honest state (#505), and nothing downstream is corrupted — unlike the - # IB and PCI renders in steps 9 and 10, which are deliberately fatal because + # IB render (step 9) and PCI render (step 3b), which are deliberately fatal because # a partial tree silently misleads its consumers. A failing `if` CONDITION # does not trip `set -e`, so this form warns and continues rather than # swallowing the error the way `|| true` would. @@ -662,24 +716,7 @@ if [ "$MOCK_IB_MODE" != "off" ] && [ -x /usr/local/bin/mock-ib ]; then fi fi -# 10. Render fake PCI sysfs tree (consumed by topology-aware DRA / device -# plugins that resolve PCIe root complex via a readlink on -# /sys/bus/pci/devices/). The renderer parses the profile's -# `pcie_topology:` block; profiles without one get a flat default -# covering every device under a single root complex (`pci0000:00`, -# NUMA 0). Failures are fatal under `set -e` for the same reason as -# the IB block above — a topology typo otherwise yields silently -# malformed sysfs that downstream `dra.k8s.io/pcieRoot` attributes -# would inherit. -PCI_ROOT="$HOST" -mkdir -p "$PCI_ROOT" -if [ -x /usr/local/bin/render-pci-sysfs ]; then - /usr/local/bin/render-pci-sysfs \ - --config /etc/nvml-mock/config.yaml \ - --output "$PCI_ROOT" -fi - -# 11. Fabric Manager: on NVSwitch platforms (HGX H100 / GB200 / GB300) the +# 10. Fabric Manager: on NVSwitch platforms (HGX H100 / GB200 / GB300) the # real nvidia-fabricmanager registers the GPUs with the NVSwitch fabric # before they are usable. When MOCK_FABRICMANAGER is enabled we start the # fake daemon, which writes a node-local readiness marker under diff --git a/local/gpu-operator/gpu-operator.values.yaml b/local/gpu-operator/gpu-operator.values.yaml index cbe340c72..b76c5796b 100644 --- a/local/gpu-operator/gpu-operator.values.yaml +++ b/local/gpu-operator/gpu-operator.values.yaml @@ -58,11 +58,21 @@ devicePlugin: value: "/var/lib/nvml-mock/driver" # GFD reads GPU attributes via mock NVML and labels the node. +# +# GFD_MACHINE_TYPE_FILE moves the machine-type read off the default +# /sys/class/dmi/id/product_name. The mock renders its DMI identity at the +# path that symlink points to, and only /sys/devices can be handed to a +# container (a bind mount at /sys/class/dmi would need a mountpoint the +# runtime cannot create on a read-only sysfs, and Kind nodes on Docker +# Desktop have no /sys/class/dmi at all). Without this the label falls back +# to the host's machine type, or to "unknown" when the host exposes no DMI. gfd: enabled: true env: - name: NVIDIA_DRIVER_ROOT value: "/var/lib/nvml-mock/driver" + - name: GFD_MACHINE_TYPE_FILE + value: "/sys/devices/virtual/dmi/id/product_name" # Validator checks that the GPU stack is functional. # diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml index c0cace998..259a5e9b9 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml @@ -398,10 +398,10 @@ nvlink: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml index d5f9d2881..92d453f16 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml @@ -390,10 +390,10 @@ nvlink: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml index ca9bb033b..4b71213ac 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml @@ -432,10 +432,10 @@ nvlink: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml index c6d7778b0..4880bf65a 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml @@ -410,10 +410,10 @@ nvlink: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml index d2cacb8c0..07e5c9a31 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml @@ -365,10 +365,10 @@ devices: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml index 10c6df2f2..c45f25628 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml @@ -337,10 +337,10 @@ devices: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index 84ea462f8..41d3b0caa 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -42,6 +42,20 @@ const ( // MOCK_TOPOLOGY_CONFIG env). defaultTopologyRelPath = "topology/topology.yaml" + // pciDevicesRelPath and sysDevicesRelPath are the two halves of the fake + // PCI sysfs tree setup.sh renders into the overlay, resolved relative to + // the host overlay path. + pciDevicesRelPath = "sys/bus/pci/devices" + sysDevicesRelPath = "sys/devices" + // pciDevicesContainerPath and sysDevicesContainerPath are the kernel + // paths the tree must appear at inside the container. Unlike the + // LD_PRELOAD-based redirection (MOCK_PCI_ROOT), these cannot be + // relocated: Go consumers such as GPU Feature Discovery and the DRA + // driver hard-code them and read them with direct syscalls, which no + // libc shim can intercept. + pciDevicesContainerPath = "/sys/bus/pci/devices" + sysDevicesContainerPath = "/sys/devices" + // DeviceInjectionModeRaw stages the mock /dev/nvidiaN nodes directly in the // adjustment. It is the default: MEP-0002 requires the raw path to stay // reachable, and it is the only mode that works on a runtime whose CDI @@ -110,6 +124,14 @@ type Config struct { // container creation. CDISpecHostPath string + // DisablePCISysfsMounts drops the fake PCI sysfs mounts from the + // adjustment. The zero value keeps them, because a consumer that reads + // sysfs directly sees no GPUs without them. It exists for clusters that + // cannot accept the host's /sys/devices being shadowed in every served + // container — the unavoidable cost of those mounts, explained in + // pciSysfsMounts. + DisablePCISysfsMounts bool + // NodeName is the Kubernetes node this plugin runs on. When set (and a // topology document is staged in the overlay) it is injected as the // default NODE_NAME so the mock NVML engine's ComputeDomain topology @@ -207,6 +229,7 @@ func Adjust(cfg Config, container Container) (Adjustment, bool, error) { }, Env: buildEnv(cfg, container.Env, topologyInjectable(cfg)), } + adjustment.Mounts = append(adjustment.Mounts, pciSysfsMounts(cfg)...) if strings.EqualFold(container.PodAnnotations[cfg.DeviceAnnotation], "true") { switch { @@ -338,6 +361,56 @@ func topologyInjectable(cfg Config) bool { return err == nil } +// pciSysfsMounts maps the rendered PCI tree onto the kernel paths inside the +// container. It returns both mounts or neither: /sys/bus/pci/devices holds +// relative symlinks into ../../../devices/pciDDDD:BB, so without +// /sys/devices every entry dangles and reads fail with ENOENT — the same +// symptom as no mount at all, only harder to diagnose. +// +// Mounting /sys/devices necessarily hides the host's other device classes +// (CPU topology among them) from the container. That is the price of serving +// consumers that resolve GPUs through sysfs: the tree cannot be assembled +// per root complex instead, because a bind mount at a path sysfs does not +// already have (say /sys/devices/pci0000:80) needs a mountpoint the runtime +// cannot create on a read-only sysfs. +// +// Missing sources are skipped rather than reported: the tree is staged by +// the main nvml-mock DaemonSet and nothing orders this plugin after it, and +// a bind mount whose source does not exist fails container creation for the +// whole pod. Silence rather than a warning because this runs for every +// container on the node, staged or not. +func pciSysfsMounts(cfg Config) []Mount { + if cfg.DisablePCISysfsMounts || cfg.HostOverlayPath == "" { + return nil + } + sysDevices := filepath.Join(cfg.HostOverlayPath, sysDevicesRelPath) + pciDevices := filepath.Join(cfg.HostOverlayPath, pciDevicesRelPath) + for _, dir := range []string{sysDevices, pciDevices} { + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + return nil + } + } + + // /sys/devices first: containerd orders mounts parent-before-child, but + // emitting them in dependency order keeps the adjustment readable and + // correct under any runtime that applies them verbatim. + return []Mount{ + { + Source: sysDevices, + Destination: sysDevicesContainerPath, + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }, + { + Source: pciDevices, + Destination: pciDevicesContainerPath, + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }, + } +} + func shouldSkip(cfg Config, container Container) bool { if strings.EqualFold(container.PodAnnotations[cfg.OptOutAnnotation], "false") { return true diff --git a/pkg/nri/nvmlmock/adjust_test.go b/pkg/nri/nvmlmock/adjust_test.go index fa195b545..ea0707fe9 100644 --- a/pkg/nri/nvmlmock/adjust_test.go +++ b/pkg/nri/nvmlmock/adjust_test.go @@ -48,6 +48,110 @@ func TestAdjustPlainContainerAddsOverlayAndEnvironment(t *testing.T) { requireNoEnvKey(t, adjustment.Env, "MOCK_IB") } +// TestAdjustMountsPCISysfsWhenStaged pins the pair of mounts that let Go +// consumers (GPU Feature Discovery, the DRA driver) see the mock PCI tree. +// They read sysfs with direct syscalls, so the LD_PRELOAD redirector never +// sees their opens and only a real mount at the canonical path works. +// +// Both mounts are required together: /sys/bus/pci/devices holds symlinks +// pointing at ../../../devices/pciDDDD:BB/, which only resolve when +// the rendered sys/devices is mounted too. +func TestAdjustMountsPCISysfsWhenStaged(t *testing.T) { + overlay := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "gpu-operator"}) + require.NoError(t, err) + require.True(t, ok) + + require.Contains(t, adjustment.Mounts, Mount{ + Source: filepath.Join(overlay, "sys/devices"), + Destination: "/sys/devices", + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }) + require.Contains(t, adjustment.Mounts, Mount{ + Source: filepath.Join(overlay, "sys/bus/pci/devices"), + Destination: "/sys/bus/pci/devices", + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }) +} + +// The mounts shadow the host's whole /sys/devices in every container the +// plugin serves, which some clusters cannot accept. Operators there turn them +// off and keep the rest of the injection, at the price of Go consumers seeing +// no mock GPUs. +func TestAdjustOmitsPCISysfsMountsWhenDisabled(t *testing.T) { + overlay := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + cfg.DisablePCISysfsMounts = true + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "gpu-operator"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "no sysfs mount may be emitted when disabled") + } + // The overlay itself must still be injected, or the opt-out would silently + // disable the whole plugin. + require.Contains(t, adjustment.Mounts, Mount{ + Source: overlay, + Destination: cfg.ContainerOverlayPath, + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }) +} + +// TestAdjustSkipsPCISysfsMountsWhenNotStaged is the fail-open case: the +// tree is staged by the main DaemonSet and nothing orders this plugin after +// it. A bind mount with a missing source fails container creation outright, +// so an unstaged node must simply get no sysfs mounts. +func TestAdjustSkipsPCISysfsMountsWhenNotStaged(t *testing.T) { + cfg := DefaultConfig() + cfg.HostOverlayPath = t.TempDir() + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "unstaged node must not get sysfs mounts, got %+v", mount) + } +} + +// TestAdjustSkipsPCIDevicesMountWithoutSysDevices guards the half-rendered +// case. Mounting the symlink directory alone yields dangling symlinks, +// which reads report as ENOENT — the exact failure the mounts exist to +// fix, but harder to diagnose because the entries appear to be there. +func TestAdjustSkipsPCIDevicesMountWithoutSysDevices(t *testing.T) { + overlay := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "a tree without sys/devices must yield no sysfs mounts, got %+v", mount) + } +} + func TestAdjustEmitsOnlyAddedOrChangedEnv(t *testing.T) { container := Container{ Namespace: "default", diff --git a/pkg/system/mockpcisysfs/config/types.go b/pkg/system/mockpcisysfs/config/types.go index d978fa192..f7b5ec11a 100644 --- a/pkg/system/mockpcisysfs/config/types.go +++ b/pkg/system/mockpcisysfs/config/types.go @@ -1,10 +1,11 @@ // Copyright 2026 NVIDIA CORPORATION // SPDX-License-Identifier: Apache-2.0 -// Package config defines the YAML schema for the `pcie_topology:` block -// embedded in mock-nvml profile configs. The renderer consumes this to -// populate a fake `/sys/bus/pci/devices` + `/sys/devices/pciDDDD:BB` tree -// under MOCK_PCI_ROOT. +// Package config defines the YAML schema for the `pcie_topology:` and +// `dmi:` blocks embedded in mock-nvml profile configs. The renderer +// consumes these to populate a fake `/sys/bus/pci/devices` + +// `/sys/devices/pciDDDD:BB` tree and a fake DMI identity under +// MOCK_PCI_ROOT. // // Keeping the schema in a standalone package (no cgo dependency on the // mocknvml engine) mirrors `pkg/network/mockibsysfs/config` and lets the @@ -25,6 +26,27 @@ type Profile struct { DeviceDefaults *DeviceDefaults `json:"device_defaults,omitempty" yaml:"device_defaults,omitempty"` Devices []Device `json:"devices" yaml:"devices"` PCIeTopology *PCIeTopology `json:"pcie_topology,omitempty" yaml:"pcie_topology,omitempty"` + DMI *DMI `json:"dmi,omitempty" yaml:"dmi,omitempty"` +} + +// DMI mirrors the profile's optional `dmi:` block, the SMBIOS identity a +// real node exposes through /sys/class/dmi/id. Only product_name is +// modelled: it is what GPU Feature Discovery reads to derive the +// nvidia.com/gpu.machine label, and it is the only DMI field any mock +// consumer has needed so far. +type DMI struct { + ProductName string `json:"product_name,omitempty" yaml:"product_name,omitempty"` +} + +// DMIProductName returns the profile's SMBIOS product name, or "" when the +// profile declares no `dmi:` block. Callers treat "" as "render no DMI +// identity" rather than substituting a placeholder, so a node keeps +// reporting an honest "unknown" machine type until a profile opts in. +func (p *Profile) DMIProductName() string { + if p.DMI == nil { + return "" + } + return strings.TrimSpace(p.DMI.ProductName) } // DeviceDefaults mirrors the profile's `device_defaults:` block. Only the diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 0c9b8a135..5bd4be9f6 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -57,6 +57,12 @@ type Options struct { // unconditionally). A non-nil Topology with a non-empty Output is // required; otherwise Render returns an error. Output string + + // DMIProductName is the SMBIOS product name to expose as the node's + // machine type, from the profile's `dmi:` block. Empty renders no DMI + // identity at all, leaving consumers to report their own "unknown" + // rather than a placeholder machine. + DMIProductName string } // Render writes the entire tree. It is idempotent: existing directories @@ -64,7 +70,7 @@ type Options struct { // symlinks are removed and recreated so a stale relative target does not // linger across re-renders. func Render(o Options) error { - if o.Topology == nil || len(o.Topology.RootComplexes) == 0 { + if !o.hasTopology() && o.DMIProductName == "" { // Nothing to do — caller decided to render a profile with no // declared topology and no devices. Treat as a no-op so the // renderer can be invoked unconditionally from setup.sh. @@ -74,6 +80,24 @@ func Render(o Options) error { return errors.New("pcisysfs render: Output is required") } + // The DMI identity is independent of the PCI tree: a profile may name its + // machine type without declaring a topology. + if o.DMIProductName != "" { + if err := renderDMI(o.Output, o.DMIProductName); err != nil { + return err + } + } + if !o.hasTopology() { + return nil + } + return renderTopology(o) +} + +func (o Options) hasTopology() bool { + return o.Topology != nil && len(o.Topology.RootComplexes) > 0 +} + +func renderTopology(o Options) error { root := o.Output if err := mkdirAll(root, "sys/bus/pci/devices"); err != nil { return err @@ -90,6 +114,19 @@ func Render(o Options) error { return nil } +// dmiIDDir is where the kernel materializes the SMBIOS identity. The +// familiar /sys/class/dmi/id path is only a symlink into this directory, +// so rendering here is what makes the mock machine type resolvable through +// both paths once the tree is bind-mounted over /sys/devices. +const dmiIDDir = "sys/devices/virtual/dmi/id" + +// renderDMI writes the node's mock machine type. GPU Feature Discovery +// reads this file (default --machine-type-file) to derive +// nvidia.com/gpu.machine; without it a mock node labels itself "unknown". +func renderDMI(root, productName string) error { + return writeFile(root, filepath.Join(dmiIDDir, "product_name"), productName+"\n") +} + func renderRootComplex(root string, rc config.RootComplex, ids map[string]config.PCI) error { rcDir := filepath.Join("sys/devices", rc.ID) if err := mkdirAll(root, rcDir); err != nil { diff --git a/pkg/system/mockpcisysfs/render/render_test.go b/pkg/system/mockpcisysfs/render/render_test.go index a14016ea1..c576f075b 100644 --- a/pkg/system/mockpcisysfs/render/render_test.go +++ b/pkg/system/mockpcisysfs/render/render_test.go @@ -216,6 +216,59 @@ func TestRender_NormalizesUppercaseBDF(t *testing.T) { require.NoError(t, err, "expected lowercase symlink") } +// TestRender_DMIProductName pins the DMI identity at the path the kernel +// exposes it, /sys/devices/virtual/dmi/id/product_name. It lives under +// sys/devices (not sys/class) on purpose: /sys/class/dmi/id is itself a +// symlink into that directory, and sys/devices is the subtree consumers +// can be handed as a bind mount, so a container that sees the mock tree +// resolves the mock product name through either path. +func TestRender_DMIProductName(t *testing.T) { + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{ + Topology: topo, + Output: dir, + DMIProductName: "NVIDIA GB200 NVL72", + }), "Render") + + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) + require.NoError(t, err, "read product_name") + // Trailing newline mirrors the kernel; GFD trims it before labelling. + require.Equal(t, "NVIDIA GB200 NVL72\n", string(got), "product_name") +} + +func TestRender_NoDMIWhenProductNameEmpty(t *testing.T) { + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir}), "Render") + + _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual")) + require.True(t, os.IsNotExist(err), + "profiles without a dmi: block must render no DMI identity, got err=%v", err) +} + +// TestRender_DMIWithoutTopology covers a profile that declares a machine +// type but no devices: the DMI identity is independent of the PCI tree, so +// it must still land. +func TestRender_DMIWithoutTopology(t *testing.T) { + dir := t.TempDir() + require.NoError(t, Render(Options{Output: dir, DMIProductName: "DGXA100"}), "Render") + + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) + require.NoError(t, err, "read product_name") + require.Equal(t, "DGXA100\n", string(got), "product_name") +} + // --- Config / Validate tests -------------------------------------------------- func TestValidate_AcceptsCanonicalProfile(t *testing.T) { @@ -362,6 +415,24 @@ func TestDeviceIdentities_NoDefaults(t *testing.T) { require.Equal(t, uint32(0), ids["0000:07:00.0"].DeviceID) } +func TestDMIProductName_ParsesProfileBlock(t *testing.T) { + var p config.Profile + require.NoError(t, yaml.Unmarshal([]byte(` +devices: + - index: 0 + pci: + bus_id: "0000:07:00.0" +dmi: + product_name: " DGXA100 " +`), &p), "unmarshal") + require.Equal(t, "DGXA100", p.DMIProductName(), "product_name should be trimmed") +} + +func TestDMIProductName_EmptyWithoutBlock(t *testing.T) { + p := config.Profile{} + require.Empty(t, p.DMIProductName(), "profiles without dmi: declare no machine type") +} + func TestEffectiveTopology_PrefersExplicit(t *testing.T) { p := config.Profile{ Devices: []config.Device{{Index: 0, PCI: config.PCI{BusID: "0000:07:00.0"}}}, diff --git a/tests/e2e/go/assertions/gfd_labels.go b/tests/e2e/go/assertions/gfd_labels.go index b36a3f245..cf5dae4c9 100644 --- a/tests/e2e/go/assertions/gfd_labels.go +++ b/tests/e2e/go/assertions/gfd_labels.go @@ -22,8 +22,22 @@ const ( GFDLabelProduct = "nvidia.com/gpu.product" GFDLabelMemory = "nvidia.com/gpu.memory" GFDLabelCount = "nvidia.com/gpu.count" + GFDLabelMachine = "nvidia.com/gpu.machine" ) +// ExpectedMachineTypeLabel is the gpu.machine label GFD publishes from the +// mock DMI identity. It is separate from ExpectedGFDLabels because those come +// from NVML and were already correct while this one read "unknown": it +// requires the rendered sysfs tree to reach the GFD container (#673). +// +// gpu.mode is deliberately NOT asserted here. It also needs the tree, but GFD +// derives the BDF it looks up from NVML's busId, which the mock reports in the +// wrong domain width (#671) — so mode stays "unknown" until that lands, and +// asserting it would couple this expectation to an unrelated fix. +func ExpectedMachineTypeLabel(machineType string) map[string]string { + return map[string]string{GFDLabelMachine: machineType} +} + // ExpectedGFDLabels derives the GFD labels a node must carry from the profile, // rather than from the node itself. Deriving them independently is what makes // the assertion discriminating: if GFD stopped reading the mock NVML, the count diff --git a/tests/e2e/go/assertions/gfd_labels_test.go b/tests/e2e/go/assertions/gfd_labels_test.go index 46aee6d21..00fa655a4 100644 --- a/tests/e2e/go/assertions/gfd_labels_test.go +++ b/tests/e2e/go/assertions/gfd_labels_test.go @@ -40,6 +40,25 @@ func TestExpectedGFDLabelsCountTracksProfileNotAConstant(t *testing.T) { assert.NotEqual(t, four[GFDLabelProduct], eight[GFDLabelProduct]) } +func TestExpectedMachineTypeLabelCarriesTheProfileMachine(t *testing.T) { + t.Parallel() + + want := ExpectedMachineTypeLabel("NVIDIA-GB200-NVL72") + + assert.Equal(t, "NVIDIA-GB200-NVL72", want[GFDLabelMachine]) +} + +// Profiles without a dmi: block expect the literal "unknown" rather than no +// assertion at all, so the spec still fails if gpu.machine goes missing. +func TestExpectedMachineTypeLabelKeepsUnknownAsAnExpectation(t *testing.T) { + t.Parallel() + + want := ExpectedMachineTypeLabel("unknown") + + problems := DiffGFDLabels(want, map[string]string{}) + assert.Len(t, problems, 1, "a missing gpu.machine must still be reported") +} + func TestDiffGFDLabelsReportsMissingLabel(t *testing.T) { t.Parallel() diff --git a/tests/e2e/go/assertions/pcisysfs.go b/tests/e2e/go/assertions/pcisysfs.go index 18b0644f6..637250d9d 100644 --- a/tests/e2e/go/assertions/pcisysfs.go +++ b/tests/e2e/go/assertions/pcisysfs.go @@ -65,6 +65,60 @@ func PCISysfs(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount, ex "distinct PCI root complexes\n%s", roots.Combined()) } +// KernelPCIDevicesDir is where the kernel exposes PCI devices and where the +// mock tree must appear for consumers that cannot be redirected: Go binaries +// read sysfs with direct syscalls, so the libpcimocksys.so shim never sees +// their opens and MOCK_PCI_ROOT has no effect on them. +const KernelPCIDevicesDir = "/sys/bus/pci/devices" + +// KernelDMIProductNameFile is the machine-type file GFD reads, at the location +// /sys/class/dmi/id points to. The mock renders it inside the sys/devices +// subtree because that is the only part of sysfs that can be bind-mounted into +// a container. +const KernelDMIProductNameFile = "/sys/devices/virtual/dmi/id/product_name" + +// PCISysfsAtKernelPath asserts, from inside a container the mock serves (a GPU +// Operator operand, for instance), that the rendered tree arrived at the real +// kernel paths rather than only in the overlay: +// - /sys/bus/pci/devices holds exactly the mock GPUs, so the host's own PCI +// devices are masked and consumers enumerate the profile, +// - reading a device's `vendor` yields NVIDIA, which only works when +// /sys/devices is mounted too (the entries are relative symlinks into it), +// - the DMI product name is the profile's, not the host's. +// +// machineType is the raw `dmi.product_name` from the profile; pass "" for a +// profile that declares none, which skips the DMI check. +func PCISysfsAtKernelPath(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount int, machineType string) { + ginkgo.GinkgoHelper() + + ginkgo.By(fmt.Sprintf("%d mock PCI devices visible at %s", gpuCount, KernelPCIDevicesDir)) + res, err := k.ExecSh(ctx, pod, "ls "+KernelPCIDevicesDir+" 2>/dev/null | wc -l") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "listing %s: %s", KernelPCIDevicesDir, res.Combined()) + gomega.Expect(atoiTrim(res.Stdout)).To(gomega.Equal(gpuCount), + "device count at %s — the mock tree is not mounted there\n%s", KernelPCIDevicesDir, res.Combined()) + + ginkgo.By("a device's vendor reads through the symlink into /sys/devices") + first, err := k.ExecSh(ctx, pod, "ls "+KernelPCIDevicesDir+" | sort | head -1") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + dev := strings.TrimSpace(first.Stdout) + gomega.Expect(dev).NotTo(gomega.BeEmpty(), "no PCI devices at %s", KernelPCIDevicesDir) + + vendor, err := k.ExecSh(ctx, pod, "cat "+KernelPCIDevicesDir+"/"+dev+"/vendor") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), + "reading vendor for %s — a dangling symlink means /sys/devices is missing", dev) + gomega.Expect(strings.TrimSpace(vendor.Stdout)).To(gomega.Equal("0x10de"), + "vendor for %s\n%s", dev, vendor.Combined()) + + if machineType == "" { + return + } + ginkgo.By("DMI product name is the profile's machine type") + product, err := k.ExecSh(ctx, pod, "cat "+KernelDMIProductNameFile) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "reading %s", KernelDMIProductNameFile) + gomega.Expect(strings.TrimSpace(product.Stdout)).To(gomega.Equal(machineType), + "mock DMI product name\n%s", product.Combined()) +} + func atoiTrim(s string) int { n, _ := strconv.Atoi(strings.TrimSpace(s)) return n diff --git a/tests/e2e/go/profile/gfd_test.go b/tests/e2e/go/profile/gfd_test.go index 2a4eb1c3a..cc74b8bef 100644 --- a/tests/e2e/go/profile/gfd_test.go +++ b/tests/e2e/go/profile/gfd_test.go @@ -30,6 +30,26 @@ func TestGB200ProfileDerivesObservedGFDLabelValues(t *testing.T) { assert.Equal(t, 8, p.ExpectedGPUs()) } +// gpu.machine comes from the profile's dmi: block, dashed the same way as +// gpu.product. Profiles that ship no block (commodity-server platforms) must +// resolve to the literal "unknown" GFD publishes when it cannot read DMI, so +// the e2e expectation stays an assertion rather than a skip. +func TestGFDMachineTypeComesFromTheDMIBlock(t *testing.T) { + t.Parallel() + + gb200, err := Load(profilesDir, "gb200") + require.NoError(t, err) + a100, err := Load(profilesDir, "a100") + require.NoError(t, err) + t4, err := Load(profilesDir, "t4") + require.NoError(t, err) + + assert.Equal(t, "NVIDIA-GB200-NVL72", gb200.GFDMachineType()) + assert.Equal(t, "DGXA100", a100.GFDMachineType()) + assert.Equal(t, GFDMachineTypeUnknown, t4.GFDMachineType()) + assert.NotContains(t, gb200.GFDMachineType(), " ") +} + func TestGFDProductNameReplacesSpacesWithDashes(t *testing.T) { t.Parallel() diff --git a/tests/e2e/go/profile/profile.go b/tests/e2e/go/profile/profile.go index 9d6a6bafd..d722ccec3 100644 --- a/tests/e2e/go/profile/profile.go +++ b/tests/e2e/go/profile/profile.go @@ -85,6 +85,9 @@ type rawProfile struct { ID string `json:"id"` } `json:"root_complexes"` } `json:"pcie_topology"` + DMI *struct { + ProductName string `json:"product_name"` + } `json:"dmi"` } // rawPlatform decodes a platform block, which appears both under @@ -129,6 +132,7 @@ type Profile struct { pciRoots int memoryBytes int64 + dmiProductName string architecture string shutdownThresholdC int slowdownThresholdC int @@ -156,6 +160,28 @@ func (p Profile) GFDProductName() string { // nvidia.com/gpu.memory. func (p Profile) MemoryMiB() int { return int(p.memoryBytes / bytesPerMiB) } +// GFDMachineTypeUnknown is what GPU Feature Discovery publishes when it cannot +// read a machine type. Profiles that declare no `dmi:` block (l40s, t4) expect +// exactly this, so the assertion stays meaningful for them rather than being +// skipped. +const GFDMachineTypeUnknown = "unknown" + +// DMIProductName is the profile's `dmi.product_name` verbatim — the string the +// renderer writes into the mock sysfs tree — or "" when the profile declares no +// `dmi:` block. Use GFDMachineType for the label form. +func (p Profile) DMIProductName() string { return p.dmiProductName } + +// GFDMachineType is the profile's `dmi.product_name` in the form GFD publishes +// as nvidia.com/gpu.machine: spaces become dashes, as for gpu.product. The +// renderer writes the same string into the mock sysfs tree, so this is what +// GFD reads back through its machine-type file. +func (p Profile) GFDMachineType() string { + if p.dmiProductName == "" { + return GFDMachineTypeUnknown + } + return strings.ReplaceAll(p.dmiProductName, " ", "-") +} + // Load reads profilesDir/.yaml and returns the typed Profile. func Load(profilesDir, name string) (Profile, error) { path := filepath.Join(profilesDir, name+".yaml") @@ -187,6 +213,9 @@ func Load(profilesDir, name string) (Profile, error) { architecture: strings.ToLower(strings.TrimSpace(raw.DeviceDefaults.Architecture)), } p.applyOptionalDeviceDefaults(raw) + if raw.DMI != nil { + p.dmiProductName = strings.TrimSpace(raw.DMI.ProductName) + } // render-pci-sysfs falls back to a flat single-root layout when a profile // declares no pcie_topology block, so an empty list still means 1 root. p.pciRoots = len(raw.PCIeTopology.RootComplexes) diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index ccfa7c456..c860c79a7 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -18,6 +18,7 @@ import ( "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/config" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/harness" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/helm" + "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/kube" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/runner" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/profile" ) @@ -71,9 +72,28 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( assertions.WaitGFDLabels(ctx, h.Kube, node, assertions.ExpectedGFDLabels(p.GFDProductName(), p.MemoryMiB(), p.ExpectedGPUs()), config.ReadyTimeout(), config.PollInterval()) + // gpu.machine comes from the mock DMI identity in sysfs rather + // than NVML, so it read "unknown" while the labels above + // already matched (#673). + assertions.WaitGFDLabels(ctx, h.Kube, node, + assertions.ExpectedMachineTypeLabel(p.GFDMachineType()), + config.ReadyTimeout(), config.PollInterval()) assertions.WaitAllocatableGPU(ctx, h.Kube, node, p.ExpectedGPUs(), config.ReadyTimeout(), config.PollInterval()) }) + It("serves the rendered PCI tree to the GFD container at the kernel paths", Label("device-plugin"), func(ctx SpecContext) { + // The label assertions above cannot distinguish "GFD read the + // mock tree" from "GFD read the host's sysfs and happened to + // agree", and gpu.mode — the label that would fail loudest — + // is blocked on an unrelated busId fix (#671). Reading the + // tree from inside the container pins the delivery itself. + pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") + Expect(err).NotTo(HaveOccurred()) + assertions.PCISysfsAtKernelPath(ctx, h.Kube, + kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}, + p.ExpectedGPUs(), p.DMIProductName()) + }) + It("exports DCGM device metrics that vary over time", Label("dcgm"), func(ctx SpecContext) { assertions.DCGMDeviceMetrics(ctx, h.Kube, gpuOperatorNamespace, p.DisplayName, p.ExpectedGPUs(), gpmProfiles[name], diff --git a/tests/e2e/gpu-operator-values.yaml b/tests/e2e/gpu-operator-values.yaml index 9e08a7416..f9bb1e2ea 100644 --- a/tests/e2e/gpu-operator-values.yaml +++ b/tests/e2e/gpu-operator-values.yaml @@ -77,6 +77,13 @@ gfd: env: - name: NVIDIA_DRIVER_ROOT value: "/var/lib/nvml-mock/driver" + # nvidia.com/gpu.machine source. The mock renders its DMI identity where + # /sys/class/dmi/id points (/sys/devices/virtual/dmi/id), because only + # /sys/devices can be mounted into a container: a mountpoint under + # /sys/class cannot be created on a read-only sysfs, and Kind nodes may + # expose no /sys/class/dmi at all. + - name: GFD_MACHINE_TYPE_FILE + value: "/sys/devices/virtual/dmi/id/product_name" # Validator checks that the GPU stack is functional. # driver-validation: mounts hostPath /run/nvidia/driver → our mock (via symlink). diff --git a/tests/mocknvml/util-test-config.yaml b/tests/mocknvml/util-test-config.yaml index feb692bd9..9cc6d428b 100644 --- a/tests/mocknvml/util-test-config.yaml +++ b/tests/mocknvml/util-test-config.yaml @@ -395,10 +395,10 @@ nvlink: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: From 0a24b2439095015033110968ecdd4f7c89db4881 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Tue, 18 Aug 2026 14:55:20 +0200 Subject: [PATCH 02/11] fix(nvml-mock): let GFD find the mock machine type without configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GFD's default machine-type file is /sys/class/dmi/id/product_name, and /sys/class/dmi/id is a symlink to ../../devices/virtual/dmi/id — inside the subtree the mock already mounts. So setting GFD_MACHINE_TYPE_FILE was redundant wherever the kernel exposes DMI, which is every host the e2e suite runs on in CI. Drop it from the GPU Operator overlays and document the symlink instead, keeping the flag as the fallback for kernels with no DMI at all (Docker Desktop's linuxkit VM), where /sys/class/dmi does not exist and no mount can create it on a read-only sysfs. The gpu.machine assertion moves to its own spec and skips when the container cannot reach DMI through GFD's default path: the label is then "unknown" for a reason the mock does not control. Reading the rendered file from inside the container still runs unconditionally, so delivery stays pinned everywhere. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 15 ++++---- .../nvml-mock/helm/nvml-mock/README.md | 24 +++++++++---- local/gpu-operator/gpu-operator.values.yaml | 15 ++++---- tests/e2e/go/assertions/pcisysfs.go | 28 ++++++++++++--- tests/e2e/go/scenario_gpu_operator_test.go | 35 +++++++++++++------ tests/e2e/gpu-operator-values.yaml | 12 +++---- 6 files changed, 86 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c6e9e590..5fa12b68a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,12 +42,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `dmi:` block with `product_name`, which the renderer writes to `sys/devices/virtual/dmi/id/product_name` in the mock overlay — the path `/sys/class/dmi/id` points at, and the only one a container can be handed. - Pointing GPU Feature Discovery at it with - `GFD_MACHINE_TYPE_FILE=/sys/devices/virtual/dmi/id/product_name` makes - `nvidia.com/gpu.machine` report the profile's platform (`NVIDIA-GB200-NVL72`, - `DGXA100`, …) instead of the host's or `unknown`. The NVIDIA platform - profiles set it; `l40s` and `t4` deliberately do not, since a commodity - server's machine type is a property of the chassis, not the GPU. (#673) + GPU Feature Discovery needs no configuration to pick it up, since that + symlink resolves into the mounted subtree: `nvidia.com/gpu.machine` reports + the profile's platform (`NVIDIA-GB200-NVL72`, `DGXA100`, …) instead of the + host's. The NVIDIA platform profiles set it; `l40s` and `t4` deliberately do + not, since a commodity server's machine type is a property of the chassis, + not the GPU. On a kernel that exposes no DMI at all (Docker Desktop's + linuxkit VM) `/sys/class/dmi` is absent and cannot be mounted into place, so + the label stays `unknown` unless GFD is pointed straight at the rendered file + with `GFD_MACHINE_TYPE_FILE`. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index bc010168b..75ac04630 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -755,9 +755,23 @@ dmi: The renderer writes it to `/var/lib/nvml-mock/sys/devices/virtual/dmi/id/product_name` — inside the -subtree above, which is where `/sys/class/dmi/id` points on a real node and -the only place a container can be given it. Point GPU Feature Discovery at it -to have `nvidia.com/gpu.machine` reflect the profile instead of the host: +subtree mounted above, which is where `/sys/class/dmi/id` points on a real +node and the only place a container can be given it. + +GPU Feature Discovery needs no configuration to find it: its default +machine-type file is `/sys/class/dmi/id/product_name`, and that symlink +resolves into the mounted subtree, so `nvidia.com/gpu.machine` reports the +profile's platform (`NVIDIA-GB200-NVL72`, `DGXA100`, …). + +Two cases read `unknown` instead: + +- The profile ships no `dmi:` block. `l40s` and `t4` deliberately do not, + since a commodity server's machine type is a property of the chassis rather + than the GPU, so nothing is rendered. +- The host kernel exposes no DMI at all, so `/sys/class/dmi` does not exist + and cannot be mounted into place either (a mountpoint cannot be created on a + read-only sysfs). Docker Desktop's linuxkit VM is the common case. Point GFD + straight at the rendered file there: ```yaml gfd: @@ -766,10 +780,6 @@ gfd: value: "/sys/devices/virtual/dmi/id/product_name" ``` -Profiles for commodity servers (`l40s`, `t4`) ship no `dmi:` block, since -their machine type is a property of the chassis rather than the GPU. Nothing -is rendered then, and the label stays `unknown`. - ### Cross-node `ibping` Sysfs mocking alone lets `ibstat` / `iblinkinfo` work, but real `ibping` diff --git a/local/gpu-operator/gpu-operator.values.yaml b/local/gpu-operator/gpu-operator.values.yaml index b76c5796b..21a846653 100644 --- a/local/gpu-operator/gpu-operator.values.yaml +++ b/local/gpu-operator/gpu-operator.values.yaml @@ -59,20 +59,17 @@ devicePlugin: # GFD reads GPU attributes via mock NVML and labels the node. # -# GFD_MACHINE_TYPE_FILE moves the machine-type read off the default -# /sys/class/dmi/id/product_name. The mock renders its DMI identity at the -# path that symlink points to, and only /sys/devices can be handed to a -# container (a bind mount at /sys/class/dmi would need a mountpoint the -# runtime cannot create on a read-only sysfs, and Kind nodes on Docker -# Desktop have no /sys/class/dmi at all). Without this the label falls back -# to the host's machine type, or to "unknown" when the host exposes no DMI. +# nvidia.com/gpu.machine needs no configuration: GFD's default machine-type +# file is /sys/class/dmi/id/product_name, and /sys/class/dmi/id is a symlink +# into /sys/devices/virtual/dmi/id — inside the subtree the mock mounts. On a +# kernel that exposes no DMI at all (Docker Desktop's linuxkit VM) the +# directory is absent and the label reads "unknown"; point +# GFD_MACHINE_TYPE_FILE at /sys/devices/virtual/dmi/id/product_name there. gfd: enabled: true env: - name: NVIDIA_DRIVER_ROOT value: "/var/lib/nvml-mock/driver" - - name: GFD_MACHINE_TYPE_FILE - value: "/sys/devices/virtual/dmi/id/product_name" # Validator checks that the GPU stack is functional. # diff --git a/tests/e2e/go/assertions/pcisysfs.go b/tests/e2e/go/assertions/pcisysfs.go index 637250d9d..9a5325b8e 100644 --- a/tests/e2e/go/assertions/pcisysfs.go +++ b/tests/e2e/go/assertions/pcisysfs.go @@ -71,12 +71,32 @@ func PCISysfs(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount, ex // their opens and MOCK_PCI_ROOT has no effect on them. const KernelPCIDevicesDir = "/sys/bus/pci/devices" -// KernelDMIProductNameFile is the machine-type file GFD reads, at the location -// /sys/class/dmi/id points to. The mock renders it inside the sys/devices -// subtree because that is the only part of sysfs that can be bind-mounted into -// a container. +// KernelDMIProductNameFile is where the mock renders the machine type: the +// location /sys/class/dmi/id points to, and the only part of sysfs that can be +// bind-mounted into a container. const KernelDMIProductNameFile = "/sys/devices/virtual/dmi/id/product_name" +// DefaultMachineTypeFile is where GPU Feature Discovery reads the machine type +// unless GFD_MACHINE_TYPE_FILE says otherwise. On a kernel that exposes DMI it +// is a symlink into KernelDMIProductNameFile's directory, so GFD picks up the +// mock identity with no configuration. +const DefaultMachineTypeFile = "/sys/class/dmi/id/product_name" + +// DMIExposedByKernel reports whether the container can reach DMI through the +// path GFD reads by default. It is false on hosts whose kernel exposes no DMI +// at all — Docker Desktop's linuxkit VM, for one — where /sys/class/dmi does +// not exist and no mount can create it, because a mountpoint cannot be made on +// a read-only sysfs. The gpu.machine label is then "unknown" for reasons that +// have nothing to do with the mock, so callers detect this instead of +// asserting. +func DMIExposedByKernel(ctx context.Context, k *kube.Client, pod kube.PodRef) bool { + ginkgo.GinkgoHelper() + + res, err := k.ExecSh(ctx, pod, "test -e "+DefaultMachineTypeFile+" && echo yes || echo no") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "probing %s: %s", DefaultMachineTypeFile, res.Combined()) + return strings.TrimSpace(res.Stdout) == "yes" +} + // PCISysfsAtKernelPath asserts, from inside a container the mock serves (a GPU // Operator operand, for instance), that the rendered tree arrived at the real // kernel paths rather than only in the overlay: diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index c860c79a7..72c269e75 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -72,20 +72,14 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( assertions.WaitGFDLabels(ctx, h.Kube, node, assertions.ExpectedGFDLabels(p.GFDProductName(), p.MemoryMiB(), p.ExpectedGPUs()), config.ReadyTimeout(), config.PollInterval()) - // gpu.machine comes from the mock DMI identity in sysfs rather - // than NVML, so it read "unknown" while the labels above - // already matched (#673). - assertions.WaitGFDLabels(ctx, h.Kube, node, - assertions.ExpectedMachineTypeLabel(p.GFDMachineType()), - config.ReadyTimeout(), config.PollInterval()) assertions.WaitAllocatableGPU(ctx, h.Kube, node, p.ExpectedGPUs(), config.ReadyTimeout(), config.PollInterval()) }) It("serves the rendered PCI tree to the GFD container at the kernel paths", Label("device-plugin"), func(ctx SpecContext) { - // The label assertions above cannot distinguish "GFD read the - // mock tree" from "GFD read the host's sysfs and happened to - // agree", and gpu.mode — the label that would fail loudest — - // is blocked on an unrelated busId fix (#671). Reading the + // The NVML-derived labels above cannot distinguish "GFD read + // the mock tree" from "GFD read the host's sysfs and happened + // to agree", and gpu.mode — the label that would fail loudest + // — is blocked on an unrelated busId fix (#671). Reading the // tree from inside the container pins the delivery itself. pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") Expect(err).NotTo(HaveOccurred()) @@ -94,6 +88,27 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( p.ExpectedGPUs(), p.DMIProductName()) }) + It("labels the machine type from the mock DMI identity", Label("device-plugin"), func(ctx SpecContext) { + // Nothing configures GFD to find it: its default machine-type + // file, /sys/class/dmi/id/product_name, is a symlink into the + // mounted /sys/devices subtree (#673). + // + // That holds only where the kernel exposes DMI. Where it does + // not — Docker Desktop's linuxkit VM — /sys/class/dmi is absent + // and no mount can create it, so the label reads "unknown" for + // reasons the mock cannot influence and there is nothing to + // assert. + pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") + Expect(err).NotTo(HaveOccurred()) + if !assertions.DMIExposedByKernel(ctx, h.Kube, + kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}) { + Skip("kernel exposes no DMI, so GFD's default machine-type file cannot resolve") + } + assertions.WaitGFDLabels(ctx, h.Kube, node, + assertions.ExpectedMachineTypeLabel(p.GFDMachineType()), + config.ReadyTimeout(), config.PollInterval()) + }) + It("exports DCGM device metrics that vary over time", Label("dcgm"), func(ctx SpecContext) { assertions.DCGMDeviceMetrics(ctx, h.Kube, gpuOperatorNamespace, p.DisplayName, p.ExpectedGPUs(), gpmProfiles[name], diff --git a/tests/e2e/gpu-operator-values.yaml b/tests/e2e/gpu-operator-values.yaml index f9bb1e2ea..1bbbad13e 100644 --- a/tests/e2e/gpu-operator-values.yaml +++ b/tests/e2e/gpu-operator-values.yaml @@ -77,13 +77,11 @@ gfd: env: - name: NVIDIA_DRIVER_ROOT value: "/var/lib/nvml-mock/driver" - # nvidia.com/gpu.machine source. The mock renders its DMI identity where - # /sys/class/dmi/id points (/sys/devices/virtual/dmi/id), because only - # /sys/devices can be mounted into a container: a mountpoint under - # /sys/class cannot be created on a read-only sysfs, and Kind nodes may - # expose no /sys/class/dmi at all. - - name: GFD_MACHINE_TYPE_FILE - value: "/sys/devices/virtual/dmi/id/product_name" + # nvidia.com/gpu.machine is deliberately left unconfigured: GFD's default + # /sys/class/dmi/id/product_name is a symlink into the mock's mounted + # /sys/devices/virtual/dmi/id, so it resolves on any host whose kernel + # exposes DMI. Hosts without it (Docker Desktop) read "unknown", which the + # scenario detects rather than asserts. # Validator checks that the GPU stack is functional. # driver-validation: mounts hostPath /run/nvidia/driver → our mock (via symlink). From 853cea2324a9630dadf964f14abf2d5317ef6731 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Tue, 18 Aug 2026 14:58:46 +0200 Subject: [PATCH 03/11] refactor(nvml-mock): drop the pciSysfsMounts opt-out Serving the rendered PCI tree at the kernel paths is how a mock node presents GPUs to consumers that read sysfs; it is not a tuneable. A node running nvml-mock is simulating GPU hardware, so a knob that turns the simulation half off only adds a way to end up with GFD labelling gpu.mode=unknown for a reason nothing reports. Removes nri.pciSysfsMounts, the --disable-pci-sysfs-mounts flag and its config field. Workloads that need the host's real device tree on an NRI-injected node still have nri.excludedNamespaces, which the README now points at instead. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 5 +-- cmd/nvml-mock-nri/main.go | 19 ------------ cmd/nvml-mock-nri/main_test.go | 18 ----------- .../nvml-mock/helm/nvml-mock/README.md | 6 ++-- .../nvml-mock/templates/nri-daemonset.yaml | 6 ---- .../nvml-mock/tests/nri_daemonset_test.yaml | 22 ------------- .../nvml-mock/helm/nvml-mock/values.yaml | 11 ------- pkg/nri/nvmlmock/adjust.go | 10 +----- pkg/nri/nvmlmock/adjust_test.go | 31 ------------------- 9 files changed, 7 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fa12b68a..2c2a44fa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,8 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `/sys/devices` is necessarily mounted whole — it cannot be narrowed to the profile's root complexes, because a bind mount at a path sysfs lacks needs a mountpoint the runtime cannot create on a read-only `/sys` — which hides the - host's other device classes from served containers. `nri.pciSysfsMounts=false` - opts out. (#673) + host's other device classes from served containers. Under NRI, which injects + node-wide, keep workloads that need the host's real device tree in a namespace + listed in `nri.excludedNamespaces`. (#673) - Profiles may declare the machine type a node of that platform reports, as a `dmi:` block with `product_name`, which the renderer writes to `sys/devices/virtual/dmi/id/product_name` in the mock overlay — the path diff --git a/cmd/nvml-mock-nri/main.go b/cmd/nvml-mock-nri/main.go index 00604bfc0..aa4c41ddd 100644 --- a/cmd/nvml-mock-nri/main.go +++ b/cmd/nvml-mock-nri/main.go @@ -15,7 +15,6 @@ import ( "net/http" "os" "os/signal" - "strconv" "strings" "syscall" "time" @@ -50,7 +49,6 @@ func main() { flag.StringVar(&cfg.DeviceAnnotation, "device-annotation", envOr("NVML_MOCK_DEVICE_ANNOTATION", cfg.DeviceAnnotation), "pod annotation key; value true adds /dev/nvidia* device nodes") flag.StringVar(&cfg.ImexChannelAnnotation, "imex-channel-annotation", envOr("NVML_MOCK_IMEX_CHANNEL_ANNOTATION", cfg.ImexChannelAnnotation), "pod annotation key; value true adds /dev/nvidia-caps-imex-channels/* nodes") flag.StringVar(&cfg.ImexChannelHostPath, "imex-channel-host-path", envOr("NVML_MOCK_IMEX_CHANNEL_HOST_PATH", cfg.ImexChannelHostPath), "host path containing the mock IMEX channel nodes staged by imex.mockChannels (defaults to /driver/dev/nvidia-caps-imex-channels)") - flag.BoolVar(&cfg.DisablePCISysfsMounts, "disable-pci-sysfs-mounts", boolEnvOr("NVML_MOCK_DISABLE_PCI_SYSFS_MOUNTS", cfg.DisablePCISysfsMounts), "do not mount the rendered PCI sysfs tree at /sys/bus/pci/devices and /sys/devices; consumers that read sysfs directly (GPU Feature Discovery, the DRA driver) then see no GPUs") excludedNamespaces := flag.String("excluded-namespaces", envOr("NVML_MOCK_EXCLUDED_NAMESPACES", strings.Join(cfg.ExcludedNamespaces, ",")), "comma-separated namespaces to skip") shims := flag.String("ld-preload-shims", envOr("NVML_MOCK_LD_PRELOAD_SHIMS", strings.Join(cfg.Shims, ",")), "comma-separated LD_PRELOAD shim paths relative to the overlay mount or absolute paths") flag.Parse() @@ -271,23 +269,6 @@ func envOr(key, fallback string) string { return fallback } -// boolEnvOr reads a boolean env override. An unparseable value falls back -// rather than aborting: this only ever gates an optimisation of the injection, -// and a DaemonSet that refused to start over a typo'd "yes" would take the -// whole node's mock GPUs with it. -func boolEnvOr(key string, fallback bool) bool { - value := os.Getenv(key) - if value == "" { - return fallback - } - parsed, err := strconv.ParseBool(value) - if err != nil { - log.Printf("nvml-mock-nri: %s=%q is not a boolean; using %t", key, value, fallback) - return fallback - } - return parsed -} - // major and minor decode a Linux dev_t the way glibc encodes it // (MMMM Mmmm mmmM MMmm): the major occupies bits 8-19 and 44-63, the minor // bits 0-7 and 20-43. diff --git a/cmd/nvml-mock-nri/main_test.go b/cmd/nvml-mock-nri/main_test.go index 1c2b1b5b2..339547a00 100644 --- a/cmd/nvml-mock-nri/main_test.go +++ b/cmd/nvml-mock-nri/main_test.go @@ -411,21 +411,3 @@ func TestEnvOr(t *testing.T) { require.Equal(t, "/fallback", envOr("NVML_MOCK_TEST_KEY", "/fallback")) }) } - -func TestBoolEnvOr(t *testing.T) { - t.Run("parses the environment value", func(t *testing.T) { - t.Setenv("NVML_MOCK_TEST_BOOL", "true") - require.True(t, boolEnvOr("NVML_MOCK_TEST_BOOL", false)) - }) - - t.Run("returns the fallback when unset", func(t *testing.T) { - require.True(t, boolEnvOr("NVML_MOCK_TEST_BOOL_UNSET", true)) - }) - - // A garbled value must not flip the default: the fallback is what keeps the - // node's mock GPUs visible to sysfs consumers. - t.Run("falls back on an unparseable value", func(t *testing.T) { - t.Setenv("NVML_MOCK_TEST_BOOL", "yes-please") - require.False(t, boolEnvOr("NVML_MOCK_TEST_BOOL", false)) - }) -} diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index 75ac04630..1e0da68b9 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -739,9 +739,9 @@ directory yields entries that list but whose every attribute read fails with device classes (CPU topology among them) from those containers. It cannot be narrowed to the profile's root complexes — a bind mount at a path sysfs does not already have needs a mountpoint, and the runtime cannot create one on a -read-only `/sys`. Set `nri.pciSysfsMounts=false` to drop the mounts from the -NRI path on a cluster that cannot accept this; sysfs-reading consumers then -see no GPUs there. +read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so this +is not configurable; keep workloads that need the host's real device tree off +it, or in a namespace listed in `nri.excludedNamespaces`. ### Machine type (`dmi:`) diff --git a/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml b/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml index 5c57508f5..e952569e8 100644 --- a/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/templates/nri-daemonset.yaml @@ -60,12 +60,6 @@ spec: # warns and injects everything else. - --imex-channel-annotation={{ .Values.nri.imexChannelAnnotation }} - --imex-channel-host-path={{ .Values.nri.overlay.hostPath }}/driver/dev/nvidia-caps-imex-channels - {{- if not .Values.nri.pciSysfsMounts }} - # nri.pciSysfsMounts=false: keep the host's /sys/devices visible in - # served containers, at the cost of sysfs-reading consumers (GFD, - # the DRA driver) finding no mock GPUs. - - --disable-pci-sysfs-mounts - {{- end }} - --excluded-namespaces={{ .Release.Namespace }},kube-system{{- range .Values.nri.excludedNamespaces }},{{ . }}{{- end }} # NODE_NAME (below) gates ComputeDomain topology injection: when # topology.enabled stages a topology document into the overlay, diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml b/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml index 62bd944e1..f895fbad6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/tests/nri_daemonset_test.yaml @@ -285,28 +285,6 @@ tests: path: /var/run/cdi type: DirectoryOrCreate - - it: should mount the PCI sysfs tree by default and allow opting out - set: - nri: - enabled: true - asserts: - # The flag is an opt-OUT: absent means the tree is mounted, which is what - # sysfs-reading consumers need. Its presence by default would silently - # leave GFD labelling gpu.mode=unknown. - - notContains: - path: spec.template.spec.containers[0].args - content: --disable-pci-sysfs-mounts - - - it: should drop the PCI sysfs mounts when pciSysfsMounts is false - set: - nri: - enabled: true - pciSysfsMounts: false - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --disable-pci-sysfs-mounts - - it: should switch to CDI injection and follow a relocated CDI spec dir set: nri: diff --git a/deployments/nvml-mock/helm/nvml-mock/values.yaml b/deployments/nvml-mock/helm/nvml-mock/values.yaml index 30c3affd6..482336158 100644 --- a/deployments/nvml-mock/helm/nvml-mock/values.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/values.yaml @@ -191,17 +191,6 @@ nri: # imex.mockChannels.channelCount adds real container-start cost: creating the # default 2048 nodes measures around one second. imexChannelAnnotation: nvml-mock.nvidia.com/imex-channels - # Mount the rendered PCI sysfs tree at /sys/bus/pci/devices and /sys/devices - # in every container the plugin serves. Consumers written in Go read sysfs - # with direct syscalls, which the LD_PRELOAD redirector cannot intercept, so - # without these mounts they see no GPUs (gpu.mode / pcieRoot go unknown). - # - # The cost: /sys/devices is mounted whole, hiding the host's other device - # classes (CPU topology among them) from those containers. It cannot be - # narrowed to the profile's root complexes, because a bind mount at a path - # sysfs lacks needs a mountpoint the runtime cannot create on a read-only - # /sys. Set to false on a cluster that cannot accept that. - pciSysfsMounts: true excludedNamespaces: [] resources: {} # Port serving the plugin's /healthz and /readyz endpoints. The NRI DaemonSet diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index 41d3b0caa..a90451073 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -124,14 +124,6 @@ type Config struct { // container creation. CDISpecHostPath string - // DisablePCISysfsMounts drops the fake PCI sysfs mounts from the - // adjustment. The zero value keeps them, because a consumer that reads - // sysfs directly sees no GPUs without them. It exists for clusters that - // cannot accept the host's /sys/devices being shadowed in every served - // container — the unavoidable cost of those mounts, explained in - // pciSysfsMounts. - DisablePCISysfsMounts bool - // NodeName is the Kubernetes node this plugin runs on. When set (and a // topology document is staged in the overlay) it is injected as the // default NODE_NAME so the mock NVML engine's ComputeDomain topology @@ -380,7 +372,7 @@ func topologyInjectable(cfg Config) bool { // whole pod. Silence rather than a warning because this runs for every // container on the node, staged or not. func pciSysfsMounts(cfg Config) []Mount { - if cfg.DisablePCISysfsMounts || cfg.HostOverlayPath == "" { + if cfg.HostOverlayPath == "" { return nil } sysDevices := filepath.Join(cfg.HostOverlayPath, sysDevicesRelPath) diff --git a/pkg/nri/nvmlmock/adjust_test.go b/pkg/nri/nvmlmock/adjust_test.go index ea0707fe9..a8d25d673 100644 --- a/pkg/nri/nvmlmock/adjust_test.go +++ b/pkg/nri/nvmlmock/adjust_test.go @@ -82,37 +82,6 @@ func TestAdjustMountsPCISysfsWhenStaged(t *testing.T) { }) } -// The mounts shadow the host's whole /sys/devices in every container the -// plugin serves, which some clusters cannot accept. Operators there turn them -// off and keep the rest of the injection, at the price of Go consumers seeing -// no mock GPUs. -func TestAdjustOmitsPCISysfsMountsWhenDisabled(t *testing.T) { - overlay := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) - - cfg := DefaultConfig() - cfg.HostOverlayPath = overlay - cfg.DisablePCISysfsMounts = true - - adjustment, ok, err := Adjust(cfg, Container{Namespace: "gpu-operator"}) - require.NoError(t, err) - require.True(t, ok) - - for _, mount := range adjustment.Mounts { - require.NotContains(t, mount.Destination, "/sys/", - "no sysfs mount may be emitted when disabled") - } - // The overlay itself must still be injected, or the opt-out would silently - // disable the whole plugin. - require.Contains(t, adjustment.Mounts, Mount{ - Source: overlay, - Destination: cfg.ContainerOverlayPath, - Type: "bind", - Options: []string{"rbind", "ro", "nosuid", "nodev"}, - }) -} - // TestAdjustSkipsPCISysfsMountsWhenNotStaged is the fail-open case: the // tree is staged by the main DaemonSet and nothing orders this plugin after // it. A bind mount with a missing source fails container creation outright, From 8fa144be30146a6e5a6d44f1b216d8f196d64c77 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Tue, 18 Aug 2026 15:05:50 +0200 Subject: [PATCH 04/11] docs(nvml-mock): stop naming GFD_MACHINE_TYPE_FILE The mock renders the machine type where /sys/class/dmi/id points, so GFD resolves it through its own default path and the env var is not part of the contract. Describing it as the remedy for a DMI-less kernel invited exactly the configuration this design removed; the cases that read "unknown" are still spelled out, just without the recipe. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 5 ++-- .../nvml-mock/helm/nvml-mock/README.md | 24 +++++++------------ local/gpu-operator/gpu-operator.values.yaml | 11 ++++----- pkg/system/mockpcisysfs/render/render.go | 6 ++--- tests/e2e/go/assertions/pcisysfs.go | 7 +++--- 5 files changed, 22 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c2a44fa5..4892a15ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,9 +49,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 host's. The NVIDIA platform profiles set it; `l40s` and `t4` deliberately do not, since a commodity server's machine type is a property of the chassis, not the GPU. On a kernel that exposes no DMI at all (Docker Desktop's - linuxkit VM) `/sys/class/dmi` is absent and cannot be mounted into place, so - the label stays `unknown` unless GFD is pointed straight at the rendered file - with `GFD_MACHINE_TYPE_FILE`. (#673) + linuxkit VM) `/sys/class/dmi` is absent, there is no symlink to follow and it + cannot be mounted into place, so the label stays `unknown` there. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index 1e0da68b9..0c531cd17 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -758,27 +758,21 @@ The renderer writes it to subtree mounted above, which is where `/sys/class/dmi/id` points on a real node and the only place a container can be given it. -GPU Feature Discovery needs no configuration to find it: its default -machine-type file is `/sys/class/dmi/id/product_name`, and that symlink -resolves into the mounted subtree, so `nvidia.com/gpu.machine` reports the -profile's platform (`NVIDIA-GB200-NVL72`, `DGXA100`, …). +GPU Feature Discovery needs no configuration to find it: it reads the machine +type from `/sys/class/dmi/id/product_name`, and that symlink resolves into the +mounted subtree, so `nvidia.com/gpu.machine` reports the profile's platform +(`NVIDIA-GB200-NVL72`, `DGXA100`, …). Two cases read `unknown` instead: - The profile ships no `dmi:` block. `l40s` and `t4` deliberately do not, since a commodity server's machine type is a property of the chassis rather than the GPU, so nothing is rendered. -- The host kernel exposes no DMI at all, so `/sys/class/dmi` does not exist - and cannot be mounted into place either (a mountpoint cannot be created on a - read-only sysfs). Docker Desktop's linuxkit VM is the common case. Point GFD - straight at the rendered file there: - -```yaml -gfd: - env: - - name: GFD_MACHINE_TYPE_FILE - value: "/sys/devices/virtual/dmi/id/product_name" -``` +- The host kernel exposes no DMI at all, so `/sys/class/dmi` does not exist and + there is no symlink to follow. It cannot be mounted into place either, for + the same reason the PCI tree cannot be assembled per root complex: a + mountpoint cannot be created on a read-only sysfs. Docker Desktop's linuxkit + VM is the common case. ### Cross-node `ibping` diff --git a/local/gpu-operator/gpu-operator.values.yaml b/local/gpu-operator/gpu-operator.values.yaml index 21a846653..adc884829 100644 --- a/local/gpu-operator/gpu-operator.values.yaml +++ b/local/gpu-operator/gpu-operator.values.yaml @@ -59,12 +59,11 @@ devicePlugin: # GFD reads GPU attributes via mock NVML and labels the node. # -# nvidia.com/gpu.machine needs no configuration: GFD's default machine-type -# file is /sys/class/dmi/id/product_name, and /sys/class/dmi/id is a symlink -# into /sys/devices/virtual/dmi/id — inside the subtree the mock mounts. On a -# kernel that exposes no DMI at all (Docker Desktop's linuxkit VM) the -# directory is absent and the label reads "unknown"; point -# GFD_MACHINE_TYPE_FILE at /sys/devices/virtual/dmi/id/product_name there. +# nvidia.com/gpu.machine needs nothing here: GFD reads the machine type from +# /sys/class/dmi/id/product_name, a symlink into /sys/devices/virtual/dmi/id — +# inside the subtree the mock mounts. A kernel that exposes no DMI at all +# (Docker Desktop's linuxkit VM) has no such symlink, and the label reads +# "unknown" there. gfd: enabled: true env: diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 5bd4be9f6..42f56d662 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -120,9 +120,9 @@ func renderTopology(o Options) error { // both paths once the tree is bind-mounted over /sys/devices. const dmiIDDir = "sys/devices/virtual/dmi/id" -// renderDMI writes the node's mock machine type. GPU Feature Discovery -// reads this file (default --machine-type-file) to derive -// nvidia.com/gpu.machine; without it a mock node labels itself "unknown". +// renderDMI writes the node's mock machine type. GPU Feature Discovery reads +// it through /sys/class/dmi/id/product_name to derive nvidia.com/gpu.machine; +// without it a mock node labels itself "unknown". func renderDMI(root, productName string) error { return writeFile(root, filepath.Join(dmiIDDir, "product_name"), productName+"\n") } diff --git a/tests/e2e/go/assertions/pcisysfs.go b/tests/e2e/go/assertions/pcisysfs.go index 9a5325b8e..5211b4ad7 100644 --- a/tests/e2e/go/assertions/pcisysfs.go +++ b/tests/e2e/go/assertions/pcisysfs.go @@ -76,10 +76,9 @@ const KernelPCIDevicesDir = "/sys/bus/pci/devices" // bind-mounted into a container. const KernelDMIProductNameFile = "/sys/devices/virtual/dmi/id/product_name" -// DefaultMachineTypeFile is where GPU Feature Discovery reads the machine type -// unless GFD_MACHINE_TYPE_FILE says otherwise. On a kernel that exposes DMI it -// is a symlink into KernelDMIProductNameFile's directory, so GFD picks up the -// mock identity with no configuration. +// DefaultMachineTypeFile is where GPU Feature Discovery reads the machine type. +// On a kernel that exposes DMI it is a symlink into KernelDMIProductNameFile's +// directory, so GFD picks up the mock identity with no configuration. const DefaultMachineTypeFile = "/sys/class/dmi/id/product_name" // DMIExposedByKernel reports whether the container can reach DMI through the From 3b7d68389127aadc897a5d2ffd9181196e8894a4 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Tue, 18 Aug 2026 16:18:13 +0200 Subject: [PATCH 05/11] fix(nvml-mock): keep kind's DMI mount targets when shadowing /sys/devices Bind-mounting the rendered tree over /sys/devices also replaces virtual/dmi/id, the directory /sys/class/dmi/id resolves into. kind registers /kind/bin/mount-product-files.sh as a createContainer hook for every container on the node, and wherever the host kernel exposes DMI it bind-mounts the node's product_name and product_uuid there. mount(8) cannot create a target on a read-only sysfs, so every served pod died with mount: .../sys/class/dmi/id/product_uuid: mount point does not exist failing all e2e legs on Linux while passing on Docker Desktop, whose linuxkit VM exposes no DMI at all. The renderer now mirrors those attributes out of the kernel's DMI directory, so the targets survive the shadowing and the node keeps reporting its own identity. Mocking the machine type comes out of this change: kind fakes product_name to the literal "kind" for the node and, through the same hook, for every container, so no value the renderer writes reaches GFD's default machine-type file. Tracked in #681. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 21 ++--- cmd/render-pci-sysfs/main.go | 24 +++-- .../nvml-mock/helm/nvml-mock/README.md | 43 ++------- .../helm/nvml-mock/profiles/a100.yaml | 9 -- .../helm/nvml-mock/profiles/b200.yaml | 9 -- .../helm/nvml-mock/profiles/gb200.yaml | 9 -- .../helm/nvml-mock/profiles/gb300.yaml | 9 -- .../helm/nvml-mock/profiles/h100.yaml | 9 -- .../helm/nvml-mock/profiles/l40s.yaml | 11 --- .../nvml-mock/helm/nvml-mock/profiles/t4.yaml | 11 --- .../__snapshot__/configmap_test.yaml.snap | 67 -------------- .../__snapshot__/daemonset_test.yaml.snap | 6 +- deployments/nvml-mock/scripts/setup.sh | 17 +++- local/gpu-operator/gpu-operator.values.yaml | 6 -- pkg/nri/nvmlmock/adjust.go | 5 +- pkg/system/mockpcisysfs/config/types.go | 30 +----- pkg/system/mockpcisysfs/render/render.go | 74 ++++++++++----- pkg/system/mockpcisysfs/render/render_test.go | 92 +++++++++---------- tests/e2e/go/assertions/gfd_labels.go | 14 --- tests/e2e/go/assertions/gfd_labels_test.go | 19 ---- tests/e2e/go/assertions/pcisysfs.go | 42 +-------- tests/e2e/go/profile/gfd_test.go | 20 ---- tests/e2e/go/profile/profile.go | 29 ------ tests/e2e/go/scenario_gpu_operator_test.go | 23 +---- tests/e2e/gpu-operator-values.yaml | 5 - 25 files changed, 153 insertions(+), 451 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4892a15ff..25d86bf70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,19 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mountpoint the runtime cannot create on a read-only `/sys` — which hides the host's other device classes from served containers. Under NRI, which injects node-wide, keep workloads that need the host's real device tree in a namespace - listed in `nri.excludedNamespaces`. (#673) -- Profiles may declare the machine type a node of that platform reports, as a - `dmi:` block with `product_name`, which the renderer writes to - `sys/devices/virtual/dmi/id/product_name` in the mock overlay — the path - `/sys/class/dmi/id` points at, and the only one a container can be handed. - GPU Feature Discovery needs no configuration to pick it up, since that - symlink resolves into the mounted subtree: `nvidia.com/gpu.machine` reports - the profile's platform (`NVIDIA-GB200-NVL72`, `DGXA100`, …) instead of the - host's. The NVIDIA platform profiles set it; `l40s` and `t4` deliberately do - not, since a commodity server's machine type is a property of the chassis, - not the GPU. On a kernel that exposes no DMI at all (Docker Desktop's - linuxkit VM) `/sys/class/dmi` is absent, there is no symlink to follow and it - cannot be mounted into place, so the label stays `unknown` there. (#673) + listed in `nri.excludedNamespaces`. The rendered tree also mirrors the node's + DMI attributes (`product_name`, `product_uuid`) into + `sys/devices/virtual/dmi/id`, because shadowing `/sys/devices` shadows the + directory `/sys/class/dmi/id` resolves into: kind's `mount-product-files.sh` + createContainer hook bind-mounts the node's product files there for every + container, and `mount(8)` cannot create a target on a read-only sysfs. The + attributes are mirrored, not mocked: `nvidia.com/gpu.machine` still reports + what the node itself reports, tracked in #681. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/cmd/render-pci-sysfs/main.go b/cmd/render-pci-sysfs/main.go index 7ce9e5f8a..9600fa9ca 100644 --- a/cmd/render-pci-sysfs/main.go +++ b/cmd/render-pci-sysfs/main.go @@ -33,13 +33,21 @@ import ( "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) +// defaultDMISource is where the kernel exposes the node's SMBIOS identity. +// The rendered tree mirrors it because serving the tree means bind-mounting +// it over /sys/devices, which would otherwise hide the DMI directory that +// /sys/class/dmi/id resolves into. +const defaultDMISource = "/sys/class/dmi/id" + //nolint:cyclop // existing complexity; refactor deferred func main() { var ( - cfgPath = flag.String("config", "", "path to mock-nvml profile YAML") - outDir = flag.String("output", "", "fake-root directory; tree is written under /sys/...") - strict = flag.Bool("strict", false, "fail if the profile does not declare `pcie_topology:`") - dryRun = flag.Bool("dry-run", false, "validate the config and exit without writing files") + cfgPath = flag.String("config", "", "path to mock-nvml profile YAML") + outDir = flag.String("output", "", "fake-root directory; tree is written under /sys/...") + dmiSource = flag.String("dmi-source", defaultDMISource, + "kernel DMI directory to mirror into the tree; empty mirrors nothing") + strict = flag.Bool("strict", false, "fail if the profile does not declare `pcie_topology:`") + dryRun = flag.Bool("dry-run", false, "validate the config and exit without writing files") ) flag.Parse() @@ -76,10 +84,10 @@ func main() { } if err := render.Render(render.Options{ - Topology: topo, - Identities: prof.DeviceIdentities(), - Output: *outDir, - DMIProductName: prof.DMIProductName(), + Topology: topo, + Identities: prof.DeviceIdentities(), + Output: *outDir, + DMISource: *dmiSource, }); err != nil { fatalf("render: %v", err) } diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index 0c531cd17..755641dc7 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -26,9 +26,6 @@ Deploys a DaemonSet that creates on every node: directories are additionally bind-mounted onto `/sys/bus/pci/devices` and `/sys/devices` in served containers (see [PCI sysfs in containers](#pci-sysfs-in-containers)) -- A fake DMI identity at `/var/lib/nvml-mock/sys/devices/virtual/dmi/id/product_name` - when the profile declares a `dmi:` block, which GPU Feature Discovery turns - into `nvidia.com/gpu.machine` (see [PCI sysfs in containers](#pci-sysfs-in-containers)) Consumers (DRA driver, device plugin) point at `/var/lib/nvml-mock/driver` as the NVIDIA driver root and discover GPUs through standard NVML APIs. @@ -743,36 +740,16 @@ read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so this is not configurable; keep workloads that need the host's real device tree off it, or in a namespace listed in `nri.excludedNamespaces`. -### Machine type (`dmi:`) - -A profile may declare the SMBIOS product name a node of that platform -reports: - -```yaml -dmi: - product_name: "NVIDIA GB200 NVL72" -``` - -The renderer writes it to -`/var/lib/nvml-mock/sys/devices/virtual/dmi/id/product_name` — inside the -subtree mounted above, which is where `/sys/class/dmi/id` points on a real -node and the only place a container can be given it. - -GPU Feature Discovery needs no configuration to find it: it reads the machine -type from `/sys/class/dmi/id/product_name`, and that symlink resolves into the -mounted subtree, so `nvidia.com/gpu.machine` reports the profile's platform -(`NVIDIA-GB200-NVL72`, `DGXA100`, …). - -Two cases read `unknown` instead: - -- The profile ships no `dmi:` block. `l40s` and `t4` deliberately do not, - since a commodity server's machine type is a property of the chassis rather - than the GPU, so nothing is rendered. -- The host kernel exposes no DMI at all, so `/sys/class/dmi` does not exist and - there is no symlink to follow. It cannot be mounted into place either, for - the same reason the PCI tree cannot be assembled per root complex: a - mountpoint cannot be created on a read-only sysfs. Docker Desktop's linuxkit - VM is the common case. +`/sys/devices/virtual/dmi/id` — the directory `/sys/class/dmi/id` resolves +into — is shadowed along with the rest, so the renderer mirrors the node's +`product_name` and `product_uuid` into the tree. This is not cosmetic: kind's +`mount-product-files.sh` createContainer hook bind-mounts the node's copies of +both onto every container it starts, and `mount(8)` cannot create a target on a +read-only sysfs, so a missing attribute fails container creation for every pod +the mock serves. The values are mirrored, not mocked — a node keeps reporting +its own machine type, which under kind is the literal `kind`, so +`nvidia.com/gpu.machine` does not follow the profile. Tracked in +[#681](https://github.com/NVIDIA/k8s-test-infra/issues/681). ### Cross-node `ibping` diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml index 97388cfaa..99ef06eac 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml @@ -455,12 +455,3 @@ pcie_topology: - "0000:90:00.0" - "0000:B7:00.0" - "0000:BD:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name -# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery -# derives nvidia.com/gpu.machine instead of labelling the node "unknown". -# The string is representative of the platform, not read off real hardware. -# ============================================================================= -dmi: - product_name: "DGXA100" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml index b3278ff62..ccf6756b6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml @@ -452,12 +452,3 @@ pcie_topology: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name -# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery -# derives nvidia.com/gpu.machine instead of labelling the node "unknown". -# The string is representative of the platform, not read off real hardware. -# ============================================================================= -dmi: - product_name: "NVIDIA DGX B200" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml index 58a608599..76be781f4 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml @@ -540,12 +540,3 @@ pcie_topology: devices: - "0000:CA:00.0" - "0000:CB:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name -# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery -# derives nvidia.com/gpu.machine instead of labelling the node "unknown". -# The string is representative of the platform, not read off real hardware. -# ============================================================================= -dmi: - product_name: "NVIDIA GB200 NVL72" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml index 90a831919..4167388af 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml @@ -540,12 +540,3 @@ pcie_topology: devices: - "0000:CA:00.0" - "0000:CB:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name -# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery -# derives nvidia.com/gpu.machine instead of labelling the node "unknown". -# The string is representative of the platform, not read off real hardware. -# ============================================================================= -dmi: - product_name: "NVIDIA GB300 NVL72" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml index 3d16f6e6a..6a8db6067 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml @@ -465,12 +465,3 @@ pcie_topology: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name -# (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery -# derives nvidia.com/gpu.machine instead of labelling the node "unknown". -# The string is representative of the platform, not read off real hardware. -# ============================================================================= -dmi: - product_name: "DGXH100" diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml index b75e2437d..d9e79619b 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml @@ -412,14 +412,3 @@ pcie_topology: - "0000:B2:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity, deliberately unset: an L40S ships in commodity OEM -# servers whose product_name is vendor-specific, so any value here would be -# invented. GPU Feature Discovery therefore labels nvidia.com/gpu.machine -# "unknown" — the same answer a real node gives when DMI is unreadable. Set -# one to pin a machine type: -# -# dmi: -# product_name: "PowerEdge R760xa" -# ============================================================================= diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml index e76aa66b2..aa65b16e6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml @@ -383,14 +383,3 @@ pcie_topology: - "0000:86:00.0" - "0000:AF:00.0" - "0000:D8:00.0" - -# ============================================================================= -# DMI (SMBIOS) identity, deliberately unset: T4s ship in commodity OEM -# servers and cloud VMs whose product_name is vendor-specific, so any value -# here would be invented. GPU Feature Discovery therefore labels -# nvidia.com/gpu.machine "unknown" — the same answer a real node gives when -# DMI is unreadable. Set one to pin a machine type: -# -# dmi: -# product_name: "PowerEdge R740" -# ============================================================================= diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap index 0e412040e..35aaab9fc 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap @@ -457,15 +457,6 @@ should match snapshot with b200 profile: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name - # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery - # derives nvidia.com/gpu.machine instead of labelling the node "unknown". - # The string is representative of the platform, not read off real hardware. - # ============================================================================= - dmi: - product_name: "NVIDIA DGX B200" kind: ConfigMap metadata: labels: @@ -937,15 +928,6 @@ should match snapshot with default a100 profile: - "0000:90:00.0" - "0000:B7:00.0" - "0000:BD:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name - # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery - # derives nvidia.com/gpu.machine instead of labelling the node "unknown". - # The string is representative of the platform, not read off real hardware. - # ============================================================================= - dmi: - product_name: "DGXA100" kind: ConfigMap metadata: labels: @@ -1502,15 +1484,6 @@ should match snapshot with gb200 profile: devices: - "0000:CA:00.0" - "0000:CB:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name - # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery - # derives nvidia.com/gpu.machine instead of labelling the node "unknown". - # The string is representative of the platform, not read off real hardware. - # ============================================================================= - dmi: - product_name: "NVIDIA GB200 NVL72" kind: ConfigMap metadata: labels: @@ -2067,15 +2040,6 @@ should match snapshot with gb300 profile: devices: - "0000:CA:00.0" - "0000:CB:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name - # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery - # derives nvidia.com/gpu.machine instead of labelling the node "unknown". - # The string is representative of the platform, not read off real hardware. - # ============================================================================= - dmi: - product_name: "NVIDIA GB300 NVL72" kind: ConfigMap metadata: labels: @@ -2557,15 +2521,6 @@ should match snapshot with h100 profile: - "0000:8B:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity. Rendered to /sys/devices/virtual/dmi/id/product_name - # (what /sys/class/dmi/id/product_name points at) so GPU Feature Discovery - # derives nvidia.com/gpu.machine instead of labelling the node "unknown". - # The string is representative of the platform, not read off real hardware. - # ============================================================================= - dmi: - product_name: "DGXH100" kind: ConfigMap metadata: labels: @@ -2994,17 +2949,6 @@ should match snapshot with l40s profile: - "0000:B2:00.0" - "0000:CA:00.0" - "0000:CB:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity, deliberately unset: an L40S ships in commodity OEM - # servers whose product_name is vendor-specific, so any value here would be - # invented. GPU Feature Discovery therefore labels nvidia.com/gpu.machine - # "unknown" — the same answer a real node gives when DMI is unreadable. Set - # one to pin a machine type: - # - # dmi: - # product_name: "PowerEdge R760xa" - # ============================================================================= kind: ConfigMap metadata: labels: @@ -3404,17 +3348,6 @@ should match snapshot with t4 profile: - "0000:86:00.0" - "0000:AF:00.0" - "0000:D8:00.0" - - # ============================================================================= - # DMI (SMBIOS) identity, deliberately unset: T4s ship in commodity OEM - # servers and cloud VMs whose product_name is vendor-specific, so any value - # here would be invented. GPU Feature Discovery therefore labels - # nvidia.com/gpu.machine "unknown" — the same answer a real node gives when - # DMI is unreadable. Set one to pin a machine type: - # - # dmi: - # product_name: "PowerEdge R740" - # ============================================================================= kind: ConfigMap metadata: labels: diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap index 52c85b29a..22d37bc09 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap @@ -19,7 +19,7 @@ should match snapshot with all overrides: template: metadata: annotations: - checksum/config: 13fe7d8201b73a6ab5a79ba31e4808f7061fd6c0e394cdcb6483c98c14413fa9 + checksum/config: 0a18dc0ae54a3a137218785bddf06ebc287c39fb6547e51ea7a2b650e956511b labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: custom @@ -166,7 +166,7 @@ should match snapshot with b200 profile: template: metadata: annotations: - checksum/config: 76e0a95c4f16f54fd41bcd170108cb3f5793fd9e48247f17899663db076dd946 + checksum/config: c9174f0176a7b6f1deeb10de59ce2f8811e9d25a391a628f247102ee978832f6 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME @@ -292,7 +292,7 @@ should match snapshot with default values: template: metadata: annotations: - checksum/config: 2e3e92a6183a58b6d99935e91924bf340cdbf32b5118700cbbf74a305bd50e92 + checksum/config: 1bdbe9385e4c347ca3f4d2678fbbfc885d7539e4340a6a8c2b93fe78c1ed44c5 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME diff --git a/deployments/nvml-mock/scripts/setup.sh b/deployments/nvml-mock/scripts/setup.sh index f7d70bae4..dbdb842a8 100644 --- a/deployments/nvml-mock/scripts/setup.sh +++ b/deployments/nvml-mock/scripts/setup.sh @@ -120,11 +120,11 @@ fi # each device's `vendor` / `class` to derive nvidia.com/gpu.mode). The # renderer parses the profile's `pcie_topology:` block; profiles without # one get a flat default covering every device under a single root -# complex (`pci0000:00`, NUMA 0). It also writes the profile's `dmi:` -# product name, which GFD turns into nvidia.com/gpu.machine. Failures -# are fatal under `set -e` for the same reason as the IB render below — -# a topology typo otherwise yields silently malformed sysfs that -# downstream `dra.k8s.io/pcieRoot` attributes would inherit. +# complex (`pci0000:00`, NUMA 0). It also mirrors the node's DMI identity +# into the tree, which the mount below depends on. Failures are fatal +# under `set -e` for the same reason as the IB render below — a topology +# typo otherwise yields silently malformed sysfs that downstream +# `dra.k8s.io/pcieRoot` attributes would inherit. # # This runs before the CDI spec below because that spec bind-mounts the # rendered directories into consumers: a bind mount whose source is @@ -199,6 +199,7 @@ CDI_HEADER # /sys/bus/pci/devices//vendor, and labels the node # nvidia.com/gpu.mode=unknown. # +# # Both mounts are needed: the entries under sys/bus/pci/devices are # relative symlinks into ../../../devices/pciDDDD:BB, which only resolve # when the rendered sys/devices is mounted too. That second mount hides the @@ -207,6 +208,12 @@ CDI_HEADER # is not available: sysfs is read-only inside the container, so the runtime # cannot create a mountpoint like /sys/devices/pci0000:80 that the host # does not already have, and container creation fails outright. +# +# Shadowing sys/devices also replaces virtual/dmi/id, which +# /sys/class/dmi/id resolves into. kind's mount-product-files.sh +# createContainer hook bind-mounts the node's product_name / product_uuid +# there for every container, and mount(8) cannot create a target on a +# read-only sysfs — hence the renderer mirroring those attributes above. if [ "$PCI_SYSFS_RENDERED" = "on" ]; then cat >> "$CDI_DIR/nvidia.yaml" << PCI_SYSFS_MOUNT_EOF - hostPath: /var/lib/nvml-mock/sys/devices diff --git a/local/gpu-operator/gpu-operator.values.yaml b/local/gpu-operator/gpu-operator.values.yaml index adc884829..cbe340c72 100644 --- a/local/gpu-operator/gpu-operator.values.yaml +++ b/local/gpu-operator/gpu-operator.values.yaml @@ -58,12 +58,6 @@ devicePlugin: value: "/var/lib/nvml-mock/driver" # GFD reads GPU attributes via mock NVML and labels the node. -# -# nvidia.com/gpu.machine needs nothing here: GFD reads the machine type from -# /sys/class/dmi/id/product_name, a symlink into /sys/devices/virtual/dmi/id — -# inside the subtree the mock mounts. A kernel that exposes no DMI at all -# (Docker Desktop's linuxkit VM) has no such symlink, and the label reads -# "unknown" there. gfd: enabled: true env: diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index a90451073..68964781b 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -364,7 +364,10 @@ func topologyInjectable(cfg Config) bool { // consumers that resolve GPUs through sysfs: the tree cannot be assembled // per root complex instead, because a bind mount at a path sysfs does not // already have (say /sys/devices/pci0000:80) needs a mountpoint the runtime -// cannot create on a read-only sysfs. +// cannot create on a read-only sysfs. It also shadows virtual/dmi/id, which +// is why the renderer mirrors the node's DMI attributes into the tree: kind's +// createContainer hook bind-mounts the node's product files there, and a +// missing target fails container creation. // // Missing sources are skipped rather than reported: the tree is staged by // the main nvml-mock DaemonSet and nothing orders this plugin after it, and diff --git a/pkg/system/mockpcisysfs/config/types.go b/pkg/system/mockpcisysfs/config/types.go index f7b5ec11a..d16a78d13 100644 --- a/pkg/system/mockpcisysfs/config/types.go +++ b/pkg/system/mockpcisysfs/config/types.go @@ -1,11 +1,10 @@ // Copyright 2026 NVIDIA CORPORATION // SPDX-License-Identifier: Apache-2.0 -// Package config defines the YAML schema for the `pcie_topology:` and -// `dmi:` blocks embedded in mock-nvml profile configs. The renderer -// consumes these to populate a fake `/sys/bus/pci/devices` + -// `/sys/devices/pciDDDD:BB` tree and a fake DMI identity under -// MOCK_PCI_ROOT. +// Package config defines the YAML schema for the `pcie_topology:` block +// embedded in mock-nvml profile configs. The renderer consumes it to +// populate a fake `/sys/bus/pci/devices` + `/sys/devices/pciDDDD:BB` tree +// under MOCK_PCI_ROOT. // // Keeping the schema in a standalone package (no cgo dependency on the // mocknvml engine) mirrors `pkg/network/mockibsysfs/config` and lets the @@ -26,27 +25,6 @@ type Profile struct { DeviceDefaults *DeviceDefaults `json:"device_defaults,omitempty" yaml:"device_defaults,omitempty"` Devices []Device `json:"devices" yaml:"devices"` PCIeTopology *PCIeTopology `json:"pcie_topology,omitempty" yaml:"pcie_topology,omitempty"` - DMI *DMI `json:"dmi,omitempty" yaml:"dmi,omitempty"` -} - -// DMI mirrors the profile's optional `dmi:` block, the SMBIOS identity a -// real node exposes through /sys/class/dmi/id. Only product_name is -// modelled: it is what GPU Feature Discovery reads to derive the -// nvidia.com/gpu.machine label, and it is the only DMI field any mock -// consumer has needed so far. -type DMI struct { - ProductName string `json:"product_name,omitempty" yaml:"product_name,omitempty"` -} - -// DMIProductName returns the profile's SMBIOS product name, or "" when the -// profile declares no `dmi:` block. Callers treat "" as "render no DMI -// identity" rather than substituting a placeholder, so a node keeps -// reporting an honest "unknown" machine type until a profile opts in. -func (p *Profile) DMIProductName() string { - if p.DMI == nil { - return "" - } - return strings.TrimSpace(p.DMI.ProductName) } // DeviceDefaults mirrors the profile's `device_defaults:` block. Only the diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 42f56d662..17813fd76 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -58,11 +58,12 @@ type Options struct { // required; otherwise Render returns an error. Output string - // DMIProductName is the SMBIOS product name to expose as the node's - // machine type, from the profile's `dmi:` block. Empty renders no DMI - // identity at all, leaving consumers to report their own "unknown" - // rather than a placeholder machine. - DMIProductName string + // DMISource is the directory holding the node's kernel DMI identity, + // normally /sys/class/dmi/id. The attributes found there are mirrored + // into the tree so that bind-mounting sys/devices over the kernel's + // does not take the DMI directory with it — see renderDMI. Empty + // mirrors nothing. + DMISource string } // Render writes the entire tree. It is idempotent: existing directories @@ -70,7 +71,7 @@ type Options struct { // symlinks are removed and recreated so a stale relative target does not // linger across re-renders. func Render(o Options) error { - if !o.hasTopology() && o.DMIProductName == "" { + if !o.hasTopology() { // Nothing to do — caller decided to render a profile with no // declared topology and no devices. Treat as a no-op so the // renderer can be invoked unconditionally from setup.sh. @@ -80,17 +81,10 @@ func Render(o Options) error { return errors.New("pcisysfs render: Output is required") } - // The DMI identity is independent of the PCI tree: a profile may name its - // machine type without declaring a topology. - if o.DMIProductName != "" { - if err := renderDMI(o.Output, o.DMIProductName); err != nil { - return err - } - } - if !o.hasTopology() { - return nil + if err := renderTopology(o); err != nil { + return err } - return renderTopology(o) + return renderDMI(o.Output, o.DMISource) } func (o Options) hasTopology() bool { @@ -114,17 +108,47 @@ func renderTopology(o Options) error { return nil } -// dmiIDDir is where the kernel materializes the SMBIOS identity. The -// familiar /sys/class/dmi/id path is only a symlink into this directory, -// so rendering here is what makes the mock machine type resolvable through -// both paths once the tree is bind-mounted over /sys/devices. +// dmiIDDir is where the kernel materializes the SMBIOS identity; the +// familiar /sys/class/dmi/id path is only a symlink into it. const dmiIDDir = "sys/devices/virtual/dmi/id" -// renderDMI writes the node's mock machine type. GPU Feature Discovery reads -// it through /sys/class/dmi/id/product_name to derive nvidia.com/gpu.machine; -// without it a mock node labels itself "unknown". -func renderDMI(root, productName string) error { - return writeFile(root, filepath.Join(dmiIDDir, "product_name"), productName+"\n") +// dmiMirroredAttrs are the DMI attributes kind's mount-product-files.sh +// createContainer hook bind-mounts the node's copies onto, for every +// container on the node. +var dmiMirroredAttrs = []string{"product_name", "product_uuid"} + +// renderDMI mirrors the node's DMI identity into the tree. Serving the tree +// means bind-mounting it over /sys/devices, which also replaces +// virtual/dmi/id — the directory /sys/class/dmi/id resolves into. Any +// attribute missing from the replacement is a bind-mount target that no +// longer exists, and mount(8) cannot create one on a read-only sysfs, so +// kind's hook fails and every injected container fails to start. +// +// Mirroring rather than mocking keeps the node's identity intact: kind +// already reports its own ("kind" as the product name, a random UUID), and +// overriding that is a separate concern with its own consumers. +func renderDMI(root, source string) error { + if source == "" { + return nil + } + for _, attr := range dmiMirroredAttrs { + src := filepath.Join(source, attr) + if _, err := os.Stat(src); err != nil { + // The kernel exposes no such attribute, so nothing bind-mounts + // it either and a stand-in would only invent an identity. + continue + } + // product_uuid is mode 0400 on most kernels: unreadable to a + // non-root renderer, yet still needed as a mount target. + contents, err := os.ReadFile(src) + if err != nil { + contents = nil + } + if err := writeFile(root, filepath.Join(dmiIDDir, attr), string(contents)); err != nil { + return err + } + } + return nil } func renderRootComplex(root string, rc config.RootComplex, ids map[string]config.PCI) error { diff --git a/pkg/system/mockpcisysfs/render/render_test.go b/pkg/system/mockpcisysfs/render/render_test.go index c576f075b..47bcd5032 100644 --- a/pkg/system/mockpcisysfs/render/render_test.go +++ b/pkg/system/mockpcisysfs/render/render_test.go @@ -216,13 +216,16 @@ func TestRender_NormalizesUppercaseBDF(t *testing.T) { require.NoError(t, err, "expected lowercase symlink") } -// TestRender_DMIProductName pins the DMI identity at the path the kernel -// exposes it, /sys/devices/virtual/dmi/id/product_name. It lives under -// sys/devices (not sys/class) on purpose: /sys/class/dmi/id is itself a -// symlink into that directory, and sys/devices is the subtree consumers -// can be handed as a bind mount, so a container that sees the mock tree -// resolves the mock product name through either path. -func TestRender_DMIProductName(t *testing.T) { +// TestRender_MirrorsKernelDMI covers the reason the tree carries a DMI +// directory at all: bind-mounting it over /sys/devices hides the real +// virtual/dmi/id, which kind's mount-product-files.sh hook bind-mounts the +// node's product files onto for every container. Mirroring the attributes +// keeps those mount targets in place and leaves the node's identity intact. +func TestRender_MirrorsKernelDMI(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("kind\n"), 0o644), "stage product_name") + require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("dead-beef\n"), 0o644), "stage product_uuid") + dir := t.TempDir() topo := &config.PCIeTopology{ RootComplexes: []config.RootComplex{{ @@ -230,19 +233,25 @@ func TestRender_DMIProductName(t *testing.T) { Devices: []string{"0000:07:00.0"}, }}, } - require.NoError(t, Render(Options{ - Topology: topo, - Output: dir, - DMIProductName: "NVIDIA GB200 NVL72", - }), "Render") - - got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) - require.NoError(t, err, "read product_name") - // Trailing newline mirrors the kernel; GFD trims it before labelling. - require.Equal(t, "NVIDIA GB200 NVL72\n", string(got), "product_name") + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") + + for name, want := range map[string]string{"product_name": "kind\n", "product_uuid": "dead-beef\n"} { + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id", name)) + require.NoError(t, err, "read %s", name) + require.Equal(t, want, string(got), name) + } } -func TestRender_NoDMIWhenProductNameEmpty(t *testing.T) { +// TestRender_StandsInForUnreadableDMI covers product_uuid, which the kernel +// exposes mode 0400: the value cannot be mirrored, but the file must still +// exist, because mount(8) cannot create a target on a read-only sysfs. +func TestRender_StandsInForUnreadableDMI(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root reads any mode; the permission branch is unreachable") + } + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("secret\n"), 0o000), "stage product_uuid") + dir := t.TempDir() topo := &config.PCIeTopology{ RootComplexes: []config.RootComplex{{ @@ -250,23 +259,28 @@ func TestRender_NoDMIWhenProductNameEmpty(t *testing.T) { Devices: []string{"0000:07:00.0"}, }}, } - require.NoError(t, Render(Options{Topology: topo, Output: dir}), "Render") + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") - _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual")) - require.True(t, os.IsNotExist(err), - "profiles without a dmi: block must render no DMI identity, got err=%v", err) + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_uuid")) + require.NoError(t, err, "read product_uuid") + require.Empty(t, got, "an unreadable attribute renders as an empty stand-in") } -// TestRender_DMIWithoutTopology covers a profile that declares a machine -// type but no devices: the DMI identity is independent of the PCI tree, so -// it must still land. -func TestRender_DMIWithoutTopology(t *testing.T) { +// TestRender_NoDMIWithoutKernelDMI pins the behavior on kernels that expose +// no DMI at all (Docker Desktop's linuxkit VM, for one): nothing bind-mounts +// product files there, so inventing them would only mislead consumers. +func TestRender_NoDMIWithoutKernelDMI(t *testing.T) { dir := t.TempDir() - require.NoError(t, Render(Options{Output: dir, DMIProductName: "DGXA100"}), "Render") + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: filepath.Join(t.TempDir(), "absent")}), "Render") - got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) - require.NoError(t, err, "read product_name") - require.Equal(t, "DGXA100\n", string(got), "product_name") + _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual")) + require.True(t, os.IsNotExist(err), "expected no DMI directory, got err=%v", err) } // --- Config / Validate tests -------------------------------------------------- @@ -415,24 +429,6 @@ func TestDeviceIdentities_NoDefaults(t *testing.T) { require.Equal(t, uint32(0), ids["0000:07:00.0"].DeviceID) } -func TestDMIProductName_ParsesProfileBlock(t *testing.T) { - var p config.Profile - require.NoError(t, yaml.Unmarshal([]byte(` -devices: - - index: 0 - pci: - bus_id: "0000:07:00.0" -dmi: - product_name: " DGXA100 " -`), &p), "unmarshal") - require.Equal(t, "DGXA100", p.DMIProductName(), "product_name should be trimmed") -} - -func TestDMIProductName_EmptyWithoutBlock(t *testing.T) { - p := config.Profile{} - require.Empty(t, p.DMIProductName(), "profiles without dmi: declare no machine type") -} - func TestEffectiveTopology_PrefersExplicit(t *testing.T) { p := config.Profile{ Devices: []config.Device{{Index: 0, PCI: config.PCI{BusID: "0000:07:00.0"}}}, diff --git a/tests/e2e/go/assertions/gfd_labels.go b/tests/e2e/go/assertions/gfd_labels.go index cf5dae4c9..b36a3f245 100644 --- a/tests/e2e/go/assertions/gfd_labels.go +++ b/tests/e2e/go/assertions/gfd_labels.go @@ -22,22 +22,8 @@ const ( GFDLabelProduct = "nvidia.com/gpu.product" GFDLabelMemory = "nvidia.com/gpu.memory" GFDLabelCount = "nvidia.com/gpu.count" - GFDLabelMachine = "nvidia.com/gpu.machine" ) -// ExpectedMachineTypeLabel is the gpu.machine label GFD publishes from the -// mock DMI identity. It is separate from ExpectedGFDLabels because those come -// from NVML and were already correct while this one read "unknown": it -// requires the rendered sysfs tree to reach the GFD container (#673). -// -// gpu.mode is deliberately NOT asserted here. It also needs the tree, but GFD -// derives the BDF it looks up from NVML's busId, which the mock reports in the -// wrong domain width (#671) — so mode stays "unknown" until that lands, and -// asserting it would couple this expectation to an unrelated fix. -func ExpectedMachineTypeLabel(machineType string) map[string]string { - return map[string]string{GFDLabelMachine: machineType} -} - // ExpectedGFDLabels derives the GFD labels a node must carry from the profile, // rather than from the node itself. Deriving them independently is what makes // the assertion discriminating: if GFD stopped reading the mock NVML, the count diff --git a/tests/e2e/go/assertions/gfd_labels_test.go b/tests/e2e/go/assertions/gfd_labels_test.go index 00fa655a4..46aee6d21 100644 --- a/tests/e2e/go/assertions/gfd_labels_test.go +++ b/tests/e2e/go/assertions/gfd_labels_test.go @@ -40,25 +40,6 @@ func TestExpectedGFDLabelsCountTracksProfileNotAConstant(t *testing.T) { assert.NotEqual(t, four[GFDLabelProduct], eight[GFDLabelProduct]) } -func TestExpectedMachineTypeLabelCarriesTheProfileMachine(t *testing.T) { - t.Parallel() - - want := ExpectedMachineTypeLabel("NVIDIA-GB200-NVL72") - - assert.Equal(t, "NVIDIA-GB200-NVL72", want[GFDLabelMachine]) -} - -// Profiles without a dmi: block expect the literal "unknown" rather than no -// assertion at all, so the spec still fails if gpu.machine goes missing. -func TestExpectedMachineTypeLabelKeepsUnknownAsAnExpectation(t *testing.T) { - t.Parallel() - - want := ExpectedMachineTypeLabel("unknown") - - problems := DiffGFDLabels(want, map[string]string{}) - assert.Len(t, problems, 1, "a missing gpu.machine must still be reported") -} - func TestDiffGFDLabelsReportsMissingLabel(t *testing.T) { t.Parallel() diff --git a/tests/e2e/go/assertions/pcisysfs.go b/tests/e2e/go/assertions/pcisysfs.go index 5211b4ad7..08d720f51 100644 --- a/tests/e2e/go/assertions/pcisysfs.go +++ b/tests/e2e/go/assertions/pcisysfs.go @@ -71,43 +71,14 @@ func PCISysfs(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount, ex // their opens and MOCK_PCI_ROOT has no effect on them. const KernelPCIDevicesDir = "/sys/bus/pci/devices" -// KernelDMIProductNameFile is where the mock renders the machine type: the -// location /sys/class/dmi/id points to, and the only part of sysfs that can be -// bind-mounted into a container. -const KernelDMIProductNameFile = "/sys/devices/virtual/dmi/id/product_name" - -// DefaultMachineTypeFile is where GPU Feature Discovery reads the machine type. -// On a kernel that exposes DMI it is a symlink into KernelDMIProductNameFile's -// directory, so GFD picks up the mock identity with no configuration. -const DefaultMachineTypeFile = "/sys/class/dmi/id/product_name" - -// DMIExposedByKernel reports whether the container can reach DMI through the -// path GFD reads by default. It is false on hosts whose kernel exposes no DMI -// at all — Docker Desktop's linuxkit VM, for one — where /sys/class/dmi does -// not exist and no mount can create it, because a mountpoint cannot be made on -// a read-only sysfs. The gpu.machine label is then "unknown" for reasons that -// have nothing to do with the mock, so callers detect this instead of -// asserting. -func DMIExposedByKernel(ctx context.Context, k *kube.Client, pod kube.PodRef) bool { - ginkgo.GinkgoHelper() - - res, err := k.ExecSh(ctx, pod, "test -e "+DefaultMachineTypeFile+" && echo yes || echo no") - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "probing %s: %s", DefaultMachineTypeFile, res.Combined()) - return strings.TrimSpace(res.Stdout) == "yes" -} - // PCISysfsAtKernelPath asserts, from inside a container the mock serves (a GPU // Operator operand, for instance), that the rendered tree arrived at the real // kernel paths rather than only in the overlay: // - /sys/bus/pci/devices holds exactly the mock GPUs, so the host's own PCI // devices are masked and consumers enumerate the profile, // - reading a device's `vendor` yields NVIDIA, which only works when -// /sys/devices is mounted too (the entries are relative symlinks into it), -// - the DMI product name is the profile's, not the host's. -// -// machineType is the raw `dmi.product_name` from the profile; pass "" for a -// profile that declares none, which skips the DMI check. -func PCISysfsAtKernelPath(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount int, machineType string) { +// /sys/devices is mounted too (the entries are relative symlinks into it). +func PCISysfsAtKernelPath(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount int) { ginkgo.GinkgoHelper() ginkgo.By(fmt.Sprintf("%d mock PCI devices visible at %s", gpuCount, KernelPCIDevicesDir)) @@ -127,15 +98,6 @@ func PCISysfsAtKernelPath(ctx context.Context, k *kube.Client, pod kube.PodRef, "reading vendor for %s — a dangling symlink means /sys/devices is missing", dev) gomega.Expect(strings.TrimSpace(vendor.Stdout)).To(gomega.Equal("0x10de"), "vendor for %s\n%s", dev, vendor.Combined()) - - if machineType == "" { - return - } - ginkgo.By("DMI product name is the profile's machine type") - product, err := k.ExecSh(ctx, pod, "cat "+KernelDMIProductNameFile) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "reading %s", KernelDMIProductNameFile) - gomega.Expect(strings.TrimSpace(product.Stdout)).To(gomega.Equal(machineType), - "mock DMI product name\n%s", product.Combined()) } func atoiTrim(s string) int { diff --git a/tests/e2e/go/profile/gfd_test.go b/tests/e2e/go/profile/gfd_test.go index cc74b8bef..2a4eb1c3a 100644 --- a/tests/e2e/go/profile/gfd_test.go +++ b/tests/e2e/go/profile/gfd_test.go @@ -30,26 +30,6 @@ func TestGB200ProfileDerivesObservedGFDLabelValues(t *testing.T) { assert.Equal(t, 8, p.ExpectedGPUs()) } -// gpu.machine comes from the profile's dmi: block, dashed the same way as -// gpu.product. Profiles that ship no block (commodity-server platforms) must -// resolve to the literal "unknown" GFD publishes when it cannot read DMI, so -// the e2e expectation stays an assertion rather than a skip. -func TestGFDMachineTypeComesFromTheDMIBlock(t *testing.T) { - t.Parallel() - - gb200, err := Load(profilesDir, "gb200") - require.NoError(t, err) - a100, err := Load(profilesDir, "a100") - require.NoError(t, err) - t4, err := Load(profilesDir, "t4") - require.NoError(t, err) - - assert.Equal(t, "NVIDIA-GB200-NVL72", gb200.GFDMachineType()) - assert.Equal(t, "DGXA100", a100.GFDMachineType()) - assert.Equal(t, GFDMachineTypeUnknown, t4.GFDMachineType()) - assert.NotContains(t, gb200.GFDMachineType(), " ") -} - func TestGFDProductNameReplacesSpacesWithDashes(t *testing.T) { t.Parallel() diff --git a/tests/e2e/go/profile/profile.go b/tests/e2e/go/profile/profile.go index d722ccec3..9d6a6bafd 100644 --- a/tests/e2e/go/profile/profile.go +++ b/tests/e2e/go/profile/profile.go @@ -85,9 +85,6 @@ type rawProfile struct { ID string `json:"id"` } `json:"root_complexes"` } `json:"pcie_topology"` - DMI *struct { - ProductName string `json:"product_name"` - } `json:"dmi"` } // rawPlatform decodes a platform block, which appears both under @@ -132,7 +129,6 @@ type Profile struct { pciRoots int memoryBytes int64 - dmiProductName string architecture string shutdownThresholdC int slowdownThresholdC int @@ -160,28 +156,6 @@ func (p Profile) GFDProductName() string { // nvidia.com/gpu.memory. func (p Profile) MemoryMiB() int { return int(p.memoryBytes / bytesPerMiB) } -// GFDMachineTypeUnknown is what GPU Feature Discovery publishes when it cannot -// read a machine type. Profiles that declare no `dmi:` block (l40s, t4) expect -// exactly this, so the assertion stays meaningful for them rather than being -// skipped. -const GFDMachineTypeUnknown = "unknown" - -// DMIProductName is the profile's `dmi.product_name` verbatim — the string the -// renderer writes into the mock sysfs tree — or "" when the profile declares no -// `dmi:` block. Use GFDMachineType for the label form. -func (p Profile) DMIProductName() string { return p.dmiProductName } - -// GFDMachineType is the profile's `dmi.product_name` in the form GFD publishes -// as nvidia.com/gpu.machine: spaces become dashes, as for gpu.product. The -// renderer writes the same string into the mock sysfs tree, so this is what -// GFD reads back through its machine-type file. -func (p Profile) GFDMachineType() string { - if p.dmiProductName == "" { - return GFDMachineTypeUnknown - } - return strings.ReplaceAll(p.dmiProductName, " ", "-") -} - // Load reads profilesDir/.yaml and returns the typed Profile. func Load(profilesDir, name string) (Profile, error) { path := filepath.Join(profilesDir, name+".yaml") @@ -213,9 +187,6 @@ func Load(profilesDir, name string) (Profile, error) { architecture: strings.ToLower(strings.TrimSpace(raw.DeviceDefaults.Architecture)), } p.applyOptionalDeviceDefaults(raw) - if raw.DMI != nil { - p.dmiProductName = strings.TrimSpace(raw.DMI.ProductName) - } // render-pci-sysfs falls back to a flat single-root layout when a profile // declares no pcie_topology block, so an empty list still means 1 root. p.pciRoots = len(raw.PCIeTopology.RootComplexes) diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index 72c269e75..e9060501c 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -85,28 +85,7 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( Expect(err).NotTo(HaveOccurred()) assertions.PCISysfsAtKernelPath(ctx, h.Kube, kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}, - p.ExpectedGPUs(), p.DMIProductName()) - }) - - It("labels the machine type from the mock DMI identity", Label("device-plugin"), func(ctx SpecContext) { - // Nothing configures GFD to find it: its default machine-type - // file, /sys/class/dmi/id/product_name, is a symlink into the - // mounted /sys/devices subtree (#673). - // - // That holds only where the kernel exposes DMI. Where it does - // not — Docker Desktop's linuxkit VM — /sys/class/dmi is absent - // and no mount can create it, so the label reads "unknown" for - // reasons the mock cannot influence and there is nothing to - // assert. - pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") - Expect(err).NotTo(HaveOccurred()) - if !assertions.DMIExposedByKernel(ctx, h.Kube, - kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}) { - Skip("kernel exposes no DMI, so GFD's default machine-type file cannot resolve") - } - assertions.WaitGFDLabels(ctx, h.Kube, node, - assertions.ExpectedMachineTypeLabel(p.GFDMachineType()), - config.ReadyTimeout(), config.PollInterval()) + p.ExpectedGPUs()) }) It("exports DCGM device metrics that vary over time", Label("dcgm"), func(ctx SpecContext) { diff --git a/tests/e2e/gpu-operator-values.yaml b/tests/e2e/gpu-operator-values.yaml index 1bbbad13e..9e08a7416 100644 --- a/tests/e2e/gpu-operator-values.yaml +++ b/tests/e2e/gpu-operator-values.yaml @@ -77,11 +77,6 @@ gfd: env: - name: NVIDIA_DRIVER_ROOT value: "/var/lib/nvml-mock/driver" - # nvidia.com/gpu.machine is deliberately left unconfigured: GFD's default - # /sys/class/dmi/id/product_name is a symlink into the mock's mounted - # /sys/devices/virtual/dmi/id, so it resolves on any host whose kernel - # exposes DMI. Hosts without it (Docker Desktop) read "unknown", which the - # scenario detects rather than asserts. # Validator checks that the GPU stack is functional. # driver-validation: mounts hostPath /run/nvidia/driver → our mock (via symlink). From 1d4400a0e3869276a9343cc29262989a106ef477 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Wed, 19 Aug 2026 12:30:13 +0200 Subject: [PATCH 06/11] docs(e2e): drop the stale busId caveat from the PCI tree spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment said gpu.mode was blocked on an unrelated fix, which #672 landed on main. What the spec pins is unchanged — that the tree reaches the container — so say only that. Signed-off-by: Giulio Calzolari --- tests/e2e/go/scenario_gpu_operator_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index e9060501c..e0d9312c4 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -78,9 +78,8 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( It("serves the rendered PCI tree to the GFD container at the kernel paths", Label("device-plugin"), func(ctx SpecContext) { // The NVML-derived labels above cannot distinguish "GFD read // the mock tree" from "GFD read the host's sysfs and happened - // to agree", and gpu.mode — the label that would fail loudest - // — is blocked on an unrelated busId fix (#671). Reading the - // tree from inside the container pins the delivery itself. + // to agree". Reading the tree from inside the container pins + // the delivery itself, independent of what GFD makes of it. pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") Expect(err).NotTo(HaveOccurred()) assertions.PCISysfsAtKernelPath(ctx, h.Kube, From ffeef779e583b58fa2595a15daa5df68439acb20 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Wed, 19 Aug 2026 16:48:55 +0200 Subject: [PATCH 07/11] fix(nvml-mock): serve only what the current render put in the tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the served tree could describe something other than the profile the node is running, all found in review of #674. Rendering only ever added entries. Re-profiling a node (an a100 and an h100 share no BDFs) left both sets in place and consumers saw their union mounted at /sys/bus/pci/devices — more GPUs than the node simulates, some under a root complex no profile declares. Each render now prunes first. The staged-tree guards stat the directories they mount, which the renderer creates at the very start while the DMI attributes kind's createContainer hook needs are written at its end. A tree caught mid-render passed, got mounted, and then failed container creation on a missing bind target — the failure the guard exists to prevent. The renderer marks completion with sys/.rendered and both the NRI plugin and setup.sh gate on that instead. The prune keeps the two mounted directories and the DMI directory in place for the same reason, since a CDI-served container cannot wait for a marker: the runtime applies the spec's mounts unconditionally. product_uuid was mirrored by value at mode 0644, republishing a node identifier the kernel deliberately exposes 0400 to root alone into every served container. Only the file's existence matters — kind bind-mounts the node's own copy over it — so it is now an empty stand-in, and product_name alone travels by value. The chart README's numa_node limitation gains the qualifier the row above it already has: the renderer writes numa_node for every device and it reaches served containers, so a served device plugin does get NUMA hints. The node-wide NRI mount also names nri.optOutAnnotation, which exempts a single workload, next to the namespace-level nri.excludedNamespaces. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 26 ++-- .../nvml-mock/helm/nvml-mock/README.md | 27 ++-- deployments/nvml-mock/scripts/setup.sh | 11 +- pkg/nri/nvmlmock/adjust.go | 20 ++- pkg/nri/nvmlmock/adjust_test.go | 43 +++++- pkg/system/mockpcisysfs/render/render.go | 137 ++++++++++++++--- pkg/system/mockpcisysfs/render/render_test.go | 138 ++++++++++++++++-- 7 files changed, 335 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25d86bf70..6ce267316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,15 +37,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 profile's root complexes, because a bind mount at a path sysfs lacks needs a mountpoint the runtime cannot create on a read-only `/sys` — which hides the host's other device classes from served containers. Under NRI, which injects - node-wide, keep workloads that need the host's real device tree in a namespace - listed in `nri.excludedNamespaces`. The rendered tree also mirrors the node's - DMI attributes (`product_name`, `product_uuid`) into - `sys/devices/virtual/dmi/id`, because shadowing `/sys/devices` shadows the - directory `/sys/class/dmi/id` resolves into: kind's `mount-product-files.sh` - createContainer hook bind-mounts the node's product files there for every - container, and `mount(8)` cannot create a target on a read-only sysfs. The - attributes are mirrored, not mocked: `nvidia.com/gpu.machine` still reports - what the node itself reports, tracked in #681. (#673) + node-wide, exempt a workload that needs the host's real device tree with the + `nvml-mock.nvidia.com/inject: "false"` pod annotation, or a whole namespace + with `nri.excludedNamespaces`. The rendered tree also carries the node's DMI + attributes in `sys/devices/virtual/dmi/id`, because shadowing `/sys/devices` + shadows the directory `/sys/class/dmi/id` resolves into: kind's + `mount-product-files.sh` createContainer hook bind-mounts the node's product + files there for every container, and `mount(8)` cannot create a target on a + read-only sysfs. `product_name` is mirrored by value; `product_uuid` is an + empty stand-in, since kind mounts the node's own copy over it and the value is + a node identifier the kernel exposes to root alone. The attributes are + mirrored, not mocked: `nvidia.com/gpu.machine` still reports what the node + itself reports, tracked in #681. Each render now replaces the previous tree + instead of adding to it, so re-profiling a node no longer serves both + profiles' devices, and both mount channels gate on a completion marker the + renderer writes last — the mounted directories exist from the start of a + render, so their presence alone would serve a tree still missing the bind + targets kind's hook needs. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index 755641dc7..7736e32e3 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -736,19 +736,24 @@ directory yields entries that list but whose every attribute read fails with device classes (CPU topology among them) from those containers. It cannot be narrowed to the profile's root complexes — a bind mount at a path sysfs does not already have needs a mountpoint, and the runtime cannot create one on a -read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so this -is not configurable; keep workloads that need the host's real device tree off -it, or in a namespace listed in `nri.excludedNamespaces`. +read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so +serving the tree is not itself configurable. Through CDI a container is served +only if it requests a mock GPU. Through NRI, which injects ambiently, the two +existing escape hatches cover it: the pod annotation +`nvml-mock.nvidia.com/inject: "false"` (`nri.optOutAnnotation`) exempts a +single workload, and `nri.excludedNamespaces` exempts a whole namespace. `/sys/devices/virtual/dmi/id` — the directory `/sys/class/dmi/id` resolves into — is shadowed along with the rest, so the renderer mirrors the node's -`product_name` and `product_uuid` into the tree. This is not cosmetic: kind's -`mount-product-files.sh` createContainer hook bind-mounts the node's copies of -both onto every container it starts, and `mount(8)` cannot create a target on a -read-only sysfs, so a missing attribute fails container creation for every pod -the mock serves. The values are mirrored, not mocked — a node keeps reporting -its own machine type, which under kind is the literal `kind`, so -`nvidia.com/gpu.machine` does not follow the profile. Tracked in +`product_name` there and leaves an empty `product_uuid` beside it. This is not +cosmetic: kind's `mount-product-files.sh` createContainer hook bind-mounts the +node's copies of both onto every container it starts, and `mount(8)` cannot +create a target on a read-only sysfs, so a missing attribute fails container +creation for every pod the mock serves. `product_uuid` is a node identifier the +kernel exposes to root alone and kind mounts its own copy over it, so only the +target is reproduced, never the value. `product_name` is mirrored, not mocked — +a node keeps reporting its own machine type, which under kind is the literal +`kind`, so `nvidia.com/gpu.machine` does not follow the profile. Tracked in [#681](https://github.com/NVIDIA/k8s-test-infra/issues/681). ### Cross-node `ibping` @@ -1406,7 +1411,7 @@ discovery and monitoring. Some host-level subsystems are not mocked: | What's Missing | Affected Consumer | Impact | |----------------|-------------------|--------| | `/sys/bus/pci/devices/{busID}` sysfs entries in containers the mock does **not** serve | DRA driver | The tree is now bind-mounted onto the kernel paths for CDI- and NRI-served containers, which is what Go consumers need (see [PCI sysfs in containers](#pci-sysfs-in-containers)). A consumer deployed outside those channels still reads the host's real sysfs; whether `dra.k8s.io/pcieRoot` reaches ResourceSlices is tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) | -| `/sys/bus/pci/devices/{busID}/numa_node` | Device plugin | NUMA-aware topology hints unavailable; scheduling works but NUMA affinity not enforced | +| `/sys/bus/pci/devices/{busID}/numa_node` in containers the mock does **not** serve | Device plugin | The renderer writes `numa_node` for every device and it arrives through the same mount, so a served device plugin does get NUMA hints. Outside those channels the hints are unavailable: scheduling works but NUMA affinity is not enforced | | `/sys/bus/pci/devices/*/vendor,device,class` **as NFD reads them** (`/host-sys/…`, fixed at link time) | NFD (Node Feature Discovery) | PCI feature labels not auto-detected. `nvidia.com/gpu.present` is written directly by nvml-mock; `pci-10de.present` is created by NFD from a feature file nvml-mock drops in `nodeLabels.featuresDir` — see [Node Labels](#node-labels) | ### PCIe Root Complex (DRA driver) diff --git a/deployments/nvml-mock/scripts/setup.sh b/deployments/nvml-mock/scripts/setup.sh index dbdb842a8..b36890be7 100644 --- a/deployments/nvml-mock/scripts/setup.sh +++ b/deployments/nvml-mock/scripts/setup.sh @@ -121,7 +121,9 @@ fi # renderer parses the profile's `pcie_topology:` block; profiles without # one get a flat default covering every device under a single root # complex (`pci0000:00`, NUMA 0). It also mirrors the node's DMI identity -# into the tree, which the mount below depends on. Failures are fatal +# into the tree, which the mount below depends on. Each run replaces the +# previous tree rather than adding to it, so re-profiling a node does not +# leave it serving both profiles' devices. Failures are fatal # under `set -e` for the same reason as the IB render below — a topology # typo otherwise yields silently malformed sysfs that downstream # `dra.k8s.io/pcieRoot` attributes would inherit. @@ -131,12 +133,17 @@ fi # missing fails container creation for the whole pod. PCI_ROOT="$HOST" mkdir -p "$PCI_ROOT" +# Keep in sync with render.MarkerRelPath (pkg/system/mockpcisysfs/render): the +# renderer writes it last, once the whole tree is on disk. Gating on the +# directories instead would accept a tree from a previous profile that this +# run had nothing to render over, and would say "rendered" partway through. +PCI_SYSFS_MARKER=sys/.rendered PCI_SYSFS_RENDERED=off if [ -x /usr/local/bin/render-pci-sysfs ]; then /usr/local/bin/render-pci-sysfs \ --config /etc/nvml-mock/config.yaml \ --output "$PCI_ROOT" - if [ -d "$PCI_ROOT/sys/bus/pci/devices" ] && [ -d "$PCI_ROOT/sys/devices" ]; then + if [ -f "$PCI_ROOT/$PCI_SYSFS_MARKER" ]; then PCI_SYSFS_RENDERED=on fi fi diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index 68964781b..96335ecfa 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -11,6 +11,8 @@ import ( "path/filepath" "sort" "strings" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) // warnf logs a non-fatal condition. It is a package var so tests can capture @@ -369,15 +371,23 @@ func topologyInjectable(cfg Config) bool { // createContainer hook bind-mounts the node's product files there, and a // missing target fails container creation. // -// Missing sources are skipped rather than reported: the tree is staged by -// the main nvml-mock DaemonSet and nothing orders this plugin after it, and -// a bind mount whose source does not exist fails container creation for the -// whole pod. Silence rather than a warning because this runs for every -// container on the node, staged or not. +// An unfinished tree is skipped rather than reported: it is staged by the +// main nvml-mock DaemonSet and nothing orders this plugin after it, and a +// mount the tree cannot honour fails container creation for the whole pod. +// Silence rather than a warning because this runs for every container on the +// node, staged or not. +// +// "Finished" is the renderer's marker, not the presence of the directories +// mounted here: those exist from the start of a render while the DMI +// attributes kind's hook needs are written at its end, so a tree caught +// mid-render would otherwise pass and then fail every container on the node. func pciSysfsMounts(cfg Config) []Mount { if cfg.HostOverlayPath == "" { return nil } + if _, err := os.Stat(filepath.Join(cfg.HostOverlayPath, render.MarkerRelPath)); err != nil { + return nil + } sysDevices := filepath.Join(cfg.HostOverlayPath, sysDevicesRelPath) pciDevices := filepath.Join(cfg.HostOverlayPath, pciDevicesRelPath) for _, dir := range []string{sysDevices, pciDevices} { diff --git a/pkg/nri/nvmlmock/adjust_test.go b/pkg/nri/nvmlmock/adjust_test.go index a8d25d673..3e0946171 100644 --- a/pkg/nri/nvmlmock/adjust_test.go +++ b/pkg/nri/nvmlmock/adjust_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) func TestAdjustPlainContainerAddsOverlayAndEnvironment(t *testing.T) { @@ -58,8 +60,7 @@ func TestAdjustPlainContainerAddsOverlayAndEnvironment(t *testing.T) { // the rendered sys/devices is mounted too. func TestAdjustMountsPCISysfsWhenStaged(t *testing.T) { overlay := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) + stagePCISysfs(t, overlay) cfg := DefaultConfig() cfg.HostOverlayPath = overlay @@ -106,7 +107,8 @@ func TestAdjustSkipsPCISysfsMountsWhenNotStaged(t *testing.T) { // fix, but harder to diagnose because the entries appear to be there. func TestAdjustSkipsPCIDevicesMountWithoutSysDevices(t *testing.T) { overlay := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) + stagePCISysfs(t, overlay) + require.NoError(t, os.RemoveAll(filepath.Join(overlay, "sys/devices"))) cfg := DefaultConfig() cfg.HostOverlayPath = overlay @@ -121,6 +123,41 @@ func TestAdjustSkipsPCIDevicesMountWithoutSysDevices(t *testing.T) { } } +// TestAdjustSkipsPCISysfsMountsWhileRenderIncomplete covers the window inside +// a render: the directories these mounts name are created at its start, while +// the DMI attributes kind's createContainer hook bind-mounts the node's +// product files onto are written at its end. Mounting in between hands the +// container a tree missing those targets, and mount(8) cannot create one on a +// read-only sysfs — container creation fails, which is the failure the guard +// exists to prevent. The renderer's completion marker is what distinguishes +// the two states. +func TestAdjustSkipsPCISysfsMountsWhileRenderIncomplete(t *testing.T) { + overlay := t.TempDir() + stagePCISysfs(t, overlay) + require.NoError(t, os.Remove(filepath.Join(overlay, render.MarkerRelPath))) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "an incomplete tree must yield no sysfs mounts, got %+v", mount) + } +} + +// stagePCISysfs stages a completely rendered PCI sysfs tree in the overlay, +// as the main DaemonSet's render-pci-sysfs run leaves it. +func stagePCISysfs(t *testing.T, overlay string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(overlay, render.MarkerRelPath), nil, 0o644)) +} + func TestAdjustEmitsOnlyAddedOrChangedEnv(t *testing.T) { container := Container{ Namespace: "default", diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 17813fd76..60e82e9ff 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -66,37 +66,106 @@ type Options struct { DMISource string } -// Render writes the entire tree. It is idempotent: existing directories -// are reused, existing files are truncated and rewritten, and existing -// symlinks are removed and recreated so a stale relative target does not -// linger across re-renders. +// MarkerRelPath is written last, once the whole tree — topology and mirrored +// DMI attributes alike — is on disk. Consumers that bind-mount the tree onto +// the kernel paths gate on it rather than on the directories they mount: +// those are created at the start of a render, so their presence says nothing +// about whether the render finished, and serving a half-rendered tree fails +// container creation on a bind target that is not there yet. +// +// It sits outside both mounted subtrees, so it is not visible to a container +// the tree is served to. +const MarkerRelPath = "sys/.rendered" + +const ( + pciDevicesRelPath = "sys/bus/pci/devices" + sysDevicesRelPath = "sys/devices" +) + +// Render writes the entire tree, replacing whatever a previous render left +// behind, and marks it complete with MarkerRelPath. Within a render existing +// files are truncated and rewritten, and existing symlinks are removed and +// recreated so a stale relative target cannot linger. func Render(o Options) error { if !o.hasTopology() { - // Nothing to do — caller decided to render a profile with no - // declared topology and no devices. Treat as a no-op so the - // renderer can be invoked unconditionally from setup.sh. - return nil + // Nothing to render — the caller passed a profile with no declared + // topology and no devices, which setup.sh does unconditionally. A tree + // left here by a previous profile would still describe the node, so it + // is emptied rather than kept. + if o.Output == "" { + return nil + } + return pruneTree(o.Output) } if o.Output == "" { return errors.New("pcisysfs render: Output is required") } + if err := pruneTree(o.Output); err != nil { + return err + } if err := renderTopology(o); err != nil { return err } - return renderDMI(o.Output, o.DMISource) + if err := renderDMI(o.Output, o.DMISource); err != nil { + return err + } + return writeFile(o.Output, MarkerRelPath, "") } func (o Options) hasTopology() bool { return o.Topology != nil && len(o.Topology.RootComplexes) > 0 } +// pruneTree drops the devices a previous render left behind. Rendering only +// ever added entries, so without this a re-profiled node keeps both profiles' +// devices (an a100 and an h100 share no BDFs) and consumers see their union +// mounted at /sys/bus/pci/devices — more GPUs than the node simulates, some +// under a root complex no profile declares. +// +// What survives is deliberate: the two directories consumers bind-mount, and +// the DMI directory holding the targets kind's createContainer hook needs. A +// container created while a render is in flight then still finds every mount's +// source and every target in place — it may see fewer devices than the profile +// declares, but it starts. Consumers served through CDI have no way to wait +// for MarkerRelPath, since the runtime applies the spec's mounts unconditionally. +func pruneTree(root string) error { + if err := os.RemoveAll(filepath.Join(root, MarkerRelPath)); err != nil { + return fmt.Errorf("clear %s: %w", MarkerRelPath, err) + } + if err := removeEntries(filepath.Join(root, pciDevicesRelPath), ""); err != nil { + return err + } + return removeEntries(filepath.Join(root, sysDevicesRelPath), dmiVirtualDirName) +} + +// removeEntries empties dir, keeping the entry named keep (if any). A missing +// dir is not an error: there is nothing to prune on a first render. +func removeEntries(dir, keep string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read %s: %w", dir, err) + } + for _, entry := range entries { + if entry.Name() == keep { + continue + } + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return fmt.Errorf("clear %s: %w", filepath.Join(dir, entry.Name()), err) + } + } + return nil +} + func renderTopology(o Options) error { root := o.Output - if err := mkdirAll(root, "sys/bus/pci/devices"); err != nil { + if err := mkdirAll(root, pciDevicesRelPath); err != nil { return err } - if err := mkdirAll(root, "sys/devices"); err != nil { + if err := mkdirAll(root, sysDevicesRelPath); err != nil { return err } @@ -110,12 +179,30 @@ func renderTopology(o Options) error { // dmiIDDir is where the kernel materializes the SMBIOS identity; the // familiar /sys/class/dmi/id path is only a symlink into it. -const dmiIDDir = "sys/devices/virtual/dmi/id" +// dmiVirtualDirName names its top-level directory under sys/devices, which +// pruneTree keeps so the mount targets inside it never go missing. +const ( + dmiVirtualDirName = "virtual" + dmiIDDir = sysDevicesRelPath + "/" + dmiVirtualDirName + "/dmi/id" +) // dmiMirroredAttrs are the DMI attributes kind's mount-product-files.sh // createContainer hook bind-mounts the node's copies onto, for every -// container on the node. -var dmiMirroredAttrs = []string{"product_name", "product_uuid"} +// container on the node. Each has to exist in the tree as a mount target; +// byValue says whether the node's value travels with it. +var dmiMirroredAttrs = []struct { + name string + byValue bool +}{ + // The node's machine type, which consumers do read: GFD's default + // machine-type file resolves here. + {name: "product_name", byValue: true}, + // A node identifier the kernel deliberately exposes 0400 to root alone. + // Only its existence matters, since kind mounts the node's own copy over + // it, so the tree carries an empty stand-in rather than republishing the + // value world-readable into every served container. + {name: "product_uuid"}, +} // renderDMI mirrors the node's DMI identity into the tree. Serving the tree // means bind-mounting it over /sys/devices, which also replaces @@ -132,19 +219,21 @@ func renderDMI(root, source string) error { return nil } for _, attr := range dmiMirroredAttrs { - src := filepath.Join(source, attr) + src := filepath.Join(source, attr.name) if _, err := os.Stat(src); err != nil { // The kernel exposes no such attribute, so nothing bind-mounts // it either and a stand-in would only invent an identity. continue } - // product_uuid is mode 0400 on most kernels: unreadable to a - // non-root renderer, yet still needed as a mount target. - contents, err := os.ReadFile(src) - if err != nil { - contents = nil + var contents []byte + if attr.byValue { + // A read failure still has to leave the file behind: the target + // matters more than the value, and mount(8) cannot create one. + if value, err := os.ReadFile(src); err == nil { + contents = value + } } - if err := writeFile(root, filepath.Join(dmiIDDir, attr), string(contents)); err != nil { + if err := writeFile(root, filepath.Join(dmiIDDir, attr.name), string(contents)); err != nil { return err } } @@ -152,7 +241,7 @@ func renderDMI(root, source string) error { } func renderRootComplex(root string, rc config.RootComplex, ids map[string]config.PCI) error { - rcDir := filepath.Join("sys/devices", rc.ID) + rcDir := filepath.Join(sysDevicesRelPath, rc.ID) if err := mkdirAll(root, rcDir); err != nil { return err } @@ -183,11 +272,11 @@ func renderRootComplex(root string, rc config.RootComplex, ids map[string]config // Relative target matches what the kernel emits, so any // readlink() consumer (`realpath`, deviceattribute, etc.) // resolves to the same canonical path it would on real Linux. - linkPath := filepath.Join(root, "sys/bus/pci/devices", bdfLC) + linkPath := filepath.Join(root, pciDevicesRelPath, bdfLC) linkTarget := filepath.Join("..", "..", "..", "devices", rc.ID, bdfLC) if err := replaceSymlink(linkPath, linkTarget); err != nil { return fmt.Errorf("symlink %s -> %s: %w", - filepath.Join("sys/bus/pci/devices", bdfLC), linkTarget, err) + filepath.Join(pciDevicesRelPath, bdfLC), linkTarget, err) } } return nil diff --git a/pkg/system/mockpcisysfs/render/render_test.go b/pkg/system/mockpcisysfs/render/render_test.go index 47bcd5032..a11fa1643 100644 --- a/pkg/system/mockpcisysfs/render/render_test.go +++ b/pkg/system/mockpcisysfs/render/render_test.go @@ -200,6 +200,111 @@ func TestRender_IdempotentRerender(t *testing.T) { require.Equal(t, "3\n", string(got), "numa_node not updated") } +// TestRender_PrunesStaleDevices covers re-profiling a node: an a100 and an +// h100 profile share no BDFs, and the renderer used to only add entries, so +// both sets stayed under the tree and consumers saw their union mounted at +// /sys/bus/pci/devices — more GPUs than the node simulates, some of them +// pointing at a root complex no profile declares. +func TestRender_PrunesStaleDevices(t *testing.T) { + dir := t.TempDir() + previous := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: previous, Output: dir}), "Render previous profile") + + current := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:c0", NUMANode: 3, + Devices: []string{"0000:1a:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: current, Output: dir}), "Render current profile") + + entries, err := os.ReadDir(filepath.Join(dir, "sys/bus/pci/devices")) + require.NoError(t, err, "read devices dir") + require.Len(t, entries, 1, "stale device symlinks survived the re-render") + require.Equal(t, "0000:1a:00.0", entries[0].Name(), "device") + + _, err = os.Stat(filepath.Join(dir, "sys/devices/pci0000:00")) + require.True(t, os.IsNotExist(err), "stale root complex survived, got err=%v", err) +} + +// TestRender_MarkerFollowsTheTree pins the signal consumers gate their bind +// mounts on. The mounted directories are created at the start of a render and +// the DMI attributes are written at its end, so their presence cannot mean +// "complete" — and mounting an incomplete tree fails container creation on a +// bind target that is not there yet. +func TestRender_MarkerFollowsTheTree(t *testing.T) { + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir}), "Render") + _, err := os.Stat(filepath.Join(dir, MarkerRelPath)) + require.NoError(t, err, "marker missing after a complete render") + + // A profile with nothing to render must leave no marker: a stale one would + // keep consumers mounting the previous profile's devices. + require.NoError(t, Render(Options{Output: dir}), "Render without topology") + _, err = os.Stat(filepath.Join(dir, MarkerRelPath)) + require.True(t, os.IsNotExist(err), "marker survived an empty render, got err=%v", err) + entries, err := os.ReadDir(filepath.Join(dir, "sys/bus/pci/devices")) + require.NoError(t, err, "read devices dir") + require.Empty(t, entries, "devices survived an empty render") +} + +// TestRender_KeepsMountedPathsWhilePruning pins what a re-render must not take +// away. Both mounted directories, and the DMI attributes kind's +// createContainer hook bind-mounts the node's product files onto, are targets +// of mounts already in effect for containers the CDI spec serves — and the CDI +// path cannot wait for the marker, since the runtime applies the spec's mounts +// unconditionally. Removing them mid-render would fail container creation for +// pods that have nothing to do with the re-profiling. +func TestRender_KeepsMountedPathsWhilePruning(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("kind\n"), 0o644), "stage product_name") + require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("dead-beef\n"), 0o644), "stage product_uuid") + + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") + + // Watch the directories across a prune: they must be the same inodes + // afterwards, since a bind mount whose source was replaced still resolves + // to the vanished original. + sysDevices := statOrFail(t, filepath.Join(dir, "sys/devices")) + pciDevices := statOrFail(t, filepath.Join(dir, "sys/bus/pci/devices")) + + require.NoError(t, pruneTree(dir), "pruneTree") + + require.True(t, os.SameFile(sysDevices, statOrFail(t, filepath.Join(dir, "sys/devices"))), + "sys/devices replaced by the prune") + require.True(t, os.SameFile(pciDevices, statOrFail(t, filepath.Join(dir, "sys/bus/pci/devices"))), + "sys/bus/pci/devices replaced by the prune") + for _, attr := range []string{"product_name", "product_uuid"} { + _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual/dmi/id", attr)) + require.NoError(t, err, "%s removed by the prune", attr) + } +} + +func statOrFail(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err, "stat %s", path) + return info +} + func TestRender_NormalizesUppercaseBDF(t *testing.T) { dir := t.TempDir() topo := &config.PCIeTopology{ @@ -219,8 +324,11 @@ func TestRender_NormalizesUppercaseBDF(t *testing.T) { // TestRender_MirrorsKernelDMI covers the reason the tree carries a DMI // directory at all: bind-mounting it over /sys/devices hides the real // virtual/dmi/id, which kind's mount-product-files.sh hook bind-mounts the -// node's product files onto for every container. Mirroring the attributes -// keeps those mount targets in place and leaves the node's identity intact. +// node's product files onto for every container. Both attributes must exist +// as mount targets; only product_name travels by value. product_uuid is a +// node identifier the kernel exposes 0400 to root alone, and kind mounts the +// node's own copy over it anyway, so the tree must not republish it into +// every served container. func TestRender_MirrorsKernelDMI(t *testing.T) { src := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("kind\n"), 0o644), "stage product_name") @@ -235,22 +343,26 @@ func TestRender_MirrorsKernelDMI(t *testing.T) { } require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") - for name, want := range map[string]string{"product_name": "kind\n", "product_uuid": "dead-beef\n"} { - got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id", name)) - require.NoError(t, err, "read %s", name) - require.Equal(t, want, string(got), name) - } + dmi := filepath.Join(dir, "sys/devices/virtual/dmi/id") + name, err := os.ReadFile(filepath.Join(dmi, "product_name")) + require.NoError(t, err, "read product_name") + require.Equal(t, "kind\n", string(name), "product_name") + + uuid, err := os.ReadFile(filepath.Join(dmi, "product_uuid")) + require.NoError(t, err, "read product_uuid") + require.Empty(t, uuid, "product_uuid exists as a mount target, without the node's value") } -// TestRender_StandsInForUnreadableDMI covers product_uuid, which the kernel -// exposes mode 0400: the value cannot be mirrored, but the file must still -// exist, because mount(8) cannot create a target on a read-only sysfs. +// TestRender_StandsInForUnreadableDMI covers an attribute the renderer cannot +// read — product_name is mode 0444 on every kernel we know of, but a mirror +// that failed on a permission error would leave the mount target missing, and +// mount(8) cannot create one on a read-only sysfs. func TestRender_StandsInForUnreadableDMI(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads any mode; the permission branch is unreachable") } src := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("secret\n"), 0o000), "stage product_uuid") + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("secret\n"), 0o000), "stage product_name") dir := t.TempDir() topo := &config.PCIeTopology{ @@ -261,8 +373,8 @@ func TestRender_StandsInForUnreadableDMI(t *testing.T) { } require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") - got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_uuid")) - require.NoError(t, err, "read product_uuid") + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) + require.NoError(t, err, "read product_name") require.Empty(t, got, "an unreadable attribute renders as an empty stand-in") } From 2c4b3dd7bd115508dfefe2c80555287e0c0823ca Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Wed, 19 Aug 2026 16:49:04 +0200 Subject: [PATCH 08/11] test(e2e): read the PCI tree from the GFD pod on the node under test FirstPodName returns Items[0] with no phase filter, so a Terminating or Pending GFD pod matches the selector as readily as the Running one and the exec into it fails. Nor is Items[0] necessarily on the node the surrounding specs assert about, so the spec could report about hardware it never checked labels for. RunningPodOnNode adds both filters, which is what a caller execing into a DaemonSet's pod wants. Signed-off-by: Giulio Calzolari --- tests/e2e/go/framework/kube/kube.go | 19 +++++++++++++++++++ tests/e2e/go/scenario_gpu_operator_test.go | 6 +++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/e2e/go/framework/kube/kube.go b/tests/e2e/go/framework/kube/kube.go index 82b493c36..b68e0fd3c 100644 --- a/tests/e2e/go/framework/kube/kube.go +++ b/tests/e2e/go/framework/kube/kube.go @@ -285,6 +285,25 @@ func (c *Client) RunningPodNames(ctx context.Context, ns, selector string) ([]st return out, nil } +// RunningPodOnNode returns the Running pod matching the selector on a given +// node. Callers that exec into a DaemonSet's pod need both filters: a +// Terminating or Pending pod matches the selector just as well and the exec +// fails, and a pod on another node answers about hardware the assertion is not +// about. +func (c *Client) RunningPodOnNode(ctx context.Context, ns, selector, node string) (string, error) { + var pl podList + if err := c.getJSON(ctx, &pl, "pods", "-n", ns, "-l", selector, + "--field-selector", "spec.nodeName="+node); err != nil { + return "", err + } + for _, p := range pl.Items { + if p.Status.Phase == "Running" { + return p.Metadata.Name, nil + } + } + return "", fmt.Errorf("no Running pod in ns %q matching %q on node %q", ns, selector, node) +} + // PodNode returns the Kubernetes node a pod is scheduled on. func (c *Client) PodNode(ctx context.Context, ns, name string) (string, error) { var p podObj diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index e0d9312c4..0cc873f6f 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -80,7 +80,11 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( // the mock tree" from "GFD read the host's sysfs and happened // to agree". Reading the tree from inside the container pins // the delivery itself, independent of what GFD makes of it. - pod, err := h.Kube.FirstPodName(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery") + // The GFD pod on `node`, and only while it is Running: the + // specs above assert about that node's labels, and a + // Terminating or Pending pod matches the selector too, which + // the exec below would fail on. + pod, err := h.Kube.RunningPodOnNode(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery", node) Expect(err).NotTo(HaveOccurred()) assertions.PCISysfsAtKernelPath(ctx, h.Kube, kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}, From 6b827e3999dddecd70d9c95842c343ff45f359e0 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Thu, 20 Aug 2026 15:03:59 +0200 Subject: [PATCH 09/11] fix(nvml-mock): clear the tree for a profile that renders no devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render-pci-sysfs returned before Render whenever the profile yielded no topology, which is any profile whose devices declare no pci.bus_id — a config gpu.customConfig makes reachable. The prune and the completion marker therefore never ran on that path: re-profiling a node from a100 onto such a config left all eight A100 symlinks and sys/.rendered on disk, so setup.sh flipped PCI_SYSFS_RENDERED=on and both channels went on mounting the previous profile's devices at the kernel paths. Render already treats a nil topology as "empty the tree", so the command now calls it unconditionally and only --dry-run short-circuits. main is split into a testable run() to cover the case that regressed. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 4 +- cmd/render-pci-sysfs/main.go | 74 +++++++++++++++-------- cmd/render-pci-sysfs/main_test.go | 81 ++++++++++++++++++++++++++ deployments/nvml-mock/scripts/setup.sh | 7 ++- 4 files changed, 139 insertions(+), 27 deletions(-) create mode 100644 cmd/render-pci-sysfs/main_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ce267316..33980e17f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mirrored, not mocked: `nvidia.com/gpu.machine` still reports what the node itself reports, tracked in #681. Each render now replaces the previous tree instead of adding to it, so re-profiling a node no longer serves both - profiles' devices, and both mount channels gate on a completion marker the + profiles' devices — including a re-profile onto a config that declares no PCI + devices at all, which clears the tree rather than leaving the previous one to + describe the node — and both mount channels gate on a completion marker the renderer writes last — the mounted directories exist from the start of a render, so their presence alone would serve a tree still missing the bind targets kind's hook needs. (#673) diff --git a/cmd/render-pci-sysfs/main.go b/cmd/render-pci-sysfs/main.go index 9600fa9ca..ae84451bb 100644 --- a/cmd/render-pci-sysfs/main.go +++ b/cmd/render-pci-sysfs/main.go @@ -39,58 +39,86 @@ import ( // /sys/class/dmi/id resolves into. const defaultDMISource = "/sys/class/dmi/id" -//nolint:cyclop // existing complexity; refactor deferred func main() { var ( - cfgPath = flag.String("config", "", "path to mock-nvml profile YAML") - outDir = flag.String("output", "", "fake-root directory; tree is written under /sys/...") + opts options dmiSource = flag.String("dmi-source", defaultDMISource, "kernel DMI directory to mirror into the tree; empty mirrors nothing") - strict = flag.Bool("strict", false, "fail if the profile does not declare `pcie_topology:`") - dryRun = flag.Bool("dry-run", false, "validate the config and exit without writing files") ) + flag.StringVar(&opts.configPath, "config", "", "path to mock-nvml profile YAML") + flag.StringVar(&opts.outputDir, "output", "", "fake-root directory; tree is written under /sys/...") + flag.BoolVar(&opts.strict, "strict", false, "fail if the profile does not declare `pcie_topology:`") + flag.BoolVar(&opts.dryRun, "dry-run", false, "validate the config and exit without writing files") flag.Parse() + opts.dmiSource = *dmiSource - if *cfgPath == "" || *outDir == "" { + if opts.configPath == "" || opts.outputDir == "" { fmt.Fprintln(os.Stderr, "usage: render-pci-sysfs --config --output [--strict] [--dry-run]") os.Exit(2) } - data, err := os.ReadFile(*cfgPath) + if err := run(opts); err != nil { + fatalf("%v", err) + } +} + +// options is the resolved command line. +type options struct { + configPath string + outputDir string + dmiSource string + strict bool + dryRun bool +} + +func run(o options) error { + data, err := os.ReadFile(o.configPath) if err != nil { - fatalf("read config: %v", err) + return fmt.Errorf("read config: %w", err) } var prof config.Profile if err := yaml.Unmarshal(data, &prof); err != nil { - fatalf("parse config: %v", err) + return fmt.Errorf("parse config: %w", err) } if err := prof.Validate(); err != nil { - fatalf("%v", err) + return err } topo := prof.EffectiveTopology() - if topo == nil { - fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render\n", *cfgPath) - return + if topo != nil && o.strict && prof.PCIeTopology == nil { + return fmt.Errorf("--strict: profile %s does not declare `pcie_topology:`", o.configPath) } - if *strict && prof.PCIeTopology == nil { - fatalf("--strict: profile %s does not declare `pcie_topology:`", *cfgPath) + if o.dryRun { + reportDryRun(o.configPath, topo) + return nil } - if *dryRun { - fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n", - len(topo.RootComplexes), countDevices(topo)) - return + // A profile with no devices still goes through Render, rather than + // returning here: a tree rendered from a previous profile is on disk and + // still served, and Render is what clears it along with its completion + // marker, so setup.sh's gate cannot flip on for devices this profile does + // not declare. + if topo == nil { + fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, clearing any previously rendered tree\n", o.configPath) } - if err := render.Render(render.Options{ Topology: topo, Identities: prof.DeviceIdentities(), - Output: *outDir, - DMISource: *dmiSource, + Output: o.outputDir, + DMISource: o.dmiSource, }); err != nil { - fatalf("render: %v", err) + return fmt.Errorf("render: %w", err) + } + return nil +} + +func reportDryRun(configPath string, topo *config.PCIeTopology) { + if topo == nil { + fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render — config OK\n", configPath) + return } + fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n", + len(topo.RootComplexes), countDevices(topo)) } func countDevices(t *config.PCIeTopology) int { diff --git a/cmd/render-pci-sysfs/main_test.go b/cmd/render-pci-sysfs/main_test.go new file mode 100644 index 000000000..52636e204 --- /dev/null +++ b/cmd/render-pci-sysfs/main_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" +) + +const profileWithDevices = ` +devices: + - index: 0 + pci: + bus_id: "0000:07:00.0" +` + +// A profile whose devices declare no bus_id renders no topology. It is +// reachable through gpu.customConfig, and it used to return before Render. +const profileWithoutBusIDs = ` +devices: + - index: 0 +` + +// TestRun_ClearsTreeWhenProfileDeclaresNoDevices covers re-profiling a node +// onto a profile with nothing to render. The tree and the completion marker +// left by the previous profile would otherwise stay on disk, and both serving +// channels would keep mounting devices this profile does not declare — +// setup.sh gates on the marker, which said "rendered" about the old tree. +func TestRun_ClearsTreeWhenProfileDeclaresNoDevices(t *testing.T) { + out := t.TempDir() + require.NoError(t, run(options{ + configPath: writeProfile(t, profileWithDevices), + outputDir: out, + }), "render a profile with devices") + require.FileExists(t, filepath.Join(out, render.MarkerRelPath), "marker after the first render") + + require.NoError(t, run(options{ + configPath: writeProfile(t, profileWithoutBusIDs), + outputDir: out, + }), "render a profile without devices") + + entries, err := os.ReadDir(filepath.Join(out, render.PCIDevicesRelPath)) + require.NoError(t, err, "read devices dir") + require.Empty(t, entries, "the previous profile's devices are still served") + require.NoFileExists(t, filepath.Join(out, render.MarkerRelPath), + "the marker still claims a rendered tree") +} + +// TestRun_DryRunWritesNothing pins that --dry-run stays a validation pass on +// both paths, including the one that now prunes. +func TestRun_DryRunWritesNothing(t *testing.T) { + for name, profile := range map[string]string{ + "with devices": profileWithDevices, + "without devices": profileWithoutBusIDs, + } { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + require.NoError(t, run(options{ + configPath: writeProfile(t, profile), + outputDir: out, + dryRun: true, + }), "dry run") + entries, err := os.ReadDir(out) + require.NoError(t, err, "read output dir") + require.Empty(t, entries, "--dry-run wrote to the output directory") + }) + } +} + +func writeProfile(t *testing.T, contents string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(contents), 0o644), "write profile") + return path +} diff --git a/deployments/nvml-mock/scripts/setup.sh b/deployments/nvml-mock/scripts/setup.sh index b36890be7..8cf73112e 100644 --- a/deployments/nvml-mock/scripts/setup.sh +++ b/deployments/nvml-mock/scripts/setup.sh @@ -134,9 +134,10 @@ fi PCI_ROOT="$HOST" mkdir -p "$PCI_ROOT" # Keep in sync with render.MarkerRelPath (pkg/system/mockpcisysfs/render): the -# renderer writes it last, once the whole tree is on disk. Gating on the -# directories instead would accept a tree from a previous profile that this -# run had nothing to render over, and would say "rendered" partway through. +# renderer writes it last, once the whole tree is on disk, and removes it when +# the profile declares no PCI devices at all. Gating on the directories instead +# would say "rendered" partway through a render, and would keep serving the +# previous profile's devices to a profile that renders none of its own. PCI_SYSFS_MARKER=sys/.rendered PCI_SYSFS_RENDERED=off if [ -x /usr/local/bin/render-pci-sysfs ]; then From 047d6e83252bfc974e89bea0f09504091921ef24 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Thu, 20 Aug 2026 15:04:12 +0200 Subject: [PATCH 10/11] refactor(nvml-mock): take the sysfs tree paths from the renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NRI guard re-declared both halves of the tree and its tests staged them from matching literals, so renaming a path in the renderer would have left pciSysfsMounts statting something nothing writes — and it fails open, dropping the mounts with no test going red. MarkerRelPath was already exported for this seam; the two directories now are too. Also drops the Options.Output claim that Render is a no-op without a topology, which stopped being true when that path started pruning. Signed-off-by: Giulio Calzolari --- pkg/nri/nvmlmock/adjust.go | 11 +++----- pkg/nri/nvmlmock/adjust_test.go | 6 ++--- pkg/system/mockpcisysfs/render/render.go | 34 ++++++++++++++---------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index 96335ecfa..d65668847 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -44,11 +44,6 @@ const ( // MOCK_TOPOLOGY_CONFIG env). defaultTopologyRelPath = "topology/topology.yaml" - // pciDevicesRelPath and sysDevicesRelPath are the two halves of the fake - // PCI sysfs tree setup.sh renders into the overlay, resolved relative to - // the host overlay path. - pciDevicesRelPath = "sys/bus/pci/devices" - sysDevicesRelPath = "sys/devices" // pciDevicesContainerPath and sysDevicesContainerPath are the kernel // paths the tree must appear at inside the container. Unlike the // LD_PRELOAD-based redirection (MOCK_PCI_ROOT), these cannot be @@ -388,8 +383,10 @@ func pciSysfsMounts(cfg Config) []Mount { if _, err := os.Stat(filepath.Join(cfg.HostOverlayPath, render.MarkerRelPath)); err != nil { return nil } - sysDevices := filepath.Join(cfg.HostOverlayPath, sysDevicesRelPath) - pciDevices := filepath.Join(cfg.HostOverlayPath, pciDevicesRelPath) + // Paths come from the renderer that writes them: a guard statting a path + // nothing renders would fail open, dropping the mounts silently. + sysDevices := filepath.Join(cfg.HostOverlayPath, render.SysDevicesRelPath) + pciDevices := filepath.Join(cfg.HostOverlayPath, render.PCIDevicesRelPath) for _, dir := range []string{sysDevices, pciDevices} { info, err := os.Stat(dir) if err != nil || !info.IsDir() { diff --git a/pkg/nri/nvmlmock/adjust_test.go b/pkg/nri/nvmlmock/adjust_test.go index 3e0946171..a1b5ae502 100644 --- a/pkg/nri/nvmlmock/adjust_test.go +++ b/pkg/nri/nvmlmock/adjust_test.go @@ -108,7 +108,7 @@ func TestAdjustSkipsPCISysfsMountsWhenNotStaged(t *testing.T) { func TestAdjustSkipsPCIDevicesMountWithoutSysDevices(t *testing.T) { overlay := t.TempDir() stagePCISysfs(t, overlay) - require.NoError(t, os.RemoveAll(filepath.Join(overlay, "sys/devices"))) + require.NoError(t, os.RemoveAll(filepath.Join(overlay, render.SysDevicesRelPath))) cfg := DefaultConfig() cfg.HostOverlayPath = overlay @@ -153,8 +153,8 @@ func TestAdjustSkipsPCISysfsMountsWhileRenderIncomplete(t *testing.T) { // as the main DaemonSet's render-pci-sysfs run leaves it. func stagePCISysfs(t *testing.T, overlay string) { t.Helper() - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/bus/pci/devices"), 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(overlay, "sys/devices"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, render.PCIDevicesRelPath), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, render.SysDevicesRelPath), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(overlay, render.MarkerRelPath), nil, 0o644)) } diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 60e82e9ff..e39875d7f 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -52,10 +52,12 @@ type Options struct { // Output is the fake-root directory. The renderer writes under // /sys/... — Output itself is created if missing. // - // When Topology is nil or has no root complexes, Render is a no-op - // even if Output is empty (so setup.sh can invoke the renderer - // unconditionally). A non-nil Topology with a non-empty Output is - // required; otherwise Render returns an error. + // A non-nil Topology requires a non-empty Output; otherwise Render + // returns an error. When Topology is nil or has no root complexes there + // is nothing to write, and Render instead empties whatever a previous + // profile left under Output and drops the completion marker — so a caller + // can invoke the renderer unconditionally without leaving a tree that + // describes the wrong profile. With Output empty too, Render does nothing. Output string // DMISource is the directory holding the node's kernel DMI identity, @@ -77,9 +79,13 @@ type Options struct { // the tree is served to. const MarkerRelPath = "sys/.rendered" +// PCIDevicesRelPath and SysDevicesRelPath are the two halves of the tree, +// relative to Options.Output. They are exported for the consumers that +// bind-mount them onto the kernel paths, so the layout has one definition +// rather than a copy per consumer. const ( - pciDevicesRelPath = "sys/bus/pci/devices" - sysDevicesRelPath = "sys/devices" + PCIDevicesRelPath = "sys/bus/pci/devices" + SysDevicesRelPath = "sys/devices" ) // Render writes the entire tree, replacing whatever a previous render left @@ -133,10 +139,10 @@ func pruneTree(root string) error { if err := os.RemoveAll(filepath.Join(root, MarkerRelPath)); err != nil { return fmt.Errorf("clear %s: %w", MarkerRelPath, err) } - if err := removeEntries(filepath.Join(root, pciDevicesRelPath), ""); err != nil { + if err := removeEntries(filepath.Join(root, PCIDevicesRelPath), ""); err != nil { return err } - return removeEntries(filepath.Join(root, sysDevicesRelPath), dmiVirtualDirName) + return removeEntries(filepath.Join(root, SysDevicesRelPath), dmiVirtualDirName) } // removeEntries empties dir, keeping the entry named keep (if any). A missing @@ -162,10 +168,10 @@ func removeEntries(dir, keep string) error { func renderTopology(o Options) error { root := o.Output - if err := mkdirAll(root, pciDevicesRelPath); err != nil { + if err := mkdirAll(root, PCIDevicesRelPath); err != nil { return err } - if err := mkdirAll(root, sysDevicesRelPath); err != nil { + if err := mkdirAll(root, SysDevicesRelPath); err != nil { return err } @@ -183,7 +189,7 @@ func renderTopology(o Options) error { // pruneTree keeps so the mount targets inside it never go missing. const ( dmiVirtualDirName = "virtual" - dmiIDDir = sysDevicesRelPath + "/" + dmiVirtualDirName + "/dmi/id" + dmiIDDir = SysDevicesRelPath + "/" + dmiVirtualDirName + "/dmi/id" ) // dmiMirroredAttrs are the DMI attributes kind's mount-product-files.sh @@ -241,7 +247,7 @@ func renderDMI(root, source string) error { } func renderRootComplex(root string, rc config.RootComplex, ids map[string]config.PCI) error { - rcDir := filepath.Join(sysDevicesRelPath, rc.ID) + rcDir := filepath.Join(SysDevicesRelPath, rc.ID) if err := mkdirAll(root, rcDir); err != nil { return err } @@ -272,11 +278,11 @@ func renderRootComplex(root string, rc config.RootComplex, ids map[string]config // Relative target matches what the kernel emits, so any // readlink() consumer (`realpath`, deviceattribute, etc.) // resolves to the same canonical path it would on real Linux. - linkPath := filepath.Join(root, pciDevicesRelPath, bdfLC) + linkPath := filepath.Join(root, PCIDevicesRelPath, bdfLC) linkTarget := filepath.Join("..", "..", "..", "devices", rc.ID, bdfLC) if err := replaceSymlink(linkPath, linkTarget); err != nil { return fmt.Errorf("symlink %s -> %s: %w", - filepath.Join(pciDevicesRelPath, bdfLC), linkTarget, err) + filepath.Join(PCIDevicesRelPath, bdfLC), linkTarget, err) } } return nil From 9974d6f28989109c82bc153dad2f089b562f5566 Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Thu, 20 Aug 2026 15:04:12 +0200 Subject: [PATCH 11/11] test(e2e): wait for a GFD pod the exec can reach A terminating pod reports phase Running until its containers exit, so the phase filter alone still handed back the pod going away; the objectMeta these decode into now carries deletionTimestamp, which is what marks it. The call site polls for the same reason waitOperatorValidatorRunning does: the operator replaces its operands a reconcile after nvml-mock rolls, so resolving the pod once can land in the gap where none is ready. Signed-off-by: Giulio Calzolari --- tests/e2e/go/framework/kube/kube.go | 19 +++++++++++++------ tests/e2e/go/scenario_gpu_operator_test.go | 19 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/tests/e2e/go/framework/kube/kube.go b/tests/e2e/go/framework/kube/kube.go index b68e0fd3c..f263f7fd2 100644 --- a/tests/e2e/go/framework/kube/kube.go +++ b/tests/e2e/go/framework/kube/kube.go @@ -79,6 +79,10 @@ type objectMeta struct { Name string `json:"name"` Labels map[string]string `json:"labels"` Annotations map[string]string `json:"annotations"` + // DeletionTimestamp is set once the object is being deleted. For a pod + // that is the only marker of "Terminating": the phase stays Running + // until its containers exit. + DeletionTimestamp string `json:"deletionTimestamp"` } type nodeCondition struct { @@ -285,11 +289,14 @@ func (c *Client) RunningPodNames(ctx context.Context, ns, selector string) ([]st return out, nil } -// RunningPodOnNode returns the Running pod matching the selector on a given -// node. Callers that exec into a DaemonSet's pod need both filters: a -// Terminating or Pending pod matches the selector just as well and the exec -// fails, and a pod on another node answers about hardware the assertion is not -// about. +// RunningPodOnNode returns a Running pod matching the selector on a given +// node, skipping pods that are on their way out. Callers that exec into a +// DaemonSet's pod need all three filters: a Pending pod matches the selector +// as readily as a Running one, a terminating pod keeps reporting phase +// Running until its containers exit (deletionTimestamp is what marks it), and +// a pod on another node answers about hardware the assertion is not about. +// +// A rollout can still leave no candidate at all, so callers poll. func (c *Client) RunningPodOnNode(ctx context.Context, ns, selector, node string) (string, error) { var pl podList if err := c.getJSON(ctx, &pl, "pods", "-n", ns, "-l", selector, @@ -297,7 +304,7 @@ func (c *Client) RunningPodOnNode(ctx context.Context, ns, selector, node string return "", err } for _, p := range pl.Items { - if p.Status.Phase == "Running" { + if p.Status.Phase == "Running" && p.Metadata.DeletionTimestamp == "" { return p.Metadata.Name, nil } } diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index 0cc873f6f..7dfdd5c63 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -80,12 +80,19 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( // the mock tree" from "GFD read the host's sysfs and happened // to agree". Reading the tree from inside the container pins // the delivery itself, independent of what GFD makes of it. - // The GFD pod on `node`, and only while it is Running: the - // specs above assert about that node's labels, and a - // Terminating or Pending pod matches the selector too, which - // the exec below would fail on. - pod, err := h.Kube.RunningPodOnNode(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery", node) - Expect(err).NotTo(HaveOccurred()) + // The GFD pod on `node`, and only while it is Running and not + // terminating: the specs above assert about that node's + // labels, and a pod the exec cannot reach fails the spec for a + // reason it is not about. Polled because the operator replaces + // its operands a reconcile after nvml-mock rolls (#602), the + // same reason waitOperatorValidatorRunning polls. + var pod string + Eventually(func() (string, error) { + p, err := h.Kube.RunningPodOnNode(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery", node) + pod = p + return p, err + }).WithContext(ctx).WithTimeout(config.ReadyTimeout()).WithPolling(config.PollInterval()). + ShouldNot(BeEmpty(), "no Running gpu-feature-discovery pod on %s", node) assertions.PCISysfsAtKernelPath(ctx, h.Kube, kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}, p.ExpectedGPUs())