fix(docker): share CUDA compatibility libraries with exec processes

Put a private per-container compatibility directory on the image's library
search path. The non-root bootstrap atomically selects the opt-in CUDA
libraries there, so the entrypoint, docker exec, and restarted processes use
the same loader configuration. Leave the directory inactive by default and
report missing compatibility libraries before starting the requested command.

Document recreation after changing the setting. The real regression checks
enabled and disabled modes, arbitrary UID/GID, exec, restart, and missing
libraries; it fails against the previous image at the exec comparison.

Validation: the rebuilt CUDA image passes make docker-check and
make docker-integration-check (1972 CPU tests and 11 integration tests),
including the separate digest bundle fix and offline Compose creation.

Author: KraHsu

assets/openwam_usage_docs/docker.md

@@ -645,7 +645,10 @@ older 535/550 data-center drivers need validation with the image's CUDA 12.8
 libraries and `torch.compile`; successful `nvidia-smi` alone is not sufficient.
 NVIDIA Container Toolkit normally handles CUDA forward compatibility. If it
 does not, test `OPENWAM_CUDA_COMPAT=1` to select `/usr/local/cuda/compat`, then run
-the GPU check in this section. This does not install or replace the host kernel driver.
+the GPU check in this section. The normal image entrypoint initializes this
+selection for both the main process and later `docker exec` commands. Recreate
+containers after changing the setting. The default `0` leaves library selection
+to the Toolkit. This does not install or replace the host kernel driver.
 See [NVIDIA's compatibility guide](https://docs.nvidia.com/deploy/cuda-compatibility/forward-compatibility.html).
 
 </details>

docker/Dockerfile

@@ -65,6 +65,7 @@ ENV XDG_CACHE_HOME=/cache \
     LD_PRELOAD=libnss_wrapper.so \
     NSS_WRAPPER_PASSWD=/tmp/openwam-nss/passwd \
     NSS_WRAPPER_GROUP=/tmp/openwam-nss/group \
+    LD_LIBRARY_PATH="/tmp/openwam-runtime/cuda-compat:${LD_LIBRARY_PATH}" \
     RUFF_CACHE_DIR=/cache/ruff \
     CUDA_CACHE_PATH=/cache/cuda \
     HF_HOME=/cache/huggingface \

docker/entrypoint.sh

@@ -12,17 +12,6 @@ for cache_dir in "${XDG_CACHE_HOME:-/cache}" "${RUFF_CACHE_DIR:-/cache/ruff}" \
     fi
 done
 
-# Usually handled by NVIDIA Container Toolkit. Opt in only when validating an
-# older data-center driver that needs the image's forward-compatibility libraries.
-if [[ "${OPENWAM_CUDA_COMPAT:-0}" == 1 ]]; then
-    compat_dir="${CUDA_HOME:-/usr/local/cuda}/compat"
-    if [[ ! -e "$compat_dir/libcuda.so.1" ]]; then
-        echo "OpenWAM: CUDA compatibility libraries missing from $compat_dir." >&2
-        exit 1
-    fi
-    export LD_LIBRARY_PATH="$compat_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
-fi
-
 case "${1:-serve}" in
     serve)
         if (( $# )); then shift; fi

docker/runtime_user.py

@@ -6,6 +6,27 @@
 from pathlib import Path
 
 
+def configure_cuda_compat():
+    # This directory is on the image's LD_LIBRARY_PATH for every process,
+    # including docker exec. Only the opt-in entrypoint creates the link.
+    link = Path("/tmp/openwam-runtime/cuda-compat")
+    try:
+        if os.environ.get("OPENWAM_CUDA_COMPAT", "0") != "1":
+            link.unlink(missing_ok=True)
+            return
+        compat = (Path(os.environ.get("CUDA_HOME", "/usr/local/cuda")) / "compat").resolve()
+        if not (compat / "libcuda.so.1").is_file():
+            raise SystemExit(f"OpenWAM: CUDA compatibility libraries missing from {compat}.")
+        link.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+        # Atomic replacement also works when Docker restarts this container.
+        with tempfile.TemporaryDirectory(prefix=".cuda-compat-", dir=link.parent) as temporary:
+            staged = Path(temporary) / "compat"
+            staged.symlink_to(compat, target_is_directory=True)
+            staged.replace(link)
+    except OSError as error:
+        raise SystemExit(f"OpenWAM: cannot configure CUDA compatibility: {error}") from error
+
+
 def main():
     home = Path(os.environ["HOME"])
     try:
@@ -34,6 +55,8 @@ def main():
         temporary.write_text("\n".join([*lines, entry]) + "\n")
         temporary.replace(path)
 
+    configure_cuda_compat()
+
     # Start bash only after the account files exist: bash itself looks up the
     # current UID at startup. exec preserves signal handling and the exit code.
     os.execv("/bin/bash", ["bash", "/opt/openwam/docker/entrypoint.sh", *sys.argv[1:]])

tests/test_docker_integration.py

@@ -662,6 +662,85 @@ def test_runtime_identity_and_home_survive_run_and_exec(self):
                     ]
                 )
 
+    def test_cuda_compat_library_selection_survives_exec_and_restart(self):
+        probe = (
+            "import ctypes, json, os\n"
+            "from pathlib import Path\n"
+            "result = {'path': os.environ.get('LD_LIBRARY_PATH'), 'compat': os.environ.get('OPENWAM_CUDA_COMPAT')}\n"
+            "result['compat_directory'] = str((Path(os.environ['CUDA_HOME']) / 'compat').resolve())\n"
+            "try:\n"
+            "    library = ctypes.CDLL('libcuda.so.1')\n"
+            "    result['libraries'] = sorted({line.split()[-1] for line in Path('/proc/self/maps').read_text().splitlines() "
+            "if '/libcuda.so.' in line})\n"
+            "except OSError as error:\n"
+            "    result['error'] = str(error)\n"
+            "print(json.dumps(result), flush=True)\n"
+        )
+        for enabled in ("0", "1"):
+            with self.subTest(enabled=enabled):
+                options = ["--network", "none", "--user", "23456:23457", "-e", "OPENWAM_CUDA_COMPAT=" + enabled]
+                initial = json.loads(self.run_command([*DOCKER, "run", "--rm", *options, IMAGE, "python", "-c", probe]))
+                if enabled == "1":
+                    self.assertNotIn("error", initial)
+                    self.assertTrue(initial["libraries"])
+                    self.assertTrue(
+                        all(path.startswith(initial["compat_directory"] + "/") for path in initial["libraries"])
+                    )
+                name = "openwam-compat-" + uuid.uuid4().hex[:12]
+                self.run_command(
+                    [
+                        *DOCKER,
+                        "run",
+                        "-d",
+                        "--name",
+                        name,
+                        *options,
+                        IMAGE,
+                        "python",
+                        "-c",
+                        probe + "import time; time.sleep(300)\n",
+                    ]
+                )
+                self.addCleanup(self.run_command, [*DOCKER, "rm", "-f", name])
+
+                def wait_for_start(count):
+                    for _ in range(100):
+                        lines = self.run_command([*DOCKER, "logs", name]).splitlines()
+                        if len(lines) >= count:
+                            self.assertEqual(json.loads(lines[-1]), initial)
+                            return
+                        time.sleep(0.1)
+                    self.fail("compatibility probe did not start")
+
+                wait_for_start(1)
+                self.assertEqual(json.loads(self.run_command([*DOCKER, "exec", name, "python", "-c", probe])), initial)
+                self.run_command([*DOCKER, "restart", "--time", "1", name])
+                wait_for_start(2)
+                self.assertEqual(json.loads(self.run_command([*DOCKER, "exec", name, "python", "-c", probe])), initial)
+
+        missing = subprocess.run(
+            [
+                *DOCKER,
+                "run",
+                "--rm",
+                "--network",
+                "none",
+                "-e",
+                "OPENWAM_CUDA_COMPAT=1",
+                "-e",
+                "CUDA_HOME=/missing-cuda",
+                IMAGE,
+                "python",
+                "-c",
+                "print('must not start')",
+            ],
+            capture_output=True,
+            text=True,
+        )
+        self.assertNotEqual(missing.returncode, 0)
+        self.assertIn("CUDA compatibility libraries missing", missing.stderr)
+        self.assertNotIn("must not start", missing.stdout)
+
     def test_bootstrap_pip_fallback_installs_the_locked_version(self):
         # Local PEP 503 indexes and real wheels make this deterministic/offline:
         # the primary lacks the package; the fallback has both 1.0 and 2.0.