diff --git a/.shellcheckrc b/.shellcheckrc index 709c50eed..006348446 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -1,9 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + # SC2059: Don't use variables in the printf format string. # Our info/warn/ok/error helpers and banner intentionally embed ANSI color # variables in printf format strings. This is safe because the variables # contain only escape sequences, never user input. # # SC1091: Not following sourced file. -# We dynamically source nvm.sh and runtime.sh at paths that shellcheck -# cannot resolve statically. +# Kept as a fallback for files that source external dependencies not in +# the repo (e.g. nvm.sh). For repo-local sources (runtime.sh), +# external-sources + source-path below let shellcheck follow them. disable=SC2059,SC1091 + +# Follow source directives so shellcheck can analyze sourced functions. +external-sources=true + +# Resolve source-path hints relative to the script's own directory, +# matching the runtime behavior of `SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"`. +source-path=SCRIPTDIR diff --git a/README.md b/README.md index 38bdb4c23..76b56656f 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,8 @@ The sandbox image is approximately 2.4 GB compressed. During image push, the Doc | Platform | Supported runtimes | Notes | |----------|--------------------|-------| -| Linux | Docker | Primary supported path today | -| macOS (Apple Silicon) | Colima, Docker Desktop | Recommended runtimes for supported macOS setups | -| macOS | Podman | Not supported yet. NemoClaw currently depends on OpenShell support for Podman on macOS. | +| Linux | Docker, Podman | Primary supported path today | +| macOS (Apple Silicon) | Colima, Docker Desktop, Podman | Supported container runtimes on macOS | | Windows WSL | Docker Desktop (WSL backend) | Supported target path | > **💡 Tip** diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js index 63d34a253..4bd5c2866 100644 --- a/bin/lib/onboard.js +++ b/bin/lib/onboard.js @@ -27,7 +27,6 @@ const { } = require("./inference-config"); const { inferContainerRuntime, - isUnsupportedMacosRuntime, shouldPatchCoredns, } = require("./platform"); const { resolveOpenshell } = require("./resolve-openshell"); @@ -1177,12 +1176,6 @@ async function preflight() { console.log(" ✓ Docker is running"); const runtime = getContainerRuntime(); - if (isUnsupportedMacosRuntime(runtime)) { - console.error(" Podman on macOS is not supported by NemoClaw at this time."); - console.error(" OpenShell currently depends on Docker host-gateway behavior that Podman on macOS does not provide."); - console.error(" Use Colima or Docker Desktop on macOS instead."); - process.exit(1); - } if (runtime !== "unknown") { console.log(` ✓ Container runtime: ${runtime}`); } @@ -1307,10 +1300,9 @@ async function startGateway(gpu) { sleep(2); } - // CoreDNS fix — always run. k3s-inside-Docker has broken DNS on all platforms. const runtime = getContainerRuntime(); if (shouldPatchCoredns(runtime)) { - console.log(" Patching CoreDNS for Colima..."); + console.log(` Patching CoreDNS for ${runtime}...`); run(`bash "${path.join(SCRIPTS, "fix-coredns.sh")}" ${GATEWAY_NAME} 2>&1 || true`, { ignoreError: true }); } // Give DNS a moment to propagate diff --git a/bin/lib/platform.js b/bin/lib/platform.js index 67c31a3f3..f57155d50 100644 --- a/bin/lib/platform.js +++ b/bin/lib/platform.js @@ -30,13 +30,8 @@ function inferContainerRuntime(info = "") { return "unknown"; } -function isUnsupportedMacosRuntime(runtime, opts = {}) { - const platform = opts.platform ?? process.platform; - return platform === "darwin" && runtime === "podman"; -} - function shouldPatchCoredns(runtime) { - return runtime === "colima"; + return runtime === "colima" || runtime === "podman"; } function getColimaDockerSocketCandidates(opts = {}) { @@ -52,6 +47,23 @@ function findColimaDockerSocket(opts = {}) { return getColimaDockerSocketCandidates(opts).find((socketPath) => existsSync(socketPath)) ?? null; } +function getPodmanSocketCandidates(opts = {}) { + const home = opts.home ?? process.env.HOME ?? "/tmp"; + const platform = opts.platform ?? process.platform; + const uid = opts.uid ?? process.getuid?.() ?? 1000; + + if (platform === "darwin") { + return [ + path.join(home, ".local/share/containers/podman/machine/podman.sock"), + ]; + } + + return [ + `/run/user/${uid}/podman/podman.sock`, + "/run/podman/podman.sock", + ]; +} + function getDockerSocketCandidates(opts = {}) { const home = opts.home ?? process.env.HOME ?? "/tmp"; const platform = opts.platform ?? process.platform; @@ -59,10 +71,19 @@ function getDockerSocketCandidates(opts = {}) { if (platform === "darwin") { return [ ...getColimaDockerSocketCandidates({ home }), + ...getPodmanSocketCandidates({ home, platform }), path.join(home, ".docker/run/docker.sock"), ]; } + if (platform === "linux") { + return [ + ...getPodmanSocketCandidates({ home, platform, uid: opts.uid }), + "/run/docker.sock", + "/var/run/docker.sock", + ]; + } + return []; } @@ -95,8 +116,8 @@ module.exports = { findColimaDockerSocket, getColimaDockerSocketCandidates, getDockerSocketCandidates, + getPodmanSocketCandidates, inferContainerRuntime, - isUnsupportedMacosRuntime, isWsl, shouldPatchCoredns, }; diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index aa654184a..9711984cd 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -189,3 +189,56 @@ $ nemoclaw logs ``` Use `--follow` to stream logs in real time while debugging. + +## Podman + +### `open /dev/kmsg: operation not permitted` + +This error appears when the Podman machine is running in rootless mode. +K3s kubelet requires `/dev/kmsg` access for its OOM watcher, which is not available in rootless containers. + +Switch the Podman machine to rootful mode and restart: + +```console +$ podman machine stop +$ podman machine set --rootful +$ podman machine start +``` + +Then destroy and recreate the gateway: + +```console +$ openshell gateway destroy --name nemoclaw +$ nemoclaw onboard +``` + +### Image push timeout with Podman + +When creating a sandbox, the 1.5 GB sandbox image push into K3s may time out through Podman's API socket. +This is a known limitation of the bollard Docker client's default timeout. + +Manually push the image using the Docker CLI, which has no such timeout: + +```console +$ docker images --format '{{.Repository}}:{{.Tag}}' | grep sandbox-from +$ docker save | \ + docker exec -i openshell-cluster-nemoclaw \ + ctr -a /run/k3s/containerd/containerd.sock -n k8s.io images import - +``` + +After the import completes, create the sandbox manually: + +```console +$ openshell sandbox create --name my-assistant --from +``` + +### Podman machine resources + +The default Podman machine has 2 GB RAM, which is insufficient for the sandbox image push and K3s cluster overhead. +Allocate at least 8 GB RAM and 4 CPUs: + +```console +$ podman machine stop +$ podman machine set --cpus 6 --memory 8192 +$ podman machine start +``` diff --git a/scripts/fix-coredns.sh b/scripts/fix-coredns.sh index 9b587ab33..c1f9f3a81 100755 --- a/scripts/fix-coredns.sh +++ b/scripts/fix-coredns.sh @@ -2,17 +2,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Fix CoreDNS on local OpenShell gateways running under Colima. +# Fix CoreDNS on local OpenShell gateways running under Colima or Podman. # # Problem: k3s CoreDNS forwards to /etc/resolv.conf which inside the -# CoreDNS pod resolves to 127.0.0.11 (Docker's embedded DNS). That -# address is NOT reachable from k3s pods, causing DNS to fail and +# CoreDNS pod resolves to 127.0.0.11 (Docker/Podman's embedded DNS). +# That address is NOT reachable from k3s pods, causing DNS to fail and # CoreDNS to CrashLoop. # # Fix: forward CoreDNS to the container's default gateway IP, which -# is reachable from pods and routes DNS through Docker to the host. +# is reachable from pods and routes DNS through Docker/Podman to the host. # -# Run this after `openshell gateway start` on Colima setups. +# Run this after `openshell gateway start` on Colima or Podman setups. # # Usage: ./scripts/fix-coredns.sh [gateway-name] @@ -23,15 +23,25 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # shellcheck source=./lib/runtime.sh . "$SCRIPT_DIR/lib/runtime.sh" -COLIMA_SOCKET="$(find_colima_docker_socket || true)" +DETECTED_RUNTIME="unknown" if [ -z "${DOCKER_HOST:-}" ]; then + COLIMA_SOCKET="$(find_colima_docker_socket || true)" if [ -n "$COLIMA_SOCKET" ]; then export DOCKER_HOST="unix://$COLIMA_SOCKET" + DETECTED_RUNTIME="colima" else - echo "Skipping CoreDNS patch: Colima socket not found." - exit 0 + PODMAN_SOCKET="$(find_podman_socket || true)" + if [ -n "$PODMAN_SOCKET" ]; then + export DOCKER_HOST="unix://$PODMAN_SOCKET" + DETECTED_RUNTIME="podman" + else + echo "Skipping CoreDNS patch: no Colima or Podman socket found." + exit 0 + fi fi +else + DETECTED_RUNTIME="$(docker_host_runtime "$DOCKER_HOST" || echo "custom")" fi # Find the cluster container @@ -48,10 +58,10 @@ fi CONTAINER_RESOLV_CONF="$(docker exec "$CLUSTER" cat /etc/resolv.conf 2>/dev/null || true)" HOST_RESOLV_CONF="$(cat /etc/resolv.conf 2>/dev/null || true)" -UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "colima" || true)" +UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "$DETECTED_RUNTIME" || true)" if [ -z "$UPSTREAM_DNS" ]; then - echo "ERROR: Could not determine a non-loopback DNS upstream for Colima." + echo "ERROR: Could not determine a non-loopback DNS upstream for $DETECTED_RUNTIME." exit 1 fi diff --git a/scripts/lib/runtime.sh b/scripts/lib/runtime.sh index a6bba65f2..bc0110119 100755 --- a/scripts/lib/runtime.sh +++ b/scripts/lib/runtime.sh @@ -9,6 +9,7 @@ socket_exists() { case ":$NEMOCLAW_TEST_SOCKET_PATHS:" in *":$socket_path:"*) return 0 ;; esac + return 1 fi [ -S "$socket_path" ] @@ -56,6 +57,11 @@ detect_docker_host() { return 0 fi + if socket_path="$(find_podman_socket "$home_dir")"; then + printf 'unix://%s\n' "$socket_path" + return 0 + fi + if socket_path="$(find_docker_desktop_socket "$home_dir")"; then printf 'unix://%s\n' "$socket_path" return 0 @@ -71,6 +77,9 @@ docker_host_runtime() { unix://*"/.colima/default/docker.sock" | unix://*"/.config/colima/default/docker.sock") printf 'colima\n' ;; + unix://*"/podman/machine/podman.sock" | unix://*"/podman/podman.sock") + printf 'podman\n' + ;; unix://*"/.docker/run/docker.sock") printf 'docker-desktop\n' ;; @@ -103,11 +112,30 @@ infer_container_runtime_from_info() { fi } -is_unsupported_macos_runtime() { - local platform="${1:-$(uname -s)}" - local runtime="${2:-unknown}" +find_podman_socket() { + local home_dir="${1:-${HOME:-/tmp}}" + local socket_path + + if [ "$(uname -s)" = "Darwin" ]; then + socket_path="$home_dir/.local/share/containers/podman/machine/podman.sock" + if socket_exists "$socket_path"; then + printf '%s\n' "$socket_path" + return 0 + fi + else + local uid + uid="$(id -u 2>/dev/null || echo 1000)" + for socket_path in \ + "/run/user/$uid/podman/podman.sock" \ + "/run/podman/podman.sock"; do + if socket_exists "$socket_path"; then + printf '%s\n' "$socket_path" + return 0 + fi + done + fi - [ "$platform" = "Darwin" ] && [ "$runtime" = "podman" ] + return 1 } is_loopback_ip() { diff --git a/scripts/setup.sh b/scripts/setup.sh index 6aeb68085..22b66e068 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -86,9 +86,6 @@ command -v docker >/dev/null || fail "docker not found" [ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY not set. Get one from build.nvidia.com" CONTAINER_RUNTIME="$(infer_container_runtime_from_info "$(docker info 2>/dev/null || true)")" -if is_unsupported_macos_runtime "$(uname -s)" "$CONTAINER_RUNTIME"; then - fail "Podman on macOS is not supported yet. NemoClaw currently depends on OpenShell support for Podman on macOS. Use Colima or Docker Desktop instead." -fi if [ "$CONTAINER_RUNTIME" != "unknown" ]; then info "Container runtime: $CONTAINER_RUNTIME" fi @@ -123,9 +120,9 @@ for i in 1 2 3 4 5; do done info "Gateway is healthy" -# 2. CoreDNS fix (Colima only) -if [ "$CONTAINER_RUNTIME" = "colima" ]; then - info "Patching CoreDNS for Colima..." +# 2. CoreDNS fix (Colima and Podman — nested K3s DNS is broken with these runtimes) +if [ "$CONTAINER_RUNTIME" = "colima" ] || [ "$CONTAINER_RUNTIME" = "podman" ]; then + info "Patching CoreDNS for $CONTAINER_RUNTIME..." bash "$SCRIPT_DIR/fix-coredns.sh" nemoclaw 2>&1 || warn "CoreDNS patch failed (may not be needed)" fi diff --git a/scripts/smoke-macos-install.sh b/scripts/smoke-macos-install.sh index 9dfd3d0ac..7119eced2 100755 --- a/scripts/smoke-macos-install.sh +++ b/scripts/smoke-macos-install.sh @@ -55,7 +55,7 @@ Usage: ./scripts/smoke-macos-install.sh [options] Options: --sandbox-name Sandbox name to feed into install.sh --log-dir Directory for install/uninstall logs - --runtime Select runtime: colima or docker-desktop + --runtime Select runtime: colima, podman, or docker-desktop --allow-existing-state Allow running even if NemoClaw/OpenShell state already exists --keep-logs Preserve log files after success --remove-openshell Allow uninstall.sh to remove openshell @@ -132,6 +132,13 @@ select_runtime() { export DOCKER_HOST="unix://$socket_path" info "Using runtime 'colima' via $socket_path" ;; + podman) + local socket_path + socket_path="$(find_podman_socket || true)" + [ -n "$socket_path" ] || fail "Requested runtime 'podman', but no Podman socket was found." + export DOCKER_HOST="unix://$socket_path" + info "Using runtime 'podman' via $socket_path" + ;; docker-desktop) local socket_path socket_path="$(find_docker_desktop_socket || true)" @@ -140,7 +147,7 @@ select_runtime() { info "Using runtime 'docker-desktop' via $socket_path" ;; *) - fail "Unsupported runtime '$RUNTIME'. Use 'colima' or 'docker-desktop'." + fail "Unsupported runtime '$RUNTIME'. Use 'colima', 'podman', or 'docker-desktop'." ;; esac } diff --git a/test/platform.test.js b/test/platform.test.js index 0eb85277e..dda7f0159 100644 --- a/test/platform.test.js +++ b/test/platform.test.js @@ -8,8 +8,8 @@ import { detectDockerHost, findColimaDockerSocket, getDockerSocketCandidates, + getPodmanSocketCandidates, inferContainerRuntime, - isUnsupportedMacosRuntime, isWsl, shouldPatchCoredns, } from "../bin/lib/platform"; @@ -33,18 +33,40 @@ describe("platform helpers", () => { }); }); + describe("getPodmanSocketCandidates", () => { + it("returns macOS Podman socket paths", () => { + const home = "/tmp/test-home"; + expect(getPodmanSocketCandidates({ platform: "darwin", home })).toEqual([ + path.join(home, ".local/share/containers/podman/machine/podman.sock"), + ]); + }); + + it("returns Linux Podman socket paths with uid", () => { + expect(getPodmanSocketCandidates({ platform: "linux", home: "/tmp/test-home", uid: 1001 })).toEqual([ + "/run/user/1001/podman/podman.sock", + "/run/podman/podman.sock", + ]); + }); + }); + describe("getDockerSocketCandidates", () => { - it("returns macOS candidates in priority order", () => { + it("returns macOS candidates in priority order (Colima > Podman > Docker Desktop)", () => { const home = "/tmp/test-home"; expect(getDockerSocketCandidates({ platform: "darwin", home })).toEqual([ path.join(home, ".colima/default/docker.sock"), path.join(home, ".config/colima/default/docker.sock"), + path.join(home, ".local/share/containers/podman/machine/podman.sock"), path.join(home, ".docker/run/docker.sock"), ]); }); - it("does not auto-detect sockets on Linux", () => { - expect(getDockerSocketCandidates({ platform: "linux", home: "/tmp/test-home" })).toEqual([]); + it("returns Linux candidates (Podman > native Docker)", () => { + expect(getDockerSocketCandidates({ platform: "linux", home: "/tmp/test-home", uid: 1000 })).toEqual([ + "/run/user/1000/podman/podman.sock", + "/run/podman/podman.sock", + "/run/docker.sock", + "/var/run/docker.sock", + ]); }); }); @@ -123,21 +145,40 @@ describe("platform helpers", () => { }); }); - describe("isUnsupportedMacosRuntime", () => { - it("flags podman on macOS", () => { - expect(isUnsupportedMacosRuntime("podman", { platform: "darwin" })).toBe(true); - }); - - it("does not flag podman on Linux", () => { - expect(isUnsupportedMacosRuntime("podman", { platform: "linux" })).toBe(false); - }); - }); - describe("shouldPatchCoredns", () => { - it("patches CoreDNS for Colima only", () => { + it("patches CoreDNS for Colima and Podman", () => { expect(shouldPatchCoredns("colima")).toBe(true); + expect(shouldPatchCoredns("podman")).toBe(true); expect(shouldPatchCoredns("docker-desktop")).toBe(false); expect(shouldPatchCoredns("docker")).toBe(false); }); }); + + describe("detectDockerHost with Podman", () => { + it("detects Podman socket on macOS when Colima is absent", () => { + const home = "/tmp/test-home"; + const podmanSocket = path.join(home, ".local/share/containers/podman/machine/podman.sock"); + const existsSync = (candidate) => candidate === podmanSocket; + + expect(detectDockerHost({ env: {}, platform: "darwin", home, existsSync })).toEqual({ + dockerHost: `unix://${podmanSocket}`, + source: "socket", + socketPath: podmanSocket, + }); + }); + + it("prefers Colima over Podman on macOS", () => { + const home = "/tmp/test-home"; + const colimaSocket = path.join(home, ".colima/default/docker.sock"); + const podmanSocket = path.join(home, ".local/share/containers/podman/machine/podman.sock"); + const sockets = new Set([colimaSocket, podmanSocket]); + const existsSync = (candidate) => sockets.has(candidate); + + expect(detectDockerHost({ env: {}, platform: "darwin", home, existsSync })).toEqual({ + dockerHost: `unix://${colimaSocket}`, + source: "socket", + socketPath: colimaSocket, + }); + }); + }); }); diff --git a/test/runtime-shell.test.js b/test/runtime-shell.test.js index 2e5702f22..799d4d5df 100644 --- a/test/runtime-shell.test.js +++ b/test/runtime-shell.test.js @@ -35,6 +35,7 @@ describe("shell runtime helpers", () => { const result = runShell(`source "${RUNTIME_SH}"; detect_docker_host`, { HOME: home, + DOCKER_HOST: "", NEMOCLAW_TEST_SOCKET_PATHS: `${colimaSocket}:${dockerDesktopSocket}`, }); @@ -49,6 +50,7 @@ describe("shell runtime helpers", () => { const result = runShell(`source "${RUNTIME_SH}"; detect_docker_host`, { HOME: home, + DOCKER_HOST: "", NEMOCLAW_TEST_SOCKET_PATHS: dockerDesktopSocket, }); @@ -103,14 +105,33 @@ describe("shell runtime helpers", () => { expect(result.stdout.trim()).toBe("podman"); }); - it("flags podman on macOS as unsupported", () => { - const result = runShell(`source "${RUNTIME_SH}"; is_unsupported_macos_runtime Darwin podman`); + it("detects Podman socket on macOS", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-shell-")); + const podmanSocket = path.join(home, ".local/share/containers/podman/machine/podman.sock"); + + const result = runShell( + `uname() { printf 'Darwin\\n'; }; source "${RUNTIME_SH}"; find_podman_socket`, + { + HOME: home, + NEMOCLAW_TEST_SOCKET_PATHS: podmanSocket, + }, + ); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(podmanSocket); + fs.rmSync(home, { recursive: true, force: true }); }); - it("does not flag podman on Linux", () => { - const result = runShell(`source "${RUNTIME_SH}"; is_unsupported_macos_runtime Linux podman`); - expect(result.status).not.toBe(0); + it("classifies a Podman DOCKER_HOST correctly", () => { + const result = runShell(`source "${RUNTIME_SH}"; docker_host_runtime "unix:///run/user/1000/podman/podman.sock"`); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe("podman"); + }); + + it("classifies a Podman machine socket correctly", () => { + const result = runShell(`source "${RUNTIME_SH}"; docker_host_runtime "unix:///Users/test/.local/share/containers/podman/machine/podman.sock"`); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe("podman"); }); it("returns the vllm-local base URL", () => { diff --git a/test/smoke-macos-install.test.js b/test/smoke-macos-install.test.js index 0ba7c5534..1fcd7a49f 100644 --- a/test/smoke-macos-install.test.js +++ b/test/smoke-macos-install.test.js @@ -41,14 +41,29 @@ describe("macOS smoke install script guardrails", () => { }); it("rejects unsupported runtimes", () => { - const result = spawnSync("bash", [SMOKE_SCRIPT, "--runtime", "podman"], { + const result = spawnSync("bash", [SMOKE_SCRIPT, "--runtime", "lxc"], { cwd: path.join(import.meta.dirname, ".."), encoding: "utf-8", env: { ...process.env, NVIDIA_API_KEY: "nvapi-test" }, }); expect(result.status).not.toBe(0); - expect(`${result.stdout}${result.stderr}`).toMatch(/Unsupported runtime 'podman'/); + expect(`${result.stdout}${result.stderr}`).toMatch(/Unsupported runtime 'lxc'/); + }); + + it("accepts podman as a runtime option", () => { + const result = spawnSync("bash", [SMOKE_SCRIPT, "--runtime", "podman"], { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + env: { + ...process.env, + NVIDIA_API_KEY: "nvapi-test", + HOME: "/tmp/nemoclaw-smoke-no-runtime", + }, + }); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch(/no Podman socket was found/); }); it("fails when a requested runtime socket is unavailable", () => {