fix(sandbox): defer symlink resolution until container filesystem is ready

The one-shot resolve ran immediately after ProcessHandle::spawn, before
the child's mount namespace and /proc/<pid>/root/ were populated. This
caused symlink_metadata to fail with ENOENT on every binary, and the
poll loop never retried because it only reloads when the policy hash
changes on the server.

Replace the synchronous resolve with an async task that probes
/proc/<pid>/root/ with retries (10 attempts, 500ms apart, 5s total).
The child's mount namespace is typically ready within a few hundred ms.

Also inline error values into warning message strings so they appear in
default log output (not just as structured tracing fields that may be
elided), and add debug-level logs before each symlink_metadata call to
aid diagnosis.

Author: johntmyers

crates/openshell-sandbox/src/lib.rs

@@ -714,30 +714,64 @@ pub async fn run_sandbox(
             .build()
     );
 
-    // Resolve policy binary symlinks now that the container filesystem is
-    // accessible via /proc/<pid>/root/. This expands symlinks like
-    // /usr/bin/python3 → /usr/bin/python3.11 in the OPA policy data so that
-    // either path matches at evaluation time.
+    // Spawn a task to resolve policy binary symlinks after the container
+    // filesystem becomes accessible via /proc/<pid>/root/. This expands
+    // symlinks like /usr/bin/python3 → /usr/bin/python3.11 in the OPA
+    // policy data so that either path matches at evaluation time.
     //
-    // If /proc/<pid>/root/ is inaccessible (restricted ptrace, rootless
-    // container, etc.), resolve_binary_in_container logs a warning per binary
-    // and falls back to literal path matching. The reload itself still
-    // succeeds — only the symlink expansion is skipped.
+    // We cannot do this synchronously here because the child process has
+    // just been spawned and its mount namespace / procfs entries may not
+    // be fully populated yet. Instead, we probe with retries until
+    // /proc/<pid>/root/ is accessible or we exhaust attempts.
     if let (Some(engine), Some(proto)) = (&opa_engine, &retained_proto) {
-        let pid = handle.pid();
-        if let Err(e) = engine.reload_from_proto_with_pid(proto, pid) {
+        let resolve_engine = engine.clone();
+        let resolve_proto = proto.clone();
+        let resolve_pid = entrypoint_pid.clone();
+        tokio::spawn(async move {
+            let pid = resolve_pid.load(Ordering::Acquire);
+            let probe_path = format!("/proc/{pid}/root/");
+            // Retry up to 10 times with 500ms intervals (5s total).
+            // The child's mount namespace is typically ready within a
+            // few hundred ms of spawn.
+            for attempt in 1..=10 {
+                tokio::time::sleep(Duration::from_millis(500)).await;
+                if std::fs::metadata(&probe_path).is_ok() {
+                    info!(
+                        pid = pid,
+                        attempt = attempt,
+                        "Container filesystem accessible, resolving policy binary symlinks"
+                    );
+                    match resolve_engine.reload_from_proto_with_pid(&resolve_proto, pid) {
+                        Ok(()) => {
+                            info!(
+                                pid = pid,
+                                "Policy binary symlink resolution complete \
+                                 (check logs above for per-binary results)"
+                            );
+                        }
+                        Err(e) => {
+                            warn!(
+                                "Failed to rebuild OPA engine with symlink resolution \
+                                 (non-fatal, falling back to literal path matching): {e}"
+                            );
+                        }
+                    }
+                    return;
+                }
+                debug!(
+                    pid = pid,
+                    attempt = attempt,
+                    probe_path = %probe_path,
+                    "Container filesystem not yet accessible, retrying symlink resolution"
+                );
+            }
             warn!(
-                error = %e,
-                "Failed to rebuild OPA engine with symlink resolution (non-fatal, \
-                 falling back to literal path matching)"
-            );
-        } else {
-            info!(
-                pid = pid,
-                "Policy binary symlink resolution attempted via container filesystem \
-                 (check logs above for per-binary results)"
+                "Container filesystem /proc/{pid}/root/ not accessible after 10 attempts (5s); \
+                 binary symlink resolution skipped. Policy binary paths will be matched literally. \
+                 If binaries are symlinks, use canonical paths in your policy \
+                 (run 'readlink -f <path>' inside the sandbox)"
             );
-        }
+        });
     }
 
     // Spawn background policy poll task (gRPC mode only).

crates/openshell-sandbox/src/opa.rs

@@ -670,6 +670,10 @@ fn resolve_binary_in_container(policy_path: &str, entrypoint_pid: u32) -> Option
     for _ in 0..40 {
         let container_path = format!("/proc/{entrypoint_pid}/root{}", resolved.display());
 
+        tracing::debug!(
+            "Symlink resolution: probing container_path={container_path} for policy_path={policy_path} pid={entrypoint_pid}"
+        );
+
         let meta = match std::fs::symlink_metadata(&container_path) {
             Ok(m) => m,
             Err(e) => {
@@ -678,23 +682,18 @@ fn resolve_binary_in_container(policy_path: &str, entrypoint_pid: u32) -> Option
                 // legitimately not exist (broken symlink chain).
                 if resolved.as_os_str() == policy_path {
                     tracing::warn!(
-                        path = %policy_path,
-                        container_path = %container_path,
-                        pid = entrypoint_pid,
-                        error = %e,
-                        "Cannot access container filesystem for symlink resolution; \
-                         binary paths in policy will be matched literally. If a policy \
-                         binary is a symlink (e.g., /usr/bin/python3 -> python3.11), \
-                         use the canonical path instead, or run with CAP_SYS_PTRACE"
+                        "Cannot access container filesystem for symlink resolution: \
+                         path={policy_path} container_path={container_path} pid={entrypoint_pid} \
+                         error={e}. Binary paths in policy will be matched literally. \
+                         If this binary is a symlink (e.g., /usr/bin/python3 -> python3.11), \
+                         use the canonical path instead, or run with CAP_SYS_PTRACE."
                     );
                 } else {
                     tracing::warn!(
-                        original = %policy_path,
-                        current = %resolved.display(),
-                        pid = entrypoint_pid,
-                        error = %e,
-                        "Symlink chain broken during resolution; \
-                         binary will be matched by original path only"
+                        "Symlink chain broken during resolution: \
+                         original={policy_path} current={} pid={entrypoint_pid} error={e}. \
+                         Binary will be matched by original path only.",
+                        resolved.display()
                     );
                 }
                 return None;
@@ -710,12 +709,10 @@ fn resolve_binary_in_container(policy_path: &str, entrypoint_pid: u32) -> Option
             Ok(t) => t,
             Err(e) => {
                 tracing::warn!(
-                    path = %policy_path,
-                    current = %resolved.display(),
-                    pid = entrypoint_pid,
-                    error = %e,
-                    "Symlink detected but read_link failed; \
-                     binary will be matched by original path only"
+                    "Symlink detected but read_link failed: \
+                     path={policy_path} current={} pid={entrypoint_pid} error={e}. \
+                     Binary will be matched by original path only.",
+                    resolved.display()
                 );
                 return None;
             }
@@ -740,10 +737,8 @@ fn resolve_binary_in_container(policy_path: &str, entrypoint_pid: u32) -> Option
         None
     } else {
         tracing::info!(
-            original = %policy_path,
-            resolved = %resolved_str,
-            pid = entrypoint_pid,
-            "Resolved policy binary symlink via container filesystem"
+            "Resolved policy binary symlink via container filesystem: \
+             original={policy_path} resolved={resolved_str} pid={entrypoint_pid}"
         );
         Some(resolved_str)
     }