From 33d8bf1551982cea466e98d9a691f8d7ba9d40ea Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 11:49:55 -0700 Subject: [PATCH 1/8] chore: bump version to 0.1.6 Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1c231b..081110f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,7 +264,7 @@ dependencies = [ [[package]] name = "devbox" -version = "0.1.5" +version = "0.1.6" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 57a1d36..baac0f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbox" -version = "0.1.5" +version = "0.1.6" edition = "2024" description = "Isolated developer VMs for AI coding agents and humans. Safe by default." license = "Apache-2.0" From 40ad40bcd783d96ca40aa9dcb13fd9faf8f19d83 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 12:12:17 -0700 Subject: [PATCH 2/8] fix: set NixOS PATH for incus exec and wait for network before provisioning incus exec doesn't source NixOS login profile, so system binaries (sudo, base64, nixos-generate-config) in /run/current-system/sw/bin/ aren't in PATH. Also wait for DNS to be ready before downloading packages. Co-Authored-By: Claude Opus 4.6 --- src/sandbox/provision.rs | 178 +++++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 83 deletions(-) diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index e756cc6..4f958d9 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -236,20 +236,16 @@ async fn provision_nixos( ) -> Result<()> { let username = whoami(); + // 0. Wait for network connectivity (DNS resolution can lag behind the agent) + wait_for_network(runtime, name).await?; + // 1. Create directory structure println!("Setting up NixOS configuration..."); + let mkdir_cmd = format!( + "{NIXOS_PATH_PREFIX}sudo mkdir -p /etc/devbox/sets /etc/devbox/help" + ); runtime - .exec_cmd( - name, - &[ - "sudo", - "mkdir", - "-p", - "/etc/devbox/sets", - "/etc/devbox/help", - ], - false, - ) + .exec_cmd(name, &["bash", "-c", &mkdir_cmd], false) .await?; // 2. Generate base NixOS config if it doesn't exist @@ -280,13 +276,14 @@ async fn provision_nixos( // NixOS Lima images use flake-based NIX_PATH (nixpkgs=flake:nixpkgs) // which doesn't include nixos-config. We must set it explicitly. println!("Installing packages via nixos-rebuild (this may take a few minutes)..."); - let rebuild_cmd = concat!( - "export NIX_PATH=\"nixos-config=/etc/nixos/configuration.nix:$NIX_PATH\" && ", - "export NIXPKGS_ALLOW_UNFREE=1 && ", - "nixos-rebuild switch" + let rebuild_cmd = format!( + "{NIXOS_PATH_PREFIX}\ + export NIX_PATH=\"nixos-config=/etc/nixos/configuration.nix:$NIX_PATH\" && \ + export NIXPKGS_ALLOW_UNFREE=1 && \ + sudo -E nixos-rebuild switch" ); let result = runtime - .exec_cmd(name, &["sudo", "bash", "-c", rebuild_cmd], true) + .exec_cmd(name, &["bash", "-c", &rebuild_cmd], true) .await?; if result.exit_code != 0 { @@ -559,9 +556,9 @@ export DEVBOX_RUNTIME="${DEVBOX_RUNTIME:-unknown}" "#; write_file_to_vm(runtime, name, &zshrc_path, zshrc).await?; - let chown_cmd = format!("chown {username}:users {zshrc_path}"); + let chown_cmd = format!("{NIXOS_PATH_PREFIX}sudo chown {username}:users {zshrc_path}"); runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await?; } @@ -576,9 +573,9 @@ export DEVBOX_RUNTIME="${DEVBOX_RUNTIME:-unknown}" export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.claude/bin:$PATH" "#; write_file_to_vm(runtime, name, &profile_path, profile).await?; - let chown_cmd = format!("chown {username}:users {profile_path}"); + let chown_cmd = format!("{NIXOS_PATH_PREFIX}sudo chown {username}:users {profile_path}"); runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await?; } @@ -606,10 +603,8 @@ async fn setup_git_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let vm_path = format!("/home/{username}/.gitconfig"); write_file_to_vm(runtime, name, &vm_path, &content).await?; - let chown_cmd = format!("chown {username}:users {vm_path}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + let chown_cmd = format!("sudo chown {username}:users {vm_path}"); + run_in_vm(runtime, name, &chown_cmd, false).await?; println!("Synced host git config to VM."); Ok(()) @@ -767,18 +762,14 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> // Ensure parent directory exists with correct ownership let vm_parent = vm_path.rsplit_once('/').map(|(p, _)| p).unwrap_or(&vm_path); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", vm_parent], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {vm_parent}"), false).await?; write_file_to_vm(runtime, name, &vm_path, &content).await?; println!(" copied: ~/{host_suffix} → {vm_path}"); // Set restrictive permissions for credential/auth files if vm_suffix.contains("credential") || vm_suffix.contains("auth") { - runtime - .exec_cmd(name, &["sudo", "chmod", "600", &vm_path], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chmod 600 {vm_path}"), false).await?; } copied_any = true; } @@ -807,11 +798,9 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> // Fix ownership for all copied files let chown_cmd = format!( - "chown -R {username}:users {vm_home}/.claude {vm_home}/.config {vm_home}/.codex {vm_home}/.devbox-ai-env 2>/dev/null; true" + "sudo chown -R {username}:users {vm_home}/.claude {vm_home}/.config {vm_home}/.codex {vm_home}/.devbox-ai-env 2>/dev/null; true" ); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &chown_cmd, false).await?; } if copied_any { @@ -822,15 +811,10 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> let has_aichat_config = home.join(".config/aichat/config.yaml").exists(); if !has_aichat_config && let Some(config) = generate_aichat_config_from_credentials(&home) { let config_dir = format!("{vm_home}/.config/aichat"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &config_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {config_dir}"), false).await?; let config_path = format!("{config_dir}/config.yaml"); write_file_to_vm(runtime, name, &config_path, &config).await?; - let chown_cmd = format!("chown -R {username}:users {config_dir}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users {config_dir}"), false).await?; println!("Generated aichat config from detected AI tool credentials."); } @@ -898,6 +882,26 @@ fn generate_aichat_config_from_credentials(home: &std::path::Path) -> Option Result { + let full_cmd = format!("{NIXOS_PATH_PREFIX}{cmd}"); + runtime.exec_cmd(name, &["bash", "-c", &full_cmd], interactive).await +} + /// Write a file into the VM using base64-encoded content via exec_cmd. async fn write_file_to_vm( runtime: &dyn Runtime, @@ -907,7 +911,7 @@ async fn write_file_to_vm( ) -> Result<()> { use base64::Engine; let encoded = base64::engine::general_purpose::STANDARD.encode(content.as_bytes()); - let cmd = format!("echo '{encoded}' | base64 -d | sudo tee {path} > /dev/null"); + let cmd = format!("{NIXOS_PATH_PREFIX}echo '{encoded}' | base64 -d | sudo tee {path} > /dev/null"); let result = runtime.exec_cmd(name, &["bash", "-c", &cmd], false).await?; if result.exit_code != 0 { eprintln!("Warning: failed to write {path}: {}", result.stderr.trim()); @@ -915,6 +919,38 @@ async fn write_file_to_vm( Ok(()) } +/// Wait for network connectivity inside the VM. +/// +/// On freshly booted Incus VMs, the network (especially DNS) may not be ready +/// even after the agent responds. We poll for DNS resolution of cache.nixos.org +/// since that's needed for `nixos-rebuild` and `nix-env` operations. +async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { + let max_attempts = 20; // 20 * 3s = 60s + for i in 0..max_attempts { + let result = run_in_vm( + runtime, + name, + "getent hosts cache.nixos.org >/dev/null 2>&1 && echo ok", + false, + ) + .await?; + if result.exit_code == 0 && result.stdout.trim() == "ok" { + if i > 0 { + println!("Network is ready."); + } + return Ok(()); + } + if i == 0 { + print!("Waiting for network connectivity..."); + } else if i % 5 == 0 { + print!(" ({}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + eprintln!("\nWarning: network may not be ready — provisioning will continue but downloads may fail."); + Ok(()) +} + /// Ensure /etc/nixos/configuration.nix and hardware-configuration.nix exist. /// /// NixOS Lima images ship with an empty /etc/nixos/ directory. @@ -933,8 +969,9 @@ async fn ensure_nixos_config(runtime: &dyn Runtime, name: &str) -> Result<()> { if hw_check.exit_code != 0 { println!(" Generating hardware configuration..."); + let gen_cmd = format!("{NIXOS_PATH_PREFIX}sudo nixos-generate-config"); let result = runtime - .exec_cmd(name, &["sudo", "nixos-generate-config"], false) + .exec_cmd(name, &["bash", "-c", &gen_cmd], false) .await?; if result.exit_code != 0 { eprintln!( @@ -1009,20 +1046,13 @@ async fn copy_devbox_to_vm(runtime: &dyn Runtime, name: &str) -> Result<()> { if let Ok(r) = result && r.exit_code == 0 { - let _ = runtime - .exec_cmd( - name, - &[ - "sudo", - "install", - "-m", - "755", - "/tmp/devbox", - "/usr/local/bin/devbox", - ], - false, - ) - .await; + let _ = run_in_vm( + runtime, + name, + "sudo install -m 755 /tmp/devbox /usr/local/bin/devbox", + false, + ) + .await; let _ = runtime.exec_cmd(name, &["rm", "/tmp/devbox"], false).await; } } @@ -1035,9 +1065,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let config_dir = format!("/home/{username}/.config/yazi"); // Create config directory - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &config_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {config_dir}"), false).await?; // Write all yazi config files let files: &[(&str, &str)] = &[ @@ -1053,9 +1081,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { // Write glow previewer plugin let plugin_dir = format!("{config_dir}/plugins/glow.yazi"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &plugin_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {plugin_dir}"), false).await?; write_file_to_vm( runtime, name, @@ -1065,10 +1091,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { .await?; // Fix ownership - let chown_cmd = format!("chown -R {username}:users /home/{username}/.config/yazi"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users /home/{username}/.config/yazi"), false).await?; Ok(()) } @@ -1080,9 +1103,7 @@ async fn setup_aichat_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let config_dir = format!("/home/{username}/.config/aichat"); let roles_dir = format!("{config_dir}/roles"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &roles_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {roles_dir}"), false).await?; // Legacy format (older aichat versions) write_file_to_vm( @@ -1102,10 +1123,7 @@ async fn setup_aichat_config(runtime: &dyn Runtime, name: &str) -> Result<()> { write_file_to_vm(runtime, name, &format!("{roles_dir}/{filename}"), content).await?; } - let chown_cmd = format!("chown -R {username}:users {config_dir}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users {config_dir}"), false).await?; Ok(()) } @@ -1119,13 +1137,7 @@ async fn setup_management_script(runtime: &dyn Runtime, name: &str) -> Result<() MANAGEMENT_SCRIPT, ) .await?; - runtime - .exec_cmd( - name, - &["sudo", "chmod", "+x", "/etc/devbox/management.sh"], - false, - ) - .await?; + run_in_vm(runtime, name, "sudo chmod +x /etc/devbox/management.sh", false).await?; Ok(()) } @@ -1188,10 +1200,10 @@ async fn install_latest_claude_code(runtime: &dyn Runtime, name: &str) { } // Fix ownership let chown_cmd = format!( - "chown {username}:users /home/{username}/.zshrc /home/{username}/.profile 2>/dev/null; true" + "{NIXOS_PATH_PREFIX}sudo chown {username}:users /home/{username}/.zshrc /home/{username}/.profile 2>/dev/null; true" ); let _ = runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await; } From 2f7100b9979e0f03acb11479320af572754092e9 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 12:16:57 -0700 Subject: [PATCH 3/8] fix: set up NixOS channel and ensure homedir before nixos-rebuild The images:nixos/25.11 Incus image has no nix channels configured, causing nixos-rebuild to fail with "nixpkgs/nixos not found". Also ensure the user's home directory exists before writing config files, since the user is created by nixos-rebuild via devbox-module.nix. Co-Authored-By: Claude Opus 4.6 --- src/sandbox/provision.rs | 69 +++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index 4f958d9..492f7fe 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -248,16 +248,34 @@ async fn provision_nixos( .exec_cmd(name, &["bash", "-c", &mkdir_cmd], false) .await?; - // 2. Generate base NixOS config if it doesn't exist + // 2. Ensure NixOS channel is available (images:nixos/* may not have it) + // nixos-rebuild needs `` in NIX_PATH, which comes from + // the nixos channel. If the channel isn't set up, add and update it. + ensure_nixos_channel(runtime, name).await?; + + // 3. Generate base NixOS config if it doesn't exist // NixOS Lima images ship with an empty /etc/nixos/ — we need to // run nixos-generate-config to create the hardware and base configs. ensure_nixos_config(runtime, name).await?; - // 3. Push devbox-state.toml (includes mount_mode for overlay setup) + // 4. Ensure user home directory exists (the user may not exist yet on + // fresh images:nixos/* images — nixos-rebuild will create it via + // devbox-module.nix, but we need the homedir for writing config files + // before rebuild. We create it manually and let NixOS fix ownership later.) + let home_dir = format!("/home/{username}"); + run_in_vm( + runtime, + name, + &format!("sudo mkdir -p {home_dir} && sudo chown $(id -u {username} 2>/dev/null || echo 1000):users {home_dir} 2>/dev/null; true"), + false, + ) + .await?; + + // 5. Push devbox-state.toml (includes mount_mode for overlay setup) let state_toml = generate_state_toml(sets, languages, &username, mount_mode); write_file_to_vm(runtime, name, "/etc/devbox/devbox-state.toml", &state_toml).await?; - // 4. Push devbox-module.nix + // 6. Push devbox-module.nix write_file_to_vm( runtime, name, @@ -266,15 +284,13 @@ async fn provision_nixos( ) .await?; - // 5. Push all set .nix files + // 7. Push all set .nix files for (filename, content) in NIX_SET_FILES { let path = format!("/etc/devbox/sets/{filename}"); write_file_to_vm(runtime, name, &path, content).await?; } - // 6. Run nixos-rebuild switch (interactive so user sees progress) - // NixOS Lima images use flake-based NIX_PATH (nixpkgs=flake:nixpkgs) - // which doesn't include nixos-config. We must set it explicitly. + // 8. Run nixos-rebuild switch (interactive so user sees progress) println!("Installing packages via nixos-rebuild (this may take a few minutes)..."); let rebuild_cmd = format!( "{NIXOS_PATH_PREFIX}\ @@ -293,7 +309,7 @@ async fn provision_nixos( println!("NixOS rebuild complete."); } - // 8. Set up user shell (zshrc with PATH, aliases, etc.) + // 9. Set up user shell (zshrc with PATH, aliases, etc.) setup_nixos_shell(runtime, name).await?; // 9. Install latest claude-code (nixpkgs version lags behind) @@ -951,6 +967,43 @@ async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { Ok(()) } +/// Ensure the NixOS channel is available so `nixos-rebuild` can find ``. +/// +/// The `images:nixos/*` Incus images may not have channels configured, causing +/// `nixos-rebuild` to fail with "file 'nixpkgs/nixos' was not found in the Nix search path". +/// We check if the nixos channel exists and add it if missing. +async fn ensure_nixos_channel(runtime: &dyn Runtime, name: &str) -> Result<()> { + // Check if nixpkgs is already in NIX_PATH + let check = run_in_vm( + runtime, + name, + "nix-instantiate --eval -E '' 2>/dev/null && echo found", + false, + ) + .await?; + + if check.stdout.contains("found") { + return Ok(()); + } + + println!("Setting up NixOS channel (required for nixos-rebuild)..."); + let channel_cmd = concat!( + "sudo nix-channel --add https://nixos.org/channels/nixos-25.05 nixos && ", + "sudo nix-channel --update" + ); + let result = run_in_vm(runtime, name, channel_cmd, false).await?; + if result.exit_code != 0 { + eprintln!( + "Warning: failed to set up NixOS channel: {}", + result.stderr.trim() + ); + } else { + println!("NixOS channel configured."); + } + + Ok(()) +} + /// Ensure /etc/nixos/configuration.nix and hardware-configuration.nix exist. /// /// NixOS Lima images ship with an empty /etc/nixos/ directory. From 002394d360e59518ff1ede33c5605385ba083792 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 13:14:58 -0700 Subject: [PATCH 4/8] feat: add Incus network diagnostics to doctor, fail fast on no connectivity - devbox doctor now checks incusbr0 bridge, IP forwarding, iptables FORWARD rules, NAT masquerade, and tests VM connectivity on Linux - devbox create now fails fast with actionable fix commands if the VM has no internet, instead of silently timing out on every download Co-Authored-By: Claude Opus 4.6 --- src/cli/doctor.rs | 117 +++++++++++++++++++++++++++++++++++++++ src/sandbox/provision.rs | 57 +++++++++++++++---- 2 files changed, 164 insertions(+), 10 deletions(-) diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index d21e921..da62f3c 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -1,6 +1,7 @@ use anyhow::Result; use clap::Args; +use crate::runtime::cmd::run_cmd; use crate::runtime::detect::detect_runtime; use crate::sandbox::SandboxManager; @@ -8,6 +9,8 @@ use crate::sandbox::SandboxManager; pub struct DoctorArgs {} pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { + #[allow(unused_assignments)] + let mut has_incus = false; println!("devbox doctor\n"); let os = std::env::consts::OS; @@ -23,6 +26,7 @@ pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { "sudo apt install incus # or: snap install incus", ); has_any_runtime |= found; + has_incus = found; // QEMU and virtiofsd are required for Incus VMs on Linux if found { @@ -126,10 +130,123 @@ pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh", ); + // Incus network diagnostics (Linux only) + if os == "linux" && has_incus { + println!("\nIncus network:"); + check_incus_network().await; + } + println!("\nAll checks complete."); Ok(()) } +/// Check Incus network configuration: bridge, NAT, IP forwarding, iptables FORWARD rules. +async fn check_incus_network() { + // 1. Check incusbr0 exists and has NAT enabled + let bridge = run_cmd("incus", &["network", "show", "incusbr0"]).await; + match bridge { + Ok(r) if r.exit_code == 0 => { + let has_nat = r.stdout.contains("ipv4.nat") && r.stdout.contains("\"true\""); + if has_nat { + println!(" Bridge (incusbr0): \x1b[32mok\x1b[0m (NAT enabled)"); + } else { + println!(" Bridge (incusbr0): \x1b[33mexists but NAT may be off\x1b[0m"); + println!(" Fix: incus network set incusbr0 ipv4.nat true"); + } + } + _ => { + println!(" Bridge (incusbr0): \x1b[31mnot found\x1b[0m"); + println!(" Fix: incus network create incusbr0"); + return; + } + } + + // 2. Check IP forwarding + let fwd = run_cmd("sysctl", &["-n", "net.ipv4.ip_forward"]).await; + match fwd { + Ok(r) if r.stdout.trim() == "1" => { + println!(" IP forwarding: \x1b[32menabled\x1b[0m"); + } + _ => { + println!(" IP forwarding: \x1b[31mdisabled\x1b[0m"); + println!(" Fix: sudo sysctl -w net.ipv4.ip_forward=1"); + println!(" Persist: echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/99-incus.conf"); + } + } + + // 3. Check iptables FORWARD chain for incusbr0 rules + let fwd_rules = run_cmd("iptables", &["-S", "FORWARD"]).await; + let has_forward_rule = match &fwd_rules { + Ok(r) => r.stdout.contains("incusbr0") && r.stdout.contains("ACCEPT"), + Err(_) => false, + }; + + if has_forward_rule { + println!(" iptables FORWARD: \x1b[32mincusbr0 allowed\x1b[0m"); + } else { + // Check FORWARD policy + let policy_drop = match &fwd_rules { + Ok(r) => r.stdout.contains("-P FORWARD DROP"), + Err(_) => false, + }; + if policy_drop { + println!(" iptables FORWARD: \x1b[31mDROP policy, no incusbr0 rule\x1b[0m"); + println!(" VM traffic is being blocked by the firewall."); + println!(" Fix:"); + println!(" sudo iptables -I FORWARD -i incusbr0 -j ACCEPT"); + println!(" sudo iptables -I FORWARD -o incusbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT"); + } else { + println!(" iptables FORWARD: \x1b[32mACCEPT policy\x1b[0m"); + } + } + + // 4. Check NAT masquerade for Incus subnet + let nat_rules = run_cmd("iptables", &["-t", "nat", "-S", "POSTROUTING"]).await; + let has_masq = match &nat_rules { + Ok(r) => r.stdout.contains("incusbr0") || r.stdout.contains("10.195.64"), + Err(_) => false, + }; + + if has_masq { + println!(" iptables NAT: \x1b[32mmasquerade configured\x1b[0m"); + } else { + println!(" iptables NAT: \x1b[33mno masquerade for Incus subnet\x1b[0m"); + println!(" Fix: sudo iptables -t nat -A POSTROUTING -s 10.195.64.0/24 ! -o incusbr0 -j MASQUERADE"); + } + + // 5. Quick connectivity test if any running VM exists + let list = run_cmd("incus", &["list", "devbox-", "--format", "json"]).await; + if let Ok(r) = list { + if let Ok(arr) = serde_json::from_str::>(&r.stdout) { + for v in &arr { + if v["status"].as_str() == Some("Running") { + let vm_name = v["name"].as_str().unwrap_or(""); + if !vm_name.is_empty() { + let ping = run_cmd( + "incus", + &["exec", vm_name, "--", "ping", "-c", "1", "-W", "3", "8.8.8.8"], + ) + .await; + match ping { + Ok(p) if p.exit_code == 0 => { + println!( + " VM connectivity ({vm_name}): \x1b[32mok\x1b[0m" + ); + } + _ => { + println!( + " VM connectivity ({vm_name}): \x1b[31mno internet\x1b[0m" + ); + } + } + break; // Only test one VM + } + } + } + } + } +} + /// Check if a binary is available. If missing, print install instructions. /// Returns true if found. fn check_binary_with_install(label: &str, name: &str, install_hint: &str) -> bool { diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index 492f7fe..5e9b6fb 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -938,23 +938,26 @@ async fn write_file_to_vm( /// Wait for network connectivity inside the VM. /// /// On freshly booted Incus VMs, the network (especially DNS) may not be ready -/// even after the agent responds. We poll for DNS resolution of cache.nixos.org -/// since that's needed for `nixos-rebuild` and `nix-env` operations. +/// even after the agent responds. We first wait for basic IP connectivity +/// (ping), then check DNS resolution. If basic connectivity never comes up, +/// we bail early with actionable diagnostics instead of letting every +/// subsequent download time out. async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { - let max_attempts = 20; // 20 * 3s = 60s - for i in 0..max_attempts { + // Phase 1: Wait for basic IP connectivity (ping 8.8.8.8) + // This distinguishes "network not ready yet" from "no route / firewall blocks" + let ping_attempts = 10; // 10 * 3s = 30s + let mut got_ping = false; + for i in 0..ping_attempts { let result = run_in_vm( runtime, name, - "getent hosts cache.nixos.org >/dev/null 2>&1 && echo ok", + "ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1 && echo ok", false, ) .await?; if result.exit_code == 0 && result.stdout.trim() == "ok" { - if i > 0 { - println!("Network is ready."); - } - return Ok(()); + got_ping = true; + break; } if i == 0 { print!("Waiting for network connectivity..."); @@ -963,7 +966,41 @@ async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { } tokio::time::sleep(std::time::Duration::from_secs(3)).await; } - eprintln!("\nWarning: network may not be ready — provisioning will continue but downloads may fail."); + + if !got_ping { + println!(); + eprintln!("\x1b[31mError: VM has no network connectivity.\x1b[0m"); + eprintln!("The VM cannot reach the internet. This is usually caused by"); + eprintln!("missing iptables FORWARD rules for the Incus bridge.\n"); + eprintln!("Quick fix (run on the host):"); + eprintln!(" sudo iptables -I FORWARD -i incusbr0 -j ACCEPT"); + eprintln!(" sudo iptables -I FORWARD -o incusbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT"); + eprintln!(" sudo iptables -t nat -A POSTROUTING -s 10.195.64.0/24 ! -o incusbr0 -j MASQUERADE\n"); + eprintln!("Run `devbox doctor` for full network diagnostics."); + anyhow::bail!("VM network connectivity check failed — cannot provision without internet access"); + } + + // Phase 2: Wait for DNS resolution + let dns_attempts = 10; // 10 * 3s = 30s + for i in 0..dns_attempts { + let result = run_in_vm( + runtime, + name, + "getent hosts cache.nixos.org >/dev/null 2>&1 && echo ok", + false, + ) + .await?; + if result.exit_code == 0 && result.stdout.trim() == "ok" { + println!(" ready."); + return Ok(()); + } + if i % 5 == 0 { + print!(" (DNS {}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + println!(); + eprintln!("Warning: DNS resolution not working yet — provisioning will continue but downloads may fail."); Ok(()) } From b77f4b40c790ec16a291af33eda30121341f952f Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 14:35:25 -0700 Subject: [PATCH 5/8] fix: set NIX_PATH for nixos-rebuild and run interactive sessions as user - Explicitly set NIX_PATH to include the channel profile path so nixos-rebuild can find after nix-channel --update - Run interactive incus exec sessions as the non-root user (UID >= 1000) with correct HOME and /workspace as working directory - Make channel update interactive so progress is visible Co-Authored-By: Claude Opus 4.6 --- src/runtime/incus.rs | 51 +++++++++++++++++++++++++++++++++++++--- src/sandbox/provision.rs | 18 ++++++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/runtime/incus.rs b/src/runtime/incus.rs index 226bc68..f934998 100644 --- a/src/runtime/incus.rs +++ b/src/runtime/incus.rs @@ -71,6 +71,33 @@ impl IncusRuntime { println!("Image '{alias}' imported successfully."); Ok(()) } + /// Detect the UID of the first non-root user in the VM. + async fn detect_vm_uid(vm: &str) -> Option { + // Find first user with UID >= 1000 (standard non-system user) + let result = run_cmd( + "incus", + &["exec", vm, "--", "bash", "-c", + "awk -F: '$3 >= 1000 && $3 < 65534 { print $3; exit }' /etc/passwd"], + ).await.ok()?; + let uid = result.stdout.trim().to_string(); + if uid.is_empty() { None } else { Some(uid) } + } + + /// Detect the HOME directory for a given UID in the VM. + async fn detect_vm_home(vm: &str, uid: &str) -> String { + let result = run_cmd( + "incus", + &["exec", vm, "--", "bash", "-c", + &format!("getent passwd {uid} | cut -d: -f6")], + ).await; + match result { + Ok(r) if !r.stdout.trim().is_empty() => { + format!("HOME={}", r.stdout.trim()) + } + _ => format!("HOME=/home/dev"), + } + } + /// Wait for the Incus VM agent to become ready (up to 120 seconds). /// The agent starts after the guest OS boots and runs incus-agent. async fn wait_for_agent(vm: &str) -> Result<()> { @@ -190,9 +217,27 @@ impl Runtime for IncusRuntime { let vm = Self::vm_name(name); if interactive { - let mut args = vec!["exec", &vm, "--"]; - args.extend_from_slice(cmd); - run_interactive("incus", &args).await + // For interactive sessions (shell attach, etc.), run as the + // non-root user with correct HOME and working directory. + let uid_str = Self::detect_vm_uid(&vm).await.unwrap_or("1000".to_string()); + let home_env = Self::detect_vm_home(&vm, &uid_str).await; + + let mut args = vec![ + "exec".to_string(), + vm, + "--user".to_string(), + uid_str, + "--cwd".to_string(), + "/workspace".to_string(), + "--env".to_string(), + home_env, + "--".to_string(), + ]; + for c in cmd { + args.push(c.to_string()); + } + let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + run_interactive("incus", &arg_refs).await } else { let mut args = vec!["exec", &vm, "--"]; args.extend_from_slice(cmd); diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index 5e9b6fb..2e6aa8f 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -291,10 +291,16 @@ async fn provision_nixos( } // 8. Run nixos-rebuild switch (interactive so user sees progress) + // We must set NIX_PATH explicitly because: + // - incus exec doesn't source /etc/profile (no login shell) + // - images:nixos/* may not have channels in the default NIX_PATH + // - After nix-channel --update, nixpkgs lives at the channel profile path println!("Installing packages via nixos-rebuild (this may take a few minutes)..."); let rebuild_cmd = format!( "{NIXOS_PATH_PREFIX}\ - export NIX_PATH=\"nixos-config=/etc/nixos/configuration.nix:$NIX_PATH\" && \ + export NIX_PATH=\"nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos:\ + nixos-config=/etc/nixos/configuration.nix:\ + /nix/var/nix/profiles/per-user/root/channels\" && \ export NIXPKGS_ALLOW_UNFREE=1 && \ sudo -E nixos-rebuild switch" ); @@ -1010,16 +1016,18 @@ async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { /// `nixos-rebuild` to fail with "file 'nixpkgs/nixos' was not found in the Nix search path". /// We check if the nixos channel exists and add it if missing. async fn ensure_nixos_channel(runtime: &dyn Runtime, name: &str) -> Result<()> { - // Check if nixpkgs is already in NIX_PATH + // Check if the nixos channel is already available for root. + // We check the channel profile path directly since NIX_PATH may not be set + // in the non-login incus exec shell. let check = run_in_vm( runtime, name, - "nix-instantiate --eval -E '' 2>/dev/null && echo found", + "test -d /nix/var/nix/profiles/per-user/root/channels/nixos && echo found", false, ) .await?; - if check.stdout.contains("found") { + if check.stdout.trim() == "found" { return Ok(()); } @@ -1028,7 +1036,7 @@ async fn ensure_nixos_channel(runtime: &dyn Runtime, name: &str) -> Result<()> { "sudo nix-channel --add https://nixos.org/channels/nixos-25.05 nixos && ", "sudo nix-channel --update" ); - let result = run_in_vm(runtime, name, channel_cmd, false).await?; + let result = run_in_vm(runtime, name, channel_cmd, true).await?; if result.exit_code != 0 { eprintln!( "Warning: failed to set up NixOS channel: {}", From 9ff803ecc76d241af5d655cc21fd7cf281747aa9 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 15:17:44 -0700 Subject: [PATCH 6/8] fix: run provisioning as root, shell attach as user on Incus exec_cmd always runs as root (needed for provisioning). New exec_as_user method runs interactive sessions as the first non-root user with correct HOME and /workspace CWD. Also fixes NIX_PATH to include channel profile path for nixos-rebuild. Co-Authored-By: Claude Opus 4.6 --- src/runtime/incus.rs | 50 +++++++++++++++++++++++++------------------- src/runtime/mod.rs | 7 +++++++ src/sandbox/mod.rs | 9 ++++---- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/runtime/incus.rs b/src/runtime/incus.rs index f934998..9f57000 100644 --- a/src/runtime/incus.rs +++ b/src/runtime/incus.rs @@ -217,27 +217,9 @@ impl Runtime for IncusRuntime { let vm = Self::vm_name(name); if interactive { - // For interactive sessions (shell attach, etc.), run as the - // non-root user with correct HOME and working directory. - let uid_str = Self::detect_vm_uid(&vm).await.unwrap_or("1000".to_string()); - let home_env = Self::detect_vm_home(&vm, &uid_str).await; - - let mut args = vec![ - "exec".to_string(), - vm, - "--user".to_string(), - uid_str, - "--cwd".to_string(), - "/workspace".to_string(), - "--env".to_string(), - home_env, - "--".to_string(), - ]; - for c in cmd { - args.push(c.to_string()); - } - let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - run_interactive("incus", &arg_refs).await + let mut args = vec!["exec", &vm, "--"]; + args.extend_from_slice(cmd); + run_interactive("incus", &args).await } else { let mut args = vec!["exec", &vm, "--"]; args.extend_from_slice(cmd); @@ -245,6 +227,32 @@ impl Runtime for IncusRuntime { } } + /// Execute an interactive command as the non-root user. + /// Incus exec defaults to root, so we detect the first UID >= 1000 + /// and set --user, HOME, and CWD for proper user sessions. + async fn exec_as_user(&self, name: &str, cmd: &[&str]) -> Result { + let vm = Self::vm_name(name); + let uid_str = Self::detect_vm_uid(&vm).await.unwrap_or("1000".to_string()); + let home_env = Self::detect_vm_home(&vm, &uid_str).await; + + let mut args = vec![ + "exec".to_string(), + vm, + "--user".to_string(), + uid_str, + "--cwd".to_string(), + "/workspace".to_string(), + "--env".to_string(), + home_env, + "--".to_string(), + ]; + for c in cmd { + args.push(c.to_string()); + } + let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + run_interactive("incus", &arg_refs).await + } + async fn destroy(&self, name: &str) -> Result<()> { let vm = Self::vm_name(name); // Stop first (ignore errors if already stopped) diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index e31ce2e..25f3538 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -126,4 +126,11 @@ pub trait Runtime: Send + Sync { /// Update mount points for an existing sandbox. /// Stops the VM, updates mounts in the config, and restarts. async fn update_mounts(&self, name: &str, mounts: &[Mount]) -> Result<()>; + + /// Execute an interactive command as the non-root user. + /// Used for shell attach — defaults to exec_cmd with interactive=true. + /// Runtimes like Incus override this to set --user, HOME, and CWD. + async fn exec_as_user(&self, name: &str, cmd: &[&str]) -> Result { + self.exec_cmd(name, cmd, true).await + } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 8404ad9..377e672 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -237,7 +237,7 @@ impl SandboxManager { if layout == "plain" { println!("Attaching to sandbox '{name}'..."); let shell = Self::probe_shell(runtime.as_ref(), name).await; - runtime.exec_cmd(name, &[&shell, "-l"], true).await?; + runtime.exec_as_user(name, &[&shell, "-l"]).await?; return Ok(()); } @@ -249,7 +249,7 @@ impl SandboxManager { // No Zellij — fall back to raw shell println!("Attaching to sandbox '{name}'..."); let shell = Self::probe_shell(runtime.as_ref(), name).await; - runtime.exec_cmd(name, &[&shell, "-l"], true).await?; + runtime.exec_as_user(name, &[&shell, "-l"]).await?; return Ok(()); } @@ -317,7 +317,7 @@ impl SandboxManager { // Reattach to existing live session println!("Reattaching to sandbox '{name}'..."); runtime - .exec_cmd(name, &["zellij", "attach", &session_name], true) + .exec_as_user(name, &["zellij", "attach", &session_name]) .await?; } else { // Create new named session with layout. @@ -336,10 +336,9 @@ impl SandboxManager { println!("Attaching to sandbox '{name}' (layout: {effective_layout})..."); runtime - .exec_cmd( + .exec_as_user( name, &["zellij", "--config", &config_path, "--layout", &layout_path], - true, ) .await?; } From 1086b79b475cd3c61bb1d9ea00b04149c7d31234 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 16:30:54 -0700 Subject: [PATCH 7/8] fix: handle incus-agent restart during nixos-rebuild, default 4GiB RAM nixos-rebuild switch stops incus-agent during activation, dropping the websocket (exit 255). Now we detect this, wait for the agent to restart, then continue provisioning. Also default Incus VMs to 4GiB memory since NixOS rebuild evaluation needs 2-4GB (default 1GB causes OOM kills). Co-Authored-By: Claude Opus 4.6 --- src/runtime/incus.rs | 11 ++++---- src/sandbox/provision.rs | 60 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/runtime/incus.rs b/src/runtime/incus.rs index 9f57000..d35b77a 100644 --- a/src/runtime/incus.rs +++ b/src/runtime/incus.rs @@ -159,12 +159,13 @@ impl Runtime for IncusRuntime { launch_args.push(&cpu_str); } + // Default to 4GiB memory for Incus VMs — NixOS rebuild needs 2-4GB + // for evaluating the full module system. The default Incus 1GB is too little. let mem_str; - if !opts.memory.is_empty() { - mem_str = format!("limits.memory={}", opts.memory); - launch_args.push("-c"); - launch_args.push(&mem_str); - } + let memory = if opts.memory.is_empty() { "4GiB" } else { &opts.memory }; + mem_str = format!("limits.memory={memory}"); + launch_args.push("-c"); + launch_args.push(&mem_str); run_ok("incus", &launch_args).await?; diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index 2e6aa8f..71a2bcd 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -306,13 +306,61 @@ async fn provision_nixos( ); let result = runtime .exec_cmd(name, &["bash", "-c", &rebuild_cmd], true) - .await?; + .await; - if result.exit_code != 0 { - eprintln!("Warning: nixos-rebuild failed (exit {})", result.exit_code); - eprintln!("You can retry with `devbox exec --name {name} -- sudo nixos-rebuild switch`"); - } else { - println!("NixOS rebuild complete."); + // nixos-rebuild switch stops incus-agent during activation, which + // drops the websocket connection (exit 255). This is expected — + // the agent restarts automatically with the new system config. + // We need to wait for it to come back before continuing. + let rebuild_ok = match &result { + Ok(r) if r.exit_code == 0 => true, + Ok(r) if r.exit_code == 255 => { + // Likely the websocket dropped during activation — probably succeeded + println!("Connection lost during system activation (expected)."); + true + } + Ok(r) => { + eprintln!("Warning: nixos-rebuild failed (exit {})", r.exit_code); + eprintln!("You can retry with `devbox exec --name {name} -- sudo nixos-rebuild switch`"); + false + } + Err(e) => { + eprintln!("Warning: nixos-rebuild error: {e}"); + false + } + }; + + if rebuild_ok { + // Wait for the VM agent to come back after system activation. + // nixos-rebuild switch restarts incus-agent as part of the new config. + println!("Waiting for VM agent to restart after system activation..."); + // Small delay to let the old agent fully stop + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + + let vm = format!("devbox-{name}"); + let max_attempts = 40; // 40 * 3s = 120s + let mut agent_ready = false; + for i in 0..max_attempts { + let check = crate::runtime::cmd::run_cmd( + "incus", + &["exec", &vm, "--", "echo", "ready"], + ) + .await; + if let Ok(r) = check { + if r.exit_code == 0 && r.stdout.trim() == "ready" { + println!("NixOS rebuild complete. VM agent is ready."); + agent_ready = true; + break; + } + } + if i > 0 && i % 10 == 0 { + println!(" Still waiting for VM agent... ({}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + if !agent_ready { + eprintln!("Warning: VM agent did not come back after nixos-rebuild. Continuing anyway..."); + } } // 9. Set up user shell (zshrc with PATH, aliases, etc.) From 868d99bd60d1d8bb1f5dd716031cdae3c05df9c0 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Mar 2026 16:44:45 -0700 Subject: [PATCH 8/8] fix: add incus-agent systemd service to NixOS module After nixos-rebuild switch, the new system config replaces the original image config. Without an explicit incus-agent service definition, the agent never restarts, making the VM unreachable via `incus exec`. Co-Authored-By: Claude Opus 4.6 --- nix/devbox-module.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nix/devbox-module.nix b/nix/devbox-module.nix index 597fe71..5c49ef9 100644 --- a/nix/devbox-module.nix +++ b/nix/devbox-module.nix @@ -50,6 +50,26 @@ in { virtualisation.docker.enable = lib.mkDefault (sets.container or false); services.tailscale.enable = lib.mkDefault (sets.network or false); + # ── Incus/LXD Agent ───────────────────────────────── + # Required for `incus exec` to work after nixos-rebuild. + # The agent binary is mounted at /run/incus_agent/ by the hypervisor; + # this service starts it on boot so the host can communicate with the VM. + systemd.services.incus-agent = { + description = "Incus VM Agent"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "local-fs.target" ]; + serviceConfig = { + Type = "notify"; + ExecStart = "/run/incus_agent/incus-agent"; + Restart = "always"; + RestartSec = 5; + WorkingDirectory = "/run/incus_agent"; + }; + unitConfig = { + ConditionPathExists = "/run/incus_agent/incus-agent"; + }; + }; + # ── Dynamic linker compat ───────────────────────── # Required for VS Code Server, Cursor, and other dynamically linked # binaries that expect a standard FHS layout (ld-linux, libc, etc.).