diff --git a/Cargo.lock b/Cargo.lock index e1c231b..081110f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,7 +264,7 @@ dependencies = [ [[package]] name = "devbox" -version = "0.1.5" +version = "0.1.6" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 57a1d36..baac0f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbox" -version = "0.1.5" +version = "0.1.6" edition = "2024" description = "Isolated developer VMs for AI coding agents and humans. Safe by default." license = "Apache-2.0" diff --git a/nix/devbox-module.nix b/nix/devbox-module.nix index 597fe71..5c49ef9 100644 --- a/nix/devbox-module.nix +++ b/nix/devbox-module.nix @@ -50,6 +50,26 @@ in { virtualisation.docker.enable = lib.mkDefault (sets.container or false); services.tailscale.enable = lib.mkDefault (sets.network or false); + # ── Incus/LXD Agent ───────────────────────────────── + # Required for `incus exec` to work after nixos-rebuild. + # The agent binary is mounted at /run/incus_agent/ by the hypervisor; + # this service starts it on boot so the host can communicate with the VM. + systemd.services.incus-agent = { + description = "Incus VM Agent"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "local-fs.target" ]; + serviceConfig = { + Type = "notify"; + ExecStart = "/run/incus_agent/incus-agent"; + Restart = "always"; + RestartSec = 5; + WorkingDirectory = "/run/incus_agent"; + }; + unitConfig = { + ConditionPathExists = "/run/incus_agent/incus-agent"; + }; + }; + # ── Dynamic linker compat ───────────────────────── # Required for VS Code Server, Cursor, and other dynamically linked # binaries that expect a standard FHS layout (ld-linux, libc, etc.). diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index d21e921..da62f3c 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -1,6 +1,7 @@ use anyhow::Result; use clap::Args; +use crate::runtime::cmd::run_cmd; use crate::runtime::detect::detect_runtime; use crate::sandbox::SandboxManager; @@ -8,6 +9,8 @@ use crate::sandbox::SandboxManager; pub struct DoctorArgs {} pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { + #[allow(unused_assignments)] + let mut has_incus = false; println!("devbox doctor\n"); let os = std::env::consts::OS; @@ -23,6 +26,7 @@ pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { "sudo apt install incus # or: snap install incus", ); has_any_runtime |= found; + has_incus = found; // QEMU and virtiofsd are required for Incus VMs on Linux if found { @@ -126,10 +130,123 @@ pub async fn run(_args: DoctorArgs, manager: &SandboxManager) -> Result<()> { "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh", ); + // Incus network diagnostics (Linux only) + if os == "linux" && has_incus { + println!("\nIncus network:"); + check_incus_network().await; + } + println!("\nAll checks complete."); Ok(()) } +/// Check Incus network configuration: bridge, NAT, IP forwarding, iptables FORWARD rules. +async fn check_incus_network() { + // 1. Check incusbr0 exists and has NAT enabled + let bridge = run_cmd("incus", &["network", "show", "incusbr0"]).await; + match bridge { + Ok(r) if r.exit_code == 0 => { + let has_nat = r.stdout.contains("ipv4.nat") && r.stdout.contains("\"true\""); + if has_nat { + println!(" Bridge (incusbr0): \x1b[32mok\x1b[0m (NAT enabled)"); + } else { + println!(" Bridge (incusbr0): \x1b[33mexists but NAT may be off\x1b[0m"); + println!(" Fix: incus network set incusbr0 ipv4.nat true"); + } + } + _ => { + println!(" Bridge (incusbr0): \x1b[31mnot found\x1b[0m"); + println!(" Fix: incus network create incusbr0"); + return; + } + } + + // 2. Check IP forwarding + let fwd = run_cmd("sysctl", &["-n", "net.ipv4.ip_forward"]).await; + match fwd { + Ok(r) if r.stdout.trim() == "1" => { + println!(" IP forwarding: \x1b[32menabled\x1b[0m"); + } + _ => { + println!(" IP forwarding: \x1b[31mdisabled\x1b[0m"); + println!(" Fix: sudo sysctl -w net.ipv4.ip_forward=1"); + println!(" Persist: echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/99-incus.conf"); + } + } + + // 3. Check iptables FORWARD chain for incusbr0 rules + let fwd_rules = run_cmd("iptables", &["-S", "FORWARD"]).await; + let has_forward_rule = match &fwd_rules { + Ok(r) => r.stdout.contains("incusbr0") && r.stdout.contains("ACCEPT"), + Err(_) => false, + }; + + if has_forward_rule { + println!(" iptables FORWARD: \x1b[32mincusbr0 allowed\x1b[0m"); + } else { + // Check FORWARD policy + let policy_drop = match &fwd_rules { + Ok(r) => r.stdout.contains("-P FORWARD DROP"), + Err(_) => false, + }; + if policy_drop { + println!(" iptables FORWARD: \x1b[31mDROP policy, no incusbr0 rule\x1b[0m"); + println!(" VM traffic is being blocked by the firewall."); + println!(" Fix:"); + println!(" sudo iptables -I FORWARD -i incusbr0 -j ACCEPT"); + println!(" sudo iptables -I FORWARD -o incusbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT"); + } else { + println!(" iptables FORWARD: \x1b[32mACCEPT policy\x1b[0m"); + } + } + + // 4. Check NAT masquerade for Incus subnet + let nat_rules = run_cmd("iptables", &["-t", "nat", "-S", "POSTROUTING"]).await; + let has_masq = match &nat_rules { + Ok(r) => r.stdout.contains("incusbr0") || r.stdout.contains("10.195.64"), + Err(_) => false, + }; + + if has_masq { + println!(" iptables NAT: \x1b[32mmasquerade configured\x1b[0m"); + } else { + println!(" iptables NAT: \x1b[33mno masquerade for Incus subnet\x1b[0m"); + println!(" Fix: sudo iptables -t nat -A POSTROUTING -s 10.195.64.0/24 ! -o incusbr0 -j MASQUERADE"); + } + + // 5. Quick connectivity test if any running VM exists + let list = run_cmd("incus", &["list", "devbox-", "--format", "json"]).await; + if let Ok(r) = list { + if let Ok(arr) = serde_json::from_str::>(&r.stdout) { + for v in &arr { + if v["status"].as_str() == Some("Running") { + let vm_name = v["name"].as_str().unwrap_or(""); + if !vm_name.is_empty() { + let ping = run_cmd( + "incus", + &["exec", vm_name, "--", "ping", "-c", "1", "-W", "3", "8.8.8.8"], + ) + .await; + match ping { + Ok(p) if p.exit_code == 0 => { + println!( + " VM connectivity ({vm_name}): \x1b[32mok\x1b[0m" + ); + } + _ => { + println!( + " VM connectivity ({vm_name}): \x1b[31mno internet\x1b[0m" + ); + } + } + break; // Only test one VM + } + } + } + } + } +} + /// Check if a binary is available. If missing, print install instructions. /// Returns true if found. fn check_binary_with_install(label: &str, name: &str, install_hint: &str) -> bool { diff --git a/src/runtime/incus.rs b/src/runtime/incus.rs index 226bc68..d35b77a 100644 --- a/src/runtime/incus.rs +++ b/src/runtime/incus.rs @@ -71,6 +71,33 @@ impl IncusRuntime { println!("Image '{alias}' imported successfully."); Ok(()) } + /// Detect the UID of the first non-root user in the VM. + async fn detect_vm_uid(vm: &str) -> Option { + // Find first user with UID >= 1000 (standard non-system user) + let result = run_cmd( + "incus", + &["exec", vm, "--", "bash", "-c", + "awk -F: '$3 >= 1000 && $3 < 65534 { print $3; exit }' /etc/passwd"], + ).await.ok()?; + let uid = result.stdout.trim().to_string(); + if uid.is_empty() { None } else { Some(uid) } + } + + /// Detect the HOME directory for a given UID in the VM. + async fn detect_vm_home(vm: &str, uid: &str) -> String { + let result = run_cmd( + "incus", + &["exec", vm, "--", "bash", "-c", + &format!("getent passwd {uid} | cut -d: -f6")], + ).await; + match result { + Ok(r) if !r.stdout.trim().is_empty() => { + format!("HOME={}", r.stdout.trim()) + } + _ => format!("HOME=/home/dev"), + } + } + /// Wait for the Incus VM agent to become ready (up to 120 seconds). /// The agent starts after the guest OS boots and runs incus-agent. async fn wait_for_agent(vm: &str) -> Result<()> { @@ -132,12 +159,13 @@ impl Runtime for IncusRuntime { launch_args.push(&cpu_str); } + // Default to 4GiB memory for Incus VMs — NixOS rebuild needs 2-4GB + // for evaluating the full module system. The default Incus 1GB is too little. let mem_str; - if !opts.memory.is_empty() { - mem_str = format!("limits.memory={}", opts.memory); - launch_args.push("-c"); - launch_args.push(&mem_str); - } + let memory = if opts.memory.is_empty() { "4GiB" } else { &opts.memory }; + mem_str = format!("limits.memory={memory}"); + launch_args.push("-c"); + launch_args.push(&mem_str); run_ok("incus", &launch_args).await?; @@ -200,6 +228,32 @@ impl Runtime for IncusRuntime { } } + /// Execute an interactive command as the non-root user. + /// Incus exec defaults to root, so we detect the first UID >= 1000 + /// and set --user, HOME, and CWD for proper user sessions. + async fn exec_as_user(&self, name: &str, cmd: &[&str]) -> Result { + let vm = Self::vm_name(name); + let uid_str = Self::detect_vm_uid(&vm).await.unwrap_or("1000".to_string()); + let home_env = Self::detect_vm_home(&vm, &uid_str).await; + + let mut args = vec![ + "exec".to_string(), + vm, + "--user".to_string(), + uid_str, + "--cwd".to_string(), + "/workspace".to_string(), + "--env".to_string(), + home_env, + "--".to_string(), + ]; + for c in cmd { + args.push(c.to_string()); + } + let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + run_interactive("incus", &arg_refs).await + } + async fn destroy(&self, name: &str) -> Result<()> { let vm = Self::vm_name(name); // Stop first (ignore errors if already stopped) diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index e31ce2e..25f3538 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -126,4 +126,11 @@ pub trait Runtime: Send + Sync { /// Update mount points for an existing sandbox. /// Stops the VM, updates mounts in the config, and restarts. async fn update_mounts(&self, name: &str, mounts: &[Mount]) -> Result<()>; + + /// Execute an interactive command as the non-root user. + /// Used for shell attach — defaults to exec_cmd with interactive=true. + /// Runtimes like Incus override this to set --user, HOME, and CWD. + async fn exec_as_user(&self, name: &str, cmd: &[&str]) -> Result { + self.exec_cmd(name, cmd, true).await + } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 8404ad9..377e672 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -237,7 +237,7 @@ impl SandboxManager { if layout == "plain" { println!("Attaching to sandbox '{name}'..."); let shell = Self::probe_shell(runtime.as_ref(), name).await; - runtime.exec_cmd(name, &[&shell, "-l"], true).await?; + runtime.exec_as_user(name, &[&shell, "-l"]).await?; return Ok(()); } @@ -249,7 +249,7 @@ impl SandboxManager { // No Zellij — fall back to raw shell println!("Attaching to sandbox '{name}'..."); let shell = Self::probe_shell(runtime.as_ref(), name).await; - runtime.exec_cmd(name, &[&shell, "-l"], true).await?; + runtime.exec_as_user(name, &[&shell, "-l"]).await?; return Ok(()); } @@ -317,7 +317,7 @@ impl SandboxManager { // Reattach to existing live session println!("Reattaching to sandbox '{name}'..."); runtime - .exec_cmd(name, &["zellij", "attach", &session_name], true) + .exec_as_user(name, &["zellij", "attach", &session_name]) .await?; } else { // Create new named session with layout. @@ -336,10 +336,9 @@ impl SandboxManager { println!("Attaching to sandbox '{name}' (layout: {effective_layout})..."); runtime - .exec_cmd( + .exec_as_user( name, &["zellij", "--config", &config_path, "--layout", &layout_path], - true, ) .await?; } diff --git a/src/sandbox/provision.rs b/src/sandbox/provision.rs index e756cc6..71a2bcd 100644 --- a/src/sandbox/provision.rs +++ b/src/sandbox/provision.rs @@ -236,32 +236,46 @@ async fn provision_nixos( ) -> Result<()> { let username = whoami(); + // 0. Wait for network connectivity (DNS resolution can lag behind the agent) + wait_for_network(runtime, name).await?; + // 1. Create directory structure println!("Setting up NixOS configuration..."); + let mkdir_cmd = format!( + "{NIXOS_PATH_PREFIX}sudo mkdir -p /etc/devbox/sets /etc/devbox/help" + ); runtime - .exec_cmd( - name, - &[ - "sudo", - "mkdir", - "-p", - "/etc/devbox/sets", - "/etc/devbox/help", - ], - false, - ) + .exec_cmd(name, &["bash", "-c", &mkdir_cmd], false) .await?; - // 2. Generate base NixOS config if it doesn't exist + // 2. Ensure NixOS channel is available (images:nixos/* may not have it) + // nixos-rebuild needs `` in NIX_PATH, which comes from + // the nixos channel. If the channel isn't set up, add and update it. + ensure_nixos_channel(runtime, name).await?; + + // 3. Generate base NixOS config if it doesn't exist // NixOS Lima images ship with an empty /etc/nixos/ — we need to // run nixos-generate-config to create the hardware and base configs. ensure_nixos_config(runtime, name).await?; - // 3. Push devbox-state.toml (includes mount_mode for overlay setup) + // 4. Ensure user home directory exists (the user may not exist yet on + // fresh images:nixos/* images — nixos-rebuild will create it via + // devbox-module.nix, but we need the homedir for writing config files + // before rebuild. We create it manually and let NixOS fix ownership later.) + let home_dir = format!("/home/{username}"); + run_in_vm( + runtime, + name, + &format!("sudo mkdir -p {home_dir} && sudo chown $(id -u {username} 2>/dev/null || echo 1000):users {home_dir} 2>/dev/null; true"), + false, + ) + .await?; + + // 5. Push devbox-state.toml (includes mount_mode for overlay setup) let state_toml = generate_state_toml(sets, languages, &username, mount_mode); write_file_to_vm(runtime, name, "/etc/devbox/devbox-state.toml", &state_toml).await?; - // 4. Push devbox-module.nix + // 6. Push devbox-module.nix write_file_to_vm( runtime, name, @@ -270,33 +284,86 @@ async fn provision_nixos( ) .await?; - // 5. Push all set .nix files + // 7. Push all set .nix files for (filename, content) in NIX_SET_FILES { let path = format!("/etc/devbox/sets/{filename}"); write_file_to_vm(runtime, name, &path, content).await?; } - // 6. Run nixos-rebuild switch (interactive so user sees progress) - // NixOS Lima images use flake-based NIX_PATH (nixpkgs=flake:nixpkgs) - // which doesn't include nixos-config. We must set it explicitly. + // 8. Run nixos-rebuild switch (interactive so user sees progress) + // We must set NIX_PATH explicitly because: + // - incus exec doesn't source /etc/profile (no login shell) + // - images:nixos/* may not have channels in the default NIX_PATH + // - After nix-channel --update, nixpkgs lives at the channel profile path println!("Installing packages via nixos-rebuild (this may take a few minutes)..."); - let rebuild_cmd = concat!( - "export NIX_PATH=\"nixos-config=/etc/nixos/configuration.nix:$NIX_PATH\" && ", - "export NIXPKGS_ALLOW_UNFREE=1 && ", - "nixos-rebuild switch" + let rebuild_cmd = format!( + "{NIXOS_PATH_PREFIX}\ + export NIX_PATH=\"nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos:\ + nixos-config=/etc/nixos/configuration.nix:\ + /nix/var/nix/profiles/per-user/root/channels\" && \ + export NIXPKGS_ALLOW_UNFREE=1 && \ + sudo -E nixos-rebuild switch" ); let result = runtime - .exec_cmd(name, &["sudo", "bash", "-c", rebuild_cmd], true) - .await?; + .exec_cmd(name, &["bash", "-c", &rebuild_cmd], true) + .await; - if result.exit_code != 0 { - eprintln!("Warning: nixos-rebuild failed (exit {})", result.exit_code); - eprintln!("You can retry with `devbox exec --name {name} -- sudo nixos-rebuild switch`"); - } else { - println!("NixOS rebuild complete."); + // nixos-rebuild switch stops incus-agent during activation, which + // drops the websocket connection (exit 255). This is expected — + // the agent restarts automatically with the new system config. + // We need to wait for it to come back before continuing. + let rebuild_ok = match &result { + Ok(r) if r.exit_code == 0 => true, + Ok(r) if r.exit_code == 255 => { + // Likely the websocket dropped during activation — probably succeeded + println!("Connection lost during system activation (expected)."); + true + } + Ok(r) => { + eprintln!("Warning: nixos-rebuild failed (exit {})", r.exit_code); + eprintln!("You can retry with `devbox exec --name {name} -- sudo nixos-rebuild switch`"); + false + } + Err(e) => { + eprintln!("Warning: nixos-rebuild error: {e}"); + false + } + }; + + if rebuild_ok { + // Wait for the VM agent to come back after system activation. + // nixos-rebuild switch restarts incus-agent as part of the new config. + println!("Waiting for VM agent to restart after system activation..."); + // Small delay to let the old agent fully stop + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + + let vm = format!("devbox-{name}"); + let max_attempts = 40; // 40 * 3s = 120s + let mut agent_ready = false; + for i in 0..max_attempts { + let check = crate::runtime::cmd::run_cmd( + "incus", + &["exec", &vm, "--", "echo", "ready"], + ) + .await; + if let Ok(r) = check { + if r.exit_code == 0 && r.stdout.trim() == "ready" { + println!("NixOS rebuild complete. VM agent is ready."); + agent_ready = true; + break; + } + } + if i > 0 && i % 10 == 0 { + println!(" Still waiting for VM agent... ({}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + if !agent_ready { + eprintln!("Warning: VM agent did not come back after nixos-rebuild. Continuing anyway..."); + } } - // 8. Set up user shell (zshrc with PATH, aliases, etc.) + // 9. Set up user shell (zshrc with PATH, aliases, etc.) setup_nixos_shell(runtime, name).await?; // 9. Install latest claude-code (nixpkgs version lags behind) @@ -559,9 +626,9 @@ export DEVBOX_RUNTIME="${DEVBOX_RUNTIME:-unknown}" "#; write_file_to_vm(runtime, name, &zshrc_path, zshrc).await?; - let chown_cmd = format!("chown {username}:users {zshrc_path}"); + let chown_cmd = format!("{NIXOS_PATH_PREFIX}sudo chown {username}:users {zshrc_path}"); runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await?; } @@ -576,9 +643,9 @@ export DEVBOX_RUNTIME="${DEVBOX_RUNTIME:-unknown}" export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.claude/bin:$PATH" "#; write_file_to_vm(runtime, name, &profile_path, profile).await?; - let chown_cmd = format!("chown {username}:users {profile_path}"); + let chown_cmd = format!("{NIXOS_PATH_PREFIX}sudo chown {username}:users {profile_path}"); runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await?; } @@ -606,10 +673,8 @@ async fn setup_git_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let vm_path = format!("/home/{username}/.gitconfig"); write_file_to_vm(runtime, name, &vm_path, &content).await?; - let chown_cmd = format!("chown {username}:users {vm_path}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + let chown_cmd = format!("sudo chown {username}:users {vm_path}"); + run_in_vm(runtime, name, &chown_cmd, false).await?; println!("Synced host git config to VM."); Ok(()) @@ -767,18 +832,14 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> // Ensure parent directory exists with correct ownership let vm_parent = vm_path.rsplit_once('/').map(|(p, _)| p).unwrap_or(&vm_path); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", vm_parent], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {vm_parent}"), false).await?; write_file_to_vm(runtime, name, &vm_path, &content).await?; println!(" copied: ~/{host_suffix} → {vm_path}"); // Set restrictive permissions for credential/auth files if vm_suffix.contains("credential") || vm_suffix.contains("auth") { - runtime - .exec_cmd(name, &["sudo", "chmod", "600", &vm_path], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chmod 600 {vm_path}"), false).await?; } copied_any = true; } @@ -807,11 +868,9 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> // Fix ownership for all copied files let chown_cmd = format!( - "chown -R {username}:users {vm_home}/.claude {vm_home}/.config {vm_home}/.codex {vm_home}/.devbox-ai-env 2>/dev/null; true" + "sudo chown -R {username}:users {vm_home}/.claude {vm_home}/.config {vm_home}/.codex {vm_home}/.devbox-ai-env 2>/dev/null; true" ); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &chown_cmd, false).await?; } if copied_any { @@ -822,15 +881,10 @@ async fn setup_ai_tool_configs(runtime: &dyn Runtime, name: &str) -> Result<()> let has_aichat_config = home.join(".config/aichat/config.yaml").exists(); if !has_aichat_config && let Some(config) = generate_aichat_config_from_credentials(&home) { let config_dir = format!("{vm_home}/.config/aichat"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &config_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {config_dir}"), false).await?; let config_path = format!("{config_dir}/config.yaml"); write_file_to_vm(runtime, name, &config_path, &config).await?; - let chown_cmd = format!("chown -R {username}:users {config_dir}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users {config_dir}"), false).await?; println!("Generated aichat config from detected AI tool credentials."); } @@ -898,6 +952,26 @@ fn generate_aichat_config_from_credentials(home: &std::path::Path) -> Option Result { + let full_cmd = format!("{NIXOS_PATH_PREFIX}{cmd}"); + runtime.exec_cmd(name, &["bash", "-c", &full_cmd], interactive).await +} + /// Write a file into the VM using base64-encoded content via exec_cmd. async fn write_file_to_vm( runtime: &dyn Runtime, @@ -907,7 +981,7 @@ async fn write_file_to_vm( ) -> Result<()> { use base64::Engine; let encoded = base64::engine::general_purpose::STANDARD.encode(content.as_bytes()); - let cmd = format!("echo '{encoded}' | base64 -d | sudo tee {path} > /dev/null"); + let cmd = format!("{NIXOS_PATH_PREFIX}echo '{encoded}' | base64 -d | sudo tee {path} > /dev/null"); let result = runtime.exec_cmd(name, &["bash", "-c", &cmd], false).await?; if result.exit_code != 0 { eprintln!("Warning: failed to write {path}: {}", result.stderr.trim()); @@ -915,6 +989,114 @@ async fn write_file_to_vm( Ok(()) } +/// Wait for network connectivity inside the VM. +/// +/// On freshly booted Incus VMs, the network (especially DNS) may not be ready +/// even after the agent responds. We first wait for basic IP connectivity +/// (ping), then check DNS resolution. If basic connectivity never comes up, +/// we bail early with actionable diagnostics instead of letting every +/// subsequent download time out. +async fn wait_for_network(runtime: &dyn Runtime, name: &str) -> Result<()> { + // Phase 1: Wait for basic IP connectivity (ping 8.8.8.8) + // This distinguishes "network not ready yet" from "no route / firewall blocks" + let ping_attempts = 10; // 10 * 3s = 30s + let mut got_ping = false; + for i in 0..ping_attempts { + let result = run_in_vm( + runtime, + name, + "ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1 && echo ok", + false, + ) + .await?; + if result.exit_code == 0 && result.stdout.trim() == "ok" { + got_ping = true; + break; + } + if i == 0 { + print!("Waiting for network connectivity..."); + } else if i % 5 == 0 { + print!(" ({}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + + if !got_ping { + println!(); + eprintln!("\x1b[31mError: VM has no network connectivity.\x1b[0m"); + eprintln!("The VM cannot reach the internet. This is usually caused by"); + eprintln!("missing iptables FORWARD rules for the Incus bridge.\n"); + eprintln!("Quick fix (run on the host):"); + eprintln!(" sudo iptables -I FORWARD -i incusbr0 -j ACCEPT"); + eprintln!(" sudo iptables -I FORWARD -o incusbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT"); + eprintln!(" sudo iptables -t nat -A POSTROUTING -s 10.195.64.0/24 ! -o incusbr0 -j MASQUERADE\n"); + eprintln!("Run `devbox doctor` for full network diagnostics."); + anyhow::bail!("VM network connectivity check failed — cannot provision without internet access"); + } + + // Phase 2: Wait for DNS resolution + let dns_attempts = 10; // 10 * 3s = 30s + for i in 0..dns_attempts { + let result = run_in_vm( + runtime, + name, + "getent hosts cache.nixos.org >/dev/null 2>&1 && echo ok", + false, + ) + .await?; + if result.exit_code == 0 && result.stdout.trim() == "ok" { + println!(" ready."); + return Ok(()); + } + if i % 5 == 0 { + print!(" (DNS {}s)", i * 3); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + println!(); + eprintln!("Warning: DNS resolution not working yet — provisioning will continue but downloads may fail."); + Ok(()) +} + +/// Ensure the NixOS channel is available so `nixos-rebuild` can find ``. +/// +/// The `images:nixos/*` Incus images may not have channels configured, causing +/// `nixos-rebuild` to fail with "file 'nixpkgs/nixos' was not found in the Nix search path". +/// We check if the nixos channel exists and add it if missing. +async fn ensure_nixos_channel(runtime: &dyn Runtime, name: &str) -> Result<()> { + // Check if the nixos channel is already available for root. + // We check the channel profile path directly since NIX_PATH may not be set + // in the non-login incus exec shell. + let check = run_in_vm( + runtime, + name, + "test -d /nix/var/nix/profiles/per-user/root/channels/nixos && echo found", + false, + ) + .await?; + + if check.stdout.trim() == "found" { + return Ok(()); + } + + println!("Setting up NixOS channel (required for nixos-rebuild)..."); + let channel_cmd = concat!( + "sudo nix-channel --add https://nixos.org/channels/nixos-25.05 nixos && ", + "sudo nix-channel --update" + ); + let result = run_in_vm(runtime, name, channel_cmd, true).await?; + if result.exit_code != 0 { + eprintln!( + "Warning: failed to set up NixOS channel: {}", + result.stderr.trim() + ); + } else { + println!("NixOS channel configured."); + } + + Ok(()) +} + /// Ensure /etc/nixos/configuration.nix and hardware-configuration.nix exist. /// /// NixOS Lima images ship with an empty /etc/nixos/ directory. @@ -933,8 +1115,9 @@ async fn ensure_nixos_config(runtime: &dyn Runtime, name: &str) -> Result<()> { if hw_check.exit_code != 0 { println!(" Generating hardware configuration..."); + let gen_cmd = format!("{NIXOS_PATH_PREFIX}sudo nixos-generate-config"); let result = runtime - .exec_cmd(name, &["sudo", "nixos-generate-config"], false) + .exec_cmd(name, &["bash", "-c", &gen_cmd], false) .await?; if result.exit_code != 0 { eprintln!( @@ -1009,20 +1192,13 @@ async fn copy_devbox_to_vm(runtime: &dyn Runtime, name: &str) -> Result<()> { if let Ok(r) = result && r.exit_code == 0 { - let _ = runtime - .exec_cmd( - name, - &[ - "sudo", - "install", - "-m", - "755", - "/tmp/devbox", - "/usr/local/bin/devbox", - ], - false, - ) - .await; + let _ = run_in_vm( + runtime, + name, + "sudo install -m 755 /tmp/devbox /usr/local/bin/devbox", + false, + ) + .await; let _ = runtime.exec_cmd(name, &["rm", "/tmp/devbox"], false).await; } } @@ -1035,9 +1211,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let config_dir = format!("/home/{username}/.config/yazi"); // Create config directory - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &config_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {config_dir}"), false).await?; // Write all yazi config files let files: &[(&str, &str)] = &[ @@ -1053,9 +1227,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { // Write glow previewer plugin let plugin_dir = format!("{config_dir}/plugins/glow.yazi"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &plugin_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {plugin_dir}"), false).await?; write_file_to_vm( runtime, name, @@ -1065,10 +1237,7 @@ async fn setup_yazi_config(runtime: &dyn Runtime, name: &str) -> Result<()> { .await?; // Fix ownership - let chown_cmd = format!("chown -R {username}:users /home/{username}/.config/yazi"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users /home/{username}/.config/yazi"), false).await?; Ok(()) } @@ -1080,9 +1249,7 @@ async fn setup_aichat_config(runtime: &dyn Runtime, name: &str) -> Result<()> { let config_dir = format!("/home/{username}/.config/aichat"); let roles_dir = format!("{config_dir}/roles"); - runtime - .exec_cmd(name, &["sudo", "mkdir", "-p", &roles_dir], false) - .await?; + run_in_vm(runtime, name, &format!("sudo mkdir -p {roles_dir}"), false).await?; // Legacy format (older aichat versions) write_file_to_vm( @@ -1102,10 +1269,7 @@ async fn setup_aichat_config(runtime: &dyn Runtime, name: &str) -> Result<()> { write_file_to_vm(runtime, name, &format!("{roles_dir}/{filename}"), content).await?; } - let chown_cmd = format!("chown -R {username}:users {config_dir}"); - runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) - .await?; + run_in_vm(runtime, name, &format!("sudo chown -R {username}:users {config_dir}"), false).await?; Ok(()) } @@ -1119,13 +1283,7 @@ async fn setup_management_script(runtime: &dyn Runtime, name: &str) -> Result<() MANAGEMENT_SCRIPT, ) .await?; - runtime - .exec_cmd( - name, - &["sudo", "chmod", "+x", "/etc/devbox/management.sh"], - false, - ) - .await?; + run_in_vm(runtime, name, "sudo chmod +x /etc/devbox/management.sh", false).await?; Ok(()) } @@ -1188,10 +1346,10 @@ async fn install_latest_claude_code(runtime: &dyn Runtime, name: &str) { } // Fix ownership let chown_cmd = format!( - "chown {username}:users /home/{username}/.zshrc /home/{username}/.profile 2>/dev/null; true" + "{NIXOS_PATH_PREFIX}sudo chown {username}:users /home/{username}/.zshrc /home/{username}/.profile 2>/dev/null; true" ); let _ = runtime - .exec_cmd(name, &["sudo", "bash", "-c", &chown_cmd], false) + .exec_cmd(name, &["bash", "-c", &chown_cmd], false) .await; }