fix(miner): reclaim worktree slots by lease age, not cross-container PID liveness

packages/loopover-miner/lib/worktree-allocator.d.ts

@@ -5,20 +5,25 @@ export type WorktreeAllocation = {
   repoFullName: string | null;
   status: "free" | "active";
   ownerPid: number | null;
+  ownerHost: string | null;
   allocatedAt: string | null;
 };
 
 export type WorktreeAllocator = {
   dbPath: string;
   worktreeBaseDir: string;
   maxConcurrency: number;
+  maxLeaseMs: number;
   processPid: number;
+  hostId: string;
   acquire(attemptId: string, repoFullName: string): WorktreeAllocation;
   release(attemptId: string): WorktreeAllocation | null;
   listSlots(): WorktreeAllocation[];
   close(): void;
 };
 
+export const DEFAULT_MAX_LEASE_MS: number;
+
 export function resolveWorktreeAllocatorDbPath(env?: Record<string, string | undefined>): string;
 
 export function resolveWorktreeBaseDir(env?: Record<string, string | undefined>): string;
@@ -29,7 +34,10 @@ export function openWorktreeAllocator(options?: {
   dbPath?: string;
   worktreeBaseDir?: string;
   maxConcurrency?: number;
+  maxLeaseMs?: number;
   processPid?: number;
+  hostId?: string;
+  nowMs?: number;
 }): WorktreeAllocator;
 
 export function acquireWorktree(attemptId: string, repoFullName: string): WorktreeAllocation;

packages/loopover-miner/lib/worktree-allocator.js

@@ -2,7 +2,7 @@
 // a filesystem directory, not a store DB path, and is deliberately out of this migration's scope. Only the DB
 // handle's own mkdir/chmod moved into openLocalStoreDb.
 import { mkdirSync } from "node:fs";
-import { homedir } from "node:os";
+import { homedir, hostname } from "node:os";
 import { join } from "node:path";
 import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";
 
@@ -20,6 +20,17 @@ const defaultWorktreeDirName = "worktrees";
 const defaultMaxConcurrency = 2;
 let defaultWorktreeAllocator = null;
 
+// Age-based orphan reclaim (#7085). Fleet mode (see DEPLOYMENT.md) runs multiple separate CONTAINERS over one
+// shared data volume, each with its own PID namespace, so a stored `owner_pid` is meaningless the moment a
+// different container opens this store — `isProcessAlive` checks the CALLING process's own namespace, not the
+// one that recorded the pid. So we mirror the age-based convention every sibling shared-lease store already uses
+// (portfolio-queue-expiry.js's DEFAULT_MAX_LEASE_MS / sweepStuckItems, claim-ledger's DEFAULT_MAX_CLAIM_AGE_MS):
+// reclaim any `active` slot older than this regardless of what the pid check reports. Kept well above
+// portfolio-queue-expiry's 30-minute floor because a single worktree lease spans a whole coding attempt (clone +
+// agent run + push), which can legitimately run for hours; the same-host `isProcessAlive` fast path still frees a
+// crashed local owner immediately, so this age fallback only ever governs the cross-container case.
+export const DEFAULT_MAX_LEASE_MS = 6 * 60 * 60 * 1000;
+
 export function resolveWorktreeAllocatorDbPath(env = process.env) {
   return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_WORKTREE_ALLOCATOR_DB", env);
 }
@@ -57,6 +68,18 @@ function normalizeMaxConcurrency(value) {
   return value;
 }
 
+function normalizeMaxLeaseMs(value) {
+  if (value === undefined || value === null) return DEFAULT_MAX_LEASE_MS;
+  if (!Number.isFinite(value) || value < 0) throw new Error("invalid_max_lease_ms");
+  return value;
+}
+
+function normalizeHostId(value) {
+  if (value === undefined || value === null) return hostname();
+  if (typeof value !== "string" || !value.trim()) throw new Error("invalid_host_id");
+  return value.trim();
+}
+
 function normalizeRepoFullName(repoFullName) {
   if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name");
   const [owner, repo, extra] = repoFullName.trim().split("/");
@@ -92,6 +115,7 @@ function rowToAllocation(row) {
     repoFullName: row.repo_full_name,
     status: row.status,
     ownerPid: row.owner_pid,
+    ownerHost: row.owner_host ?? null,
     allocatedAt: row.allocated_at,
   };
 }
@@ -105,9 +129,23 @@ function ensureSlotTable(db) {
       repo_full_name TEXT,
       status TEXT NOT NULL CHECK (status IN ('free', 'active')),
       owner_pid INTEGER,
+      owner_host TEXT,
       allocated_at TEXT
     )
   `);
+  ensureOwnerHostColumn(db);
+}
+
+// Add the owner_host column (#7085) to an on-disk file created before it existed. `CREATE TABLE IF NOT EXISTS`
+// above is a no-op against an already-existing table, so a pre-#7085 file needs this explicit ALTER — guarded by
+// a presence check (same technique as attempt-log.js's ensureOutcomeColumns). A migrated row keeps owner_host
+// NULL until its owner re-acquires, so the age-based reclaim (not the same-host pid fast path) governs it.
+function ensureOwnerHostColumn(db) {
+  const hasOwnerHost = db
+    .prepare("PRAGMA table_info(worktree_slots)")
+    .all()
+    .some((column) => column.name === "owner_host");
+  if (!hasOwnerHost) db.exec("ALTER TABLE worktree_slots ADD COLUMN owner_host TEXT");
 }
 
 function ensureSlots(db, worktreeBaseDir, maxConcurrency) {
@@ -123,66 +161,97 @@ function ensureSlots(db, worktreeBaseDir, maxConcurrency) {
   }
 }
 
-function reclaimOrphanedAllocations(db) {
+function allocationAgeMs(allocatedAt, nowMs) {
+  const allocatedMs = Date.parse(allocatedAt);
+  if (!Number.isFinite(allocatedMs)) return null;
+  return nowMs - allocatedMs;
+}
+
+/**
+ * Decide whether an `active` slot is orphaned and should be reclaimed. Two independent signals:
+ * - Age (container-agnostic): a slot whose `allocated_at` is older than `maxLeaseMs` is reclaimed regardless of
+ *   what `isProcessAlive` reports, guaranteeing eventual reclaim even when a cross-container caller observes the
+ *   owner's pid in the wrong PID namespace. This is the only signal that is sound across fleet mode's separate
+ *   containers, so it must never be gated behind the pid check.
+ * - Same-host pid liveness (fast path): only when the slot was leased by a process on THIS host (`owner_host`
+ *   matches) is `isProcessAlive` a meaningful signal — a confirmed-dead (or missing) local owner frees its slot
+ *   immediately without waiting out the lease. A foreign `owner_host` is never trusted for the pid check.
+ */
+function isSlotOrphaned(row, nowMs, maxLeaseMs, hostId) {
+  const ageMs = allocationAgeMs(row.allocated_at, nowMs);
+  if (ageMs !== null && ageMs > maxLeaseMs) return true;
+  if (row.owner_host !== null && row.owner_host === hostId) {
+    return row.owner_pid === null || !isProcessAlive(row.owner_pid);
+  }
+  return false;
+}
+
+function reclaimOrphanedAllocations(db, nowMs, maxLeaseMs, hostId) {
   const orphans = db
-    .prepare("SELECT slot_index, owner_pid FROM worktree_slots WHERE status = 'active'")
+    .prepare("SELECT slot_index, owner_pid, owner_host, allocated_at FROM worktree_slots WHERE status = 'active'")
     .all();
   const reclaim = db.prepare(`
     UPDATE worktree_slots
-    SET status = 'free', attempt_id = NULL, repo_full_name = NULL, owner_pid = NULL, allocated_at = NULL
+    SET status = 'free', attempt_id = NULL, repo_full_name = NULL, owner_pid = NULL, owner_host = NULL, allocated_at = NULL
     WHERE slot_index = ?
   `);
   for (const row of orphans) {
-    if (row.owner_pid !== null && isProcessAlive(row.owner_pid)) continue;
-    reclaim.run(row.slot_index);
+    if (isSlotOrphaned(row, nowMs, maxLeaseMs, hostId)) reclaim.run(row.slot_index);
   }
 }
 
 /**
- * Opens the local worktree allocator store. Reclaims orphaned active slots from dead owner processes on startup.
+ * Opens the local worktree allocator store. On startup reclaims orphaned active slots — any slot past its
+ * `maxLeaseMs` age (the container-agnostic guarantee for fleet mode's shared store), plus, as a same-host fast
+ * path, any slot whose owner pid is confirmed dead in THIS host's PID namespace.
  */
 export function openWorktreeAllocator(options = {}) {
   const resolvedPath = normalizeDbPath(options.dbPath);
   const worktreeBaseDir = normalizeWorktreeBaseDir(options.worktreeBaseDir);
   const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
+  const maxLeaseMs = normalizeMaxLeaseMs(options.maxLeaseMs);
+  const hostId = normalizeHostId(options.hostId);
   const processPid = Number.isInteger(options.processPid) ? options.processPid : process.pid;
+  const nowMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now();
 
   const db = openLocalStoreDb(resolvedPath);
   ensureSlotTable(db);
   ensureSlots(db, worktreeBaseDir, maxConcurrency);
-  reclaimOrphanedAllocations(db);
+  reclaimOrphanedAllocations(db, nowMs, maxLeaseMs, hostId);
 
   const getByAttempt = db.prepare(
-    "SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, allocated_at FROM worktree_slots WHERE attempt_id = ?",
+    "SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, owner_host, allocated_at FROM worktree_slots WHERE attempt_id = ?",
   );
   const countActive = db.prepare("SELECT COUNT(*) AS count FROM worktree_slots WHERE status = 'active'");
   const selectFreeSlot = db.prepare(`
-    SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, allocated_at
+    SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, owner_host, allocated_at
     FROM worktree_slots
     WHERE status = 'free'
     ORDER BY slot_index
     LIMIT 1
   `);
   const markActive = db.prepare(`
     UPDATE worktree_slots
-    SET status = 'active', attempt_id = ?, repo_full_name = ?, owner_pid = ?, allocated_at = ?
+    SET status = 'active', attempt_id = ?, repo_full_name = ?, owner_pid = ?, owner_host = ?, allocated_at = ?
     WHERE slot_index = ?
   `);
   const releaseByAttempt = db.prepare(`
     UPDATE worktree_slots
-    SET status = 'free', attempt_id = NULL, repo_full_name = NULL, owner_pid = NULL, allocated_at = NULL
+    SET status = 'free', attempt_id = NULL, repo_full_name = NULL, owner_pid = NULL, owner_host = NULL, allocated_at = NULL
     WHERE attempt_id = ? AND status = 'active'
-    RETURNING slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, allocated_at
+    RETURNING slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, owner_host, allocated_at
   `);
   const listSlots = db.prepare(
-    "SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, allocated_at FROM worktree_slots ORDER BY slot_index",
+    "SELECT slot_index, worktree_path, attempt_id, repo_full_name, status, owner_pid, owner_host, allocated_at FROM worktree_slots ORDER BY slot_index",
   );
 
   const allocator = {
     dbPath: resolvedPath,
     worktreeBaseDir,
     maxConcurrency,
+    maxLeaseMs,
     processPid,
+    hostId,
     acquire(attemptId, repoFullName) {
       const normalizedAttempt = normalizeAttemptId(attemptId);
       const normalizedRepo = normalizeRepoFullName(repoFullName);
@@ -201,14 +270,15 @@ export function openWorktreeAllocator(options = {}) {
         const slot = selectFreeSlot.get();
         if (!slot) throw new Error("worktree_capacity_exceeded");
         const allocatedAt = new Date().toISOString();
-        markActive.run(normalizedAttempt, normalizedRepo, processPid, allocatedAt, slot.slot_index);
+        markActive.run(normalizedAttempt, normalizedRepo, processPid, hostId, allocatedAt, slot.slot_index);
         db.exec("COMMIT");
         return rowToAllocation({
           ...slot,
           attempt_id: normalizedAttempt,
           repo_full_name: normalizedRepo,
           status: "active",
           owner_pid: processPid,
+          owner_host: hostId,
           allocated_at: allocatedAt,
         });
       } catch (error) {

test/unit/miner-attempt-cli.test.ts

@@ -1085,7 +1085,9 @@ describe("runAttempt (#5132)", () => {
         dbPath: ":memory:",
         worktreeBaseDir: "/tmp/unused",
         maxConcurrency: 1,
+        maxLeaseMs: 6 * 60 * 60 * 1000,
         processPid: process.pid,
+        hostId: "test-host",
         acquire: () => {
           throw new Error("no_free_worktree_slots");
         },