test: fail the daemon leak oracle on a surviving daemon and pin its rules

Review of #1859 found the oracle reported "clean" when the daemon itself
outlived shutdown: daemon pids were excluded from ownership, liveDaemonPids
never reached hasDaemonLeaks, and a live daemon even flipped daemon.json/lock
from stray to expected. stopProcessForTakeover is best-effort void, so only
smoke-daemon-clean independently asserted death.

- a live daemon pid at phase 'after-shutdown' is now itself a leak, and its
  metadata files stay stray; a live daemon remains legitimate at 'after-close'
- split the pure ownership/residue rules into daemon-leak-model.ts and pin them
  with daemon-leak-model.test.ts, using the real ps rows captured during the
  #1109 and #1324 red-proofs (the lanes' daemons own no children, so the fixture
  test is what guards those shapes in CI)
- exempt the managed tools/ install tree before the .tmp rule, so agent-browser's
  own download temporaries are no longer a false LEAK
- report empty directories as residue (an unswept session scaffold leaves no file)
- reuse src/utils/host-process.ts (expandProcessTree, uniquePositivePids) and its
  /bin/ps convention instead of re-deriving them
- assert in smoke-daemon-http on the success path, not in finally, so the settle
  window cannot replace a primary assertion's diagnostic

Refs #1781 #1431

Author: thymikee

test/integration/smoke-daemon-http.test.ts

@@ -66,8 +66,17 @@ test('daemon HTTP transport starts from CLI and accepts a command RPC', async (t
     const unauthorized = await callCommandRpc({ ...info, token: 'wrong-token' }, 'session_list');
     assert.equal(unauthorized.status, 401);
     assert.equal(unauthorized.body.error?.data?.code, 'UNAUTHORIZED');
+    // #1781 B1: the HTTP-mode daemon must exit — leaving nothing it owns and no
+    // unclassified state-dir residue. Asserted on the success path so the
+    // oracle's settle window can never replace a primary assertion's
+    // diagnostic; the `finally` below stays best-effort cleanup.
+    await stopDaemon(info);
+    await assertNoDaemonLeaks({ stateDir, daemonPids: [info.pid], phase: 'after-shutdown' });
   } finally {
-    await stopDaemonForStateDir(stateDir);
+    if (fs.existsSync(path.join(stateDir, 'daemon.json'))) {
+      await stopDaemon(readDaemonInfo(stateDir));
+    }
+    fs.rmSync(stateDir, { recursive: true, force: true });
   }
 });
 
@@ -102,21 +111,11 @@ async function callCommandRpc(
   };
 }
 
-async function stopDaemonForStateDir(stateDir: string): Promise<void> {
-  try {
-    const infoPath = path.join(stateDir, 'daemon.json');
-    if (!fs.existsSync(infoPath)) return;
-    const info = readDaemonInfo(stateDir);
-    if (!Number.isInteger(info.pid) || info.pid <= 0) return;
-    await stopProcessForTakeover(info.pid, {
-      termTimeoutMs: 1500,
-      killTimeoutMs: 1500,
-      expectedStartTime: info.processStartTime,
-    });
-    // #1781 B1: the HTTP-mode daemon must exit without owned processes or
-    // unclassified state-dir residue.
-    await assertNoDaemonLeaks({ stateDir, daemonPids: [info.pid], phase: 'after-shutdown' });
-  } finally {
-    fs.rmSync(stateDir, { recursive: true, force: true });
-  }
+async function stopDaemon(info: DaemonInfo): Promise<void> {
+  if (!Number.isInteger(info.pid) || info.pid <= 0) return;
+  await stopProcessForTakeover(info.pid, {
+    termTimeoutMs: 1500,
+    killTimeoutMs: 1500,
+    expectedStartTime: info.processStartTime,
+  });
 }

test/integration/support/daemon-leak-model.test.ts

@@ -0,0 +1,213 @@
+import { describe, expect, test } from 'vitest';
+import {
+  evaluateDaemonLeaks,
+  formatDaemonLeakReport,
+  hasDaemonLeaks,
+  type DaemonLeakObservation,
+  type DaemonLeakPhase,
+  type HostProcess,
+  type StateEntry,
+} from './daemon-leak-model.ts';
+
+// The lanes that call the oracle run device-free daemons, so the ownership rules
+// would otherwise only ever see an empty process set. These fixtures are the
+// real `ps` shapes captured during the #1109 and #1324 red-proofs (SHAs in the
+// #1781 B1 PR), so a regex or ordering edit that stops catching either leak
+// fails here instead of silently going quiet in CI.
+const STATE_DIR = '/tmp/agent-device-lane-abc';
+const DAEMON_PID = 4340;
+const OBSERVER_PID = 999;
+
+function proc(overrides: Partial<HostProcess> & Pick<HostProcess, 'pid'>): HostProcess {
+  return { ppid: 1, pgid: overrides.pid, command: 'unrelated', env: '', ...overrides };
+}
+
+function observe(overrides: Partial<DaemonLeakObservation> = {}): DaemonLeakObservation {
+  return {
+    stateDir: STATE_DIR,
+    daemonPids: [DAEMON_PID],
+    livePids: [],
+    phase: 'after-shutdown',
+    processes: [],
+    excludedPids: [OBSERVER_PID],
+    stateEntries: [],
+    ...overrides,
+  };
+}
+
+function file(entryPath: string, descriptorLifecycle?: string): StateEntry {
+  return { path: entryPath, kind: 'file', ...(descriptorLifecycle ? { descriptorLifecycle } : {}) };
+}
+
+describe('owned-process rules', () => {
+  // #1324: `simctl io … recordVideo` reparents to launchd (ppid 1) but keeps the
+  // dead daemon's process group, and simctl only finalizes the mp4 on SIGINT.
+  test('flags a recorder orphaned into the dead daemon process group', () => {
+    const recorder = proc({
+      pid: 52420,
+      ppid: 1,
+      pgid: DAEMON_PID,
+      command: '/…/simctl io 416440AE recordVideo /tmp/out.mp4',
+    });
+    const snapshot = evaluateDaemonLeaks(observe({ processes: [recorder] }));
+
+    expect(snapshot.ownedProcesses).toEqual([
+      expect.objectContaining({ pid: 52420, reasons: ['process-group'] }),
+    ]);
+    expect(hasDaemonLeaks(snapshot)).toBe(true);
+  });
+
+  // #1109: the agent-browser daemon setsids away from the daemon's group, so
+  // only the inherited state-dir environment and its argv identify the fleet.
+  test('flags an agent-browser fleet by inherited state dir, including its children', () => {
+    const browserDaemon = proc({
+      pid: 47515,
+      ppid: 1,
+      pgid: 47515,
+      command: `${STATE_DIR}/tools/agent-browser/0.27.1/package/…/agent-browser-darwin-arm64`,
+      env: `HOME=/tmp AGENT_DEVICE_STATE_DIR=${STATE_DIR}`,
+    });
+    const chrome = proc({
+      pid: 47586,
+      ppid: 47515,
+      pgid: 47586,
+      command: 'Google Chrome for Testing',
+    });
+    const renderer = proc({
+      pid: 47953,
+      ppid: 47586,
+      pgid: 47586,
+      command: 'Chrome Helper (Renderer)',
+    });
+    const snapshot = evaluateDaemonLeaks(observe({ processes: [browserDaemon, chrome, renderer] }));
+
+    expect(snapshot.ownedProcesses.map((owned) => owned.pid)).toEqual([47515, 47586, 47953]);
+    expect(snapshot.ownedProcesses[0]?.reasons).toEqual(['state-dir-env', 'state-dir-argv']);
+    // The fleet below the matched root is owned transitively, not by its own argv.
+    expect(snapshot.ownedProcesses[1]?.reasons).toEqual(['descendant']);
+  });
+
+  test('ignores foreign processes, the observer chain, and the daemons themselves', () => {
+    const simulator = proc({ pid: 700, command: '/…/CoreSimulator … SimulatorTrampoline' });
+    const neighbourStateDir = proc({
+      pid: 701,
+      command: `node --state-dir ${STATE_DIR}-other/daemon.ts`,
+      env: `AGENT_DEVICE_STATE_DIR=${STATE_DIR}-other`,
+    });
+    const observer = proc({ pid: OBSERVER_PID, pgid: DAEMON_PID });
+    const daemon = proc({ pid: DAEMON_PID, pgid: DAEMON_PID });
+    const snapshot = evaluateDaemonLeaks(
+      observe({ processes: [simulator, neighbourStateDir, observer, daemon] }),
+    );
+
+    expect(snapshot.ownedProcesses).toEqual([]);
+    expect(hasDaemonLeaks(snapshot)).toBe(false);
+  });
+});
+
+describe('surviving-daemon rule', () => {
+  // stopProcessForTakeover is best-effort void: it returns silently on identity
+  // mismatch, signal failure, or kill timeout, so a daemon can outlive the stop.
+  test('a daemon still alive after shutdown is itself a leak', () => {
+    const snapshot = evaluateDaemonLeaks(
+      observe({ livePids: [DAEMON_PID], stateEntries: [file('daemon.log')] }),
+    );
+
+    expect(snapshot.liveDaemonPids).toEqual([DAEMON_PID]);
+    expect(hasDaemonLeaks(snapshot)).toBe(true);
+    expect(formatDaemonLeakReport(snapshot)).toContain('daemons that outlived shutdown: 1');
+  });
+
+  test('its metadata files stay stray rather than being excused by its own survival', () => {
+    const snapshot = evaluateDaemonLeaks(
+      observe({ livePids: [DAEMON_PID], stateEntries: [file('daemon.json'), file('daemon.lock')] }),
+    );
+
+    expect(snapshot.strayStateEntries).toEqual(['daemon.json', 'daemon.lock']);
+  });
+
+  test('a live daemon is expected while a session merely closed', () => {
+    const snapshot = evaluateDaemonLeaks(
+      observe({
+        phase: 'after-close',
+        livePids: [DAEMON_PID],
+        stateEntries: [file('daemon.json'), file('daemon.lock')],
+      }),
+    );
+
+    expect(hasDaemonLeaks(snapshot)).toBe(false);
+  });
+});
+
+describe('state-dir residue rules', () => {
+  test.each<[string, StateEntry, DaemonLeakPhase, 'expected' | 'stray']>([
+    ['session event log', file('sessions/default/events.ndjson'), 'after-shutdown', 'expected'],
+    ['request diagnostics', file('sessions/d/requests/abc.ndjson'), 'after-shutdown', 'expected'],
+    ['shutdown report', file('daemon-shutdown.json'), 'after-shutdown', 'expected'],
+    ['torn publish temporary', file('device-claims/a.json.55.tmp'), 'after-shutdown', 'stray'],
+    [
+      'managed tool download',
+      file('tools/agent-browser/0.27.1/dl.tmp'),
+      'after-shutdown',
+      'expected',
+    ],
+    [
+      'unswept artifact scaffold',
+      { path: 'sessions/d/artifacts/pending/', kind: 'empty-directory' },
+      'after-shutdown',
+      'stray',
+    ],
+    [
+      'unswept session scaffold',
+      { path: 'sessions/leaked/requests/', kind: 'empty-directory' },
+      'after-shutdown',
+      'stray',
+    ],
+    ['unknown artifact', file('sessions/d/mystery.bin'), 'after-shutdown', 'stray'],
+    [
+      'open capture descriptor',
+      file('sessions/d/screen-recording.resource.json', 'open'),
+      'after-shutdown',
+      'stray',
+    ],
+    [
+      'completed capture descriptor',
+      file('sessions/d/screen-recording.resource.json', 'completed'),
+      'after-shutdown',
+      'expected',
+    ],
+    [
+      'open capture during close',
+      file('sessions/d/screen-recording.resource.json', 'open'),
+      'after-close',
+      'expected',
+    ],
+    ['legacy app-log marker', file('sessions/d/app-log.pid'), 'after-shutdown', 'stray'],
+  ])('%s is %s at %s', (_name, entry, phase, verdict) => {
+    const snapshot = evaluateDaemonLeaks(observe({ phase, stateEntries: [entry] }));
+
+    expect(snapshot.strayStateEntries).toEqual(verdict === 'stray' ? [entry.path] : []);
+  });
+
+  // A managed install tree is third-party output the daemon neither writes nor
+  // owns, so its own temporaries must not be read as our torn publish.
+  test('the managed tools exemption does not leak into daemon-written paths', () => {
+    const snapshot = evaluateDaemonLeaks(
+      observe({ stateEntries: [file('sessions/d/tools/pending.tmp')] }),
+    );
+
+    expect(snapshot.strayStateEntries).toEqual(['sessions/d/tools/pending.tmp']);
+  });
+});
+
+test('a clean shutdown reports no leak', () => {
+  const snapshot = evaluateDaemonLeaks(
+    observe({
+      processes: [proc({ pid: 700, command: 'unrelated' })],
+      stateEntries: [file('daemon.log'), file('sessions/default/events.ndjson')],
+    }),
+  );
+
+  expect(hasDaemonLeaks(snapshot)).toBe(false);
+  expect(formatDaemonLeakReport(snapshot)).toContain('daemon leak oracle: clean (after-shutdown)');
+});