diff --git a/.gitignore b/.gitignore index cd137414..31eec3a4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ node_modules # local collab/scratch notes collab/ .cmux/worktrees/ +results/qa-video/ +results-qa-video-run.log diff --git a/docs/qa-video-harness.md b/docs/qa-video-harness.md new file mode 100644 index 00000000..3afc7895 --- /dev/null +++ b/docs/qa-video-harness.md @@ -0,0 +1,189 @@ +# QA video harness — video ground truth for cmuxlayer claims + +cmuxlayer is normally the only witness to cmuxlayer. Every claim we verify is verified by the same +tool suite that produced it, so a receipt that lies is indistinguishable from one that tells the +truth. This harness gets evidence from **outside** the system under test: it screen-records a live +probe in an isolated cmux window, then has cheap vision sub-agents adjudicate the frames one narrow +question at a time. + +The product of a run is the **contradictions** — places where the tool receipt and the pixels +disagree. No unit test can produce those, because a unit test is inside the same box. + +- Runner: `scripts/qa-video-harness.mjs` +- Decision logic (pure, unit-tested): `scripts/qa-video-lib.mjs` +- Tests: `tests/qa-video-harness.test.ts` +- Reports land in `docs.local/reports/qa-video-.md` + +## What it probes + +Each probe re-runs a repro the fleet actually reported, and every tool receipt is captured verbatim +next to its position on the recording clock. + +| probe | repro | issues | +| --- | --- | --- | +| `busy-send` | `send_to` a BUSY agent — does the text appear in the composer, and does it submit? | #432, #484 | +| `stale-terminal-send` | `send_to` a pane whose registry row is terminal but whose pane is live | #484 | +| `close-agent` | `close_surface(scope:"agent")` — does the pane actually disappear? | #485 | +| `list-closure-flap` | `list_agents` ×3 over ~20s — does `closure` flap for an unchanged agent? | #488 | +| `wait-for-working` | `wait_for` on a working agent — does it block, or claim "already completed"? | #473 | +| `spawn-under-keystrokes` | spawn while keystrokes are injected — does the launcher line recover? | #434, #440 | + +## Running it + +The harness drives a live cmux, opens a window, records the screen and spawns real agents. It is +opt-in and it is loud about it. + +```bash +# 1. Always start here. Trivial probe, no agents, proves the whole pipeline. +CMUX_QA_VIDEO=1 npm run qa:video:dry-run + +# 2. Full run. Runs the dry-run as a preflight first and refuses to continue if it produced no frames. +bun run build # the harness talks to dist/index.js +CMUX_QA_VIDEO=1 npm run qa:video -- --cli cursor --repo cmuxlayer +``` + +Useful flags: `--capture-fps`, `--frame-fps`, `--scale-width`, `--max-frames` (default 750), +`--keep-runs` (default 5), `--keep-window`, `--root `, `--server-command`/`--server-arg`, +`--skip-preflight` (don't). + +Output under `results/qa-video//` (gitignored): + +- `video.mov` — the recording, cropped to the probe window +- `run.json` — every probe step, every tool receipt verbatim, every mark on the recording clock +- `questions.json` — the receipt-free adjudicator payload, with each narrow question and its frames +- `expectations.json` — receipt claims and expected verdicts, held back until report generation +- `frames//f-.jpg` — JPEG frames named with ffmpeg's real filtered-frame PTS + +The harness refuses extraction when the planned questions exceed `--max-frames`, prints the total +artifact size on completion, and automatically retains only the newest `--keep-runs` completed runs +under the default gitignored output path. Custom `--root` paths are never pruned automatically. To +remove older default-output artifacts by age instead: + +```bash +find results/qa-video -mindepth 1 -maxdepth 1 -type d -mtime +7 -print -exec rm -rf -- {} + +``` + +## Adjudicating + +Sub-agents are **Sonnet** — cheap vision, high frame density — and they run **in-process**, not as +cmux panes. Each answers exactly one question and returns the frame it used. + +For each entry in `questions.json`, dispatch one sub-agent with: + +- the `question` text verbatim, +- the absolute path to its `frame_dir` and the `frame_times` mapping. These times are read from the + `-frame_pts 1` JPEG filenames after extraction, so a missing frame leaves a timestamp gap instead + of shifting every later image, +- the instruction to return only + `{"id": ..., "verdict": "YES"|"NO"|"NOT_OBSERVABLE", "frame": ..., "note": ...}`. + +`questions.json` contains no receipt claim or expected answer. Do not give the adjudicator +`expectations.json`, `run.json`, or permission to inspect parent/sibling paths outside the listed +`frame_dir`; the split keeps receipt claims out of the adjudication inputs. + +Collect the verdicts into a JSON array and render the report: + +```bash +node scripts/qa-video-harness.mjs report \ + results/qa-video/ \ + results/qa-video//verdicts.json \ + docs.local/reports/qa-video-$(date +%F).md +``` + +The report reconciles each verdict against what the receipt claimed: + +- **AGREE** — the frames match what the receipt implies. +- **CONTRADICT** — they do not. This is the product. +- **NOT OBSERVABLE** — the sub-agent could not tell, or the step never reached its mark. An honest + gap, never a pass. +- **MISSING** — no verdict came back. Loud on purpose, so a partial adjudication cannot read clean. + +## Design notes, and the traps already hit + +**Isolation is by window, and where possible by display.** `cmux new-window` gets a window containing +only the probe panes, renamed to a unique `QAV-` title. Teardown closes exactly that window, +refuses to touch any window that existed beforehand, and also runs on SIGINT/SIGTERM so an +interrupted run does not leave an orphan window on the operator's desktop. + +The recorder captures a screen **region**, so isolation is not free — whatever is stacked over that +region silently becomes the evidence. Five things had to be true before the frames could be trusted, +and each was learned by getting it wrong first: + +1. **Address the window, never "the frontmost process."** That resolved to whatever the human last + touched; the first recording cropped to a browser window and captured private content. +2. **Resolve geometry through CoreGraphics, not System Events.** A freshly created cmux window is + intermittently absent from the accessibility window list entirely, and it drops out of the + on-screen list whenever its Space is not active — so a single miss is a flap, not an absence. +3. **Record the display the window is actually on.** cmux does not always open on the main display. + Two runs' worth of frames showed display 0 while the probe window sat on display 1, and every one + of those frames looked completely plausible. +4. **Do not fight for z-order; take a display.** The harness runs from inside a cmux pane, so the + operator's own cmux window is being activated constantly by the commands driving the probe. + `isolateProbeWindowOnOwnDisplay` moves the probe window to the least-occupied display, because + occlusion is per-display and stacking then stops mattering. On a single-display machine this is a + no-op and the z-order checks carry the weight instead. +5. **Raise with AXRaise only.** An earlier per-probe re-assert also called `cmux focus-window`, which + churned cmux's surface topology hard enough that `spawn_agent` began failing with `not live or + uniquely resolvable in a complete fresh topology`. An observer that changes the observed state + produces evidence about itself. + +Every mark records whether the window was unoccluded at that instant, and +`buildAdjudicationManifest` refuses to generate a question for any mark where it was not. A +compromised probe reports NOT OBSERVABLE; it never reports as a pass. If the window cannot be made +clear at all, the run aborts rather than recording the desktop. + +**Rejected: capturing the window's own content.** `screencapture -l ` composites a single +window and is immune to occlusion, display placement and focus stealing — it looked like the answer. +It is not: cmux renders its terminals with Metal, so window capture returns the chrome and sidebar +with a **blank content area**. The terminal text, which is the entire point, is absent. Do not retry +this without looking at a frame. + +**The harness must carry no caller identity.** cmux exports the operator pane's `CMUX_SURFACE_ID`, +`CMUX_TAB_ID`, `CMUX_WORKSPACE_ID` and friends into every child process. With those inherited, the +MCP server resolved the *caller* as whatever agent was running the harness and started guarding that +agent's surface — the first full run died on `refusing terminal I/O`. They are stripped before the +server is spawned. `CMUX_SOCKET_PATH` is kept: it addresses the daemon, it does not identify anyone. + +**Recorder: ffmpeg + avfoundation**, not `screencapture -v`. Not because ffmpeg records better, but +because `-progress` makes it report its own clock. The first progress block with a real `out_time_us` +is captured exactly once as the immutable anchor that maps wall-clock instants onto recording +seconds; later progress blocks cannot move it. The recording is verified with `ffprobe` (non-empty, +real duration, decodable frames) before anything is derived from it. + +**Frames are sampled densely at transitions, not uniformly.** Each question gets its own window +around its own mark, at 10 fps. The window reaches well past the mark because a mark is the instant +of the tool *call* and the pixels lag it — a dry-run measured that lag at ~1.7s for a plain +`cmux send`, so a tight window would simply miss the event it exists to catch. Frames are JPEG rather +than PNG because terminal-text adjudication does not need archival losslessness; `-q:v 2` now applies +to the actual encoder instead of being ignored. + +**The dry-run is a clock test, not just a smoke test.** It prints a unique nonce into the probe pane +at a known instant and asks a sub-agent whether that nonce is visible in the frames the harness +predicted. If the wall-clock-to-video mapping were wrong, that question would fail. + +## Requirements + +- macOS with **Screen Recording** permission for whatever runs the harness (ffmpeg inherits it). +- **Accessibility** permission for the same process, for the AXRaise/window-move AppleScript. +- `python3` with `pyobjc` Quartz bindings, for `scripts/qa-video-windows.py`. This is how the harness + finds the probe window, its display and its occluders; without it the run aborts rather than + guessing. +- `ffmpeg` and `ffprobe` on PATH (`brew install ffmpeg`). +- A running cmux, and `bun run build` so `dist/index.js` exists (or pass `--server-command`). +- A second display is not required, but it is the difference between reliable isolation and fighting + the operator's own cmux window for z-order. + +## Known limits + +- Activating the probe window steals focus for the length of the run. That is inherent to recording + a live GUI; run it when the machine is free. Running the harness from inside a cmux pane makes this + materially worse, because the operator's own cmux window is being raised by the very commands + driving the probe. +- The composer questions are binary, and Cursor's TUI has a third state: a queued "follow-ups" panel + that is neither the composer nor the transcript. The first live run landed exactly there. The + question wording should become three-way before the next run. +- Anything that is not on screen cannot be adjudicated. Registry-internal fields — `closure` most of + all — have no pixels. `list-closure-flap` therefore asks the nearest observable question ("did the + pane visibly change at all during the span?") and will often be NOT OBSERVABLE. That is the honest + answer, and it is still useful: a `closure` that flaps while the pane is provably static is a + receipt with no referent. diff --git a/package.json b/package.json index 65cd06fe..407c5284 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,9 @@ "install:fleet-sidebar:dev": "node scripts/install-fleet-sidebar-dev.mjs", "bench:daemon": "node scripts/bench-daemon.mjs", "test:watch": "vitest", - "live:harness": "node scripts/run-live-agent-harness.mjs" + "live:harness": "node scripts/run-live-agent-harness.mjs", + "qa:video": "node scripts/qa-video-harness.mjs", + "qa:video:dry-run": "node scripts/qa-video-harness.mjs --dry-run" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.12.0", diff --git a/scripts/qa-video-harness.mjs b/scripts/qa-video-harness.mjs new file mode 100755 index 00000000..998235d8 --- /dev/null +++ b/scripts/qa-video-harness.mjs @@ -0,0 +1,1490 @@ +#!/usr/bin/env node +/** + * QA video harness (lane QA-V) — video ground truth for cmuxlayer claims. + * + * cmuxlayer is normally the only witness to cmuxlayer: every claim is verified + * by the same tool suite that produced it. This harness records a live probe + * from OUTSIDE the system under test — a macOS screen recording of an isolated + * cmux window — and emits an adjudication manifest that cheap vision sub-agents + * answer one narrow question at a time. + * + * It never merges the two sources itself. The script produces receipts + frames; + * an Opus lane dispatches Sonnet sub-agents over the manifest and renders the + * report. See docs/qa-video-harness.md. + * + * AIDEV-NOTE: this script only orchestrates. All decision logic that could be + * wrong in an interesting way lives in scripts/qa-video-lib.mjs and is covered + * by tests/qa-video-harness.test.ts. + */ + +import { execFile, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +import { + DRY_RUN_SPEC, + PROBE_SPECS, + assertFrameBudget, + buildAdjudicationManifest, + combineAdjudicationArtifacts, + splitAdjudicationManifest, + wallToVideoSeconds, +} from "./qa-video-lib.mjs"; + +const execFileAsync = promisify(execFile); +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, ".."); + +const DEFAULTS = { + mode: "full", + cli: "cursor", + repo: "cmuxlayer", + captureFps: 15, + scaleWidth: 0, + frameFps: 10, + maxFrames: 750, + keepRuns: 5, + waitTimeoutMs: 8_000, + agentReadyTimeoutMs: 180_000, + root: "", + serverCommand: "", + serverArgs: [], + skipPreflight: false, + keepWindow: false, +}; + +function usage() { + process.stderr.write(`Usage: qa-video-harness.mjs [options] + +Requires CMUX_QA_VIDEO=1 (this drives a live cmux and records the screen). + +Options: + --dry-run Recorder + frame-extraction self-test only. No agents. + --cli Agent CLI for probe agents (default: ${DEFAULTS.cli}) + --repo repoGolem repo for spawns (default: ${DEFAULTS.repo}) + --capture-fps Screen capture framerate (default: ${DEFAULTS.captureFps}) + --scale-width Downscale the recording to this width (default: native) + --frame-fps Frame extraction density around each mark (default: ${DEFAULTS.frameFps}) + --max-frames Hard cap across all extracted question frames (default: ${DEFAULTS.maxFrames}) + --keep-runs Retain this many completed default-output runs (default: ${DEFAULTS.keepRuns}) + --wait-timeout-ms Timeout for the wait_for probe (default: ${DEFAULTS.waitTimeoutMs}) + --root Output dir (default: results/qa-video/) + --server-command MCP server executable + --server-arg Repeatable MCP server arg + --skip-preflight Skip the recorder self-test before a full run (NOT recommended) + --keep-window Leave the isolated window open for inspection + --help +`); +} + +export function parseArgs(argv) { + const options = { ...DEFAULTS }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--help": + case "-h": + usage(); + process.exit(0); + break; + case "--dry-run": + options.mode = "dry-run"; + break; + case "--cli": + options.cli = argv[++index]; + break; + case "--repo": + options.repo = argv[++index]; + break; + case "--capture-fps": + options.captureFps = Number(argv[++index]); + break; + case "--scale-width": + options.scaleWidth = Number(argv[++index]); + break; + case "--frame-fps": + options.frameFps = Number(argv[++index]); + break; + case "--max-frames": + options.maxFrames = Number(argv[++index]); + break; + case "--keep-runs": + options.keepRuns = Number(argv[++index]); + break; + case "--wait-timeout-ms": + options.waitTimeoutMs = Number(argv[++index]); + break; + case "--root": + options.root = resolve(argv[++index]); + break; + case "--server-command": + options.serverCommand = argv[++index]; + break; + case "--server-arg": + options.serverArgs.push(argv[++index]); + break; + case "--skip-preflight": + options.skipPreflight = true; + break; + case "--keep-window": + options.keepWindow = true; + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + for (const key of ["captureFps", "frameFps", "waitTimeoutMs"]) { + if (!Number.isFinite(options[key]) || options[key] <= 0) { + throw new Error(`--${key} must be a positive number`); + } + } + if (!Number.isFinite(options.scaleWidth) || options.scaleWidth < 0) { + throw new Error("--scaleWidth must be zero or a positive number"); + } + for (const key of ["maxFrames", "keepRuns"]) { + if (!Number.isSafeInteger(options[key]) || options[key] <= 0) { + throw new Error(`--${key} must be a positive integer`); + } + } + if (!["claude", "codex", "cursor", "gemini", "kiro"].includes(options.cli)) { + throw new Error("--cli must be one of: claude, codex, cursor, gemini, kiro"); + } + return options; +} + +export function assertOptIn(env = process.env) { + if (env.CMUX_QA_VIDEO === "1") return; + throw new Error( + "Refusing to run the QA video harness: it drives a live cmux and records the screen. Set CMUX_QA_VIDEO=1 to opt in.", + ); +} + +const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); +const nowMs = () => Date.now(); + +function makeNonce(tag) { + const suffix = Math.random().toString(36).slice(2, 6).toUpperCase(); + return `QAV-${tag}-${suffix}`; +} + +// --------------------------------------------------------------------------- +// cmux CLI +// --------------------------------------------------------------------------- + +async function cmux(args, { json = false } = {}) { + const { stdout } = await execFileAsync("cmux", args, { + env: { ...process.env, CMUX_QUIET: "1" }, + maxBuffer: 16 * 1024 * 1024, + }); + if (!json) return stdout.trim(); + return JSON.parse(stdout); +} + +async function listWindowIds() { + const raw = await cmux(["list-windows"]); + return raw + .split("\n") + .map((line) => line.trim().match(/^\*?\s*\d+:\s+([0-9A-Fa-f-]{36})/)) + .filter(Boolean) + .map((match) => match[1]); +} + +/** + * Read cmux's on-screen windows from CoreGraphics. + * + * AIDEV-NOTE: an earlier implementation used System Events / AppleScript and was + * wrong in three escalating ways. It asked for "the frontmost process", which on + * a live desktop is whatever the human last touched — the first recording cropped + * to a browser window and captured private content. Targeting cmux by window + * title then failed because a freshly created cmux window is intermittently + * absent from the accessibility window list. And even with correct bounds, cmux + * does not always open its new window on the main display, so the recorder was + * capturing a different screen entirely. CoreGraphics is the window server's own + * list: no Accessibility grant, it always sees the window, and it hands back the + * CGWindowID that makes all of the above moot. + */ +async function readWindowState() { + const { stdout } = await execFileAsync("python3", [join(__dirname, "qa-video-windows.py")], { + maxBuffer: 8 * 1024 * 1024, + }); + const parsed = JSON.parse(stdout); + if (parsed.error) throw new Error(`window enumeration failed: ${parsed.error}`); + return parsed; +} + +export function rectsIntersect(a, b) { + return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h; +} + +export function displayContaining(displays, bounds) { + const centre = { x: bounds.x + bounds.w / 2, y: bounds.y + bounds.h / 2 }; + return ( + displays.find( + (display) => + centre.x >= display.bounds.x && + centre.x < display.bounds.x + display.bounds.w && + centre.y >= display.bounds.y && + centre.y < display.bounds.y + display.bounds.h, + ) ?? displays.find((display) => display.main) ?? displays[0] ?? null + ); +} + +/** + * Everything the recorder needs about the probe window: which display it is on, + * the crop rectangle in that display's capture pixels, and whether any other + * ordinary window is covering it. + * + * AIDEV-NOTE: cmux does NOT always open its new window on the main display. + * Recording display 0 while the probe window sat on display 1 is what made two + * whole runs' worth of frames show the wrong screen while looking perfectly + * plausible. + */ +async function probeWindowGeometry(title, { attempts = 3, gapMs = 400 } = {}) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const found = await probeWindowGeometryOnce(title); + if (found) return found; + if (attempt < attempts - 1) await sleep(gapMs); + } + return null; +} + +/** + * AIDEV-NOTE: a cmux window drops out of the on-screen window list whenever its + * Space is not the active one on its display, which it does intermittently right + * after being moved. A single miss is a flap, not an absence, hence the retry + * above — an early version treated the first miss as "the window is gone" and + * aborted runs that were perfectly fine a moment later. + */ +async function probeWindowGeometryOnce(title) { + const state = await readWindowState(); + return probeWindowGeometryFromState(state, title); +} + +export function probeWindowGeometryFromState(state, title) { + const cmuxWindows = state.windows.filter((window) => window.owner === "cmux"); + // The short sibling window is cmux's tab strip, not the window we mean. + const probe = cmuxWindows.find((window) => window.name === title && window.bounds.h > 200); + if (!probe) return null; + + const display = displayContaining(state.displays, probe.bounds); + if (!display) return null; + + // Anything at the same window layer sitting ahead of the probe in front-to-back + // order and overlapping it is, literally, covering the evidence. + const occluders = state.windows + .slice(0, state.windows.indexOf(probe)) + .filter( + (window) => + window.layer === probe.layer && + window.bounds.w > 40 && + window.bounds.h > 40 && + rectsIntersect(window.bounds, probe.bounds), + ) + .map((window) => `${window.owner}: ${window.name || "(untitled)"}`); + + const scale = display.scale || 1; + return { + id: probe.id, + bounds: probe.bounds, + display, + occluders, + clear: occluders.length === 0, + crop: { + x: Math.round((probe.bounds.x - display.bounds.x) * scale), + y: Math.round((probe.bounds.y - display.bounds.y) * scale), + width: Math.round(probe.bounds.w * scale), + height: Math.round(probe.bounds.h * scale), + }, + }; +} + +/** True when the probe window exists and nothing ordinary is covering it. */ +async function probeWindowIsFrontmost(title) { + try { + const geometry = await probeWindowGeometry(title); + return Boolean(geometry?.clear); + } catch { + return false; + } +} + +/** Raise the probe window so the recorded region shows it and not something else. */ +async function focusProbeWindow(windowId, title) { + try { + await cmux(["focus-window", "--window", windowId]); + } catch { + /* non-fatal */ + } + try { + await execFileAsync("osascript", ["-e", 'tell application "cmux" to activate']); + } catch { + /* cmux may be named differently in a fresh install */ + } + if (!title) return; + try { + await execFileAsync("osascript", [ + "-e", + `tell application "System Events" to tell process "cmux" + set frontmost to true + perform action "AXRaise" of (first window whose name is "${title}") + end tell`, + ]); + } catch { + /* best effort; the recorder does not care */ + } +} + +/** + * AIDEV-NOTE: a cmux window that is not on the active Space is absent from the + * on-screen window list and cannot be captured at all ("could not create image + * from window"). Focusing it brings its Space forward, so the wait re-focuses + * each round rather than passively polling. + */ +/** + * Restack-only raise: AXRaise plus app activation, and nothing that touches + * cmux's own state. + * + * AIDEV-NOTE: the per-step re-assert deliberately does NOT call + * `cmux focus-window`. Doing so between probes churned cmux's surface topology + * badly enough that spawn_agent started failing with "not live or uniquely + * resolvable in a complete fresh topology" — the harness was perturbing the + * system it exists to observe. AXRaise returns before the window server has + * restacked, hence the settle-and-verify loop. + */ +async function raiseProbeWindow(title, { attempts = 4, settleMs = 400 } = {}) { + if (!title) return false; + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + await execFileAsync("osascript", [ + "-e", + `tell application "System Events" to tell process "cmux" + set frontmost to true + perform action "AXRaise" of (first window whose name is "${title}") + end tell`, + ]); + } catch { + /* verified below by CoreGraphics, not by the raise's own report */ + } + await sleep(settleMs); + if (await probeWindowIsFrontmost(title)) return true; + } + return false; +} + +/** + * Move the probe window onto a display that no other ordinary window occupies. + * + * AIDEV-NOTE: this is the fix that finally made isolation hold. The harness runs + * from inside a cmux pane, so the operator's own cmux window is being activated + * constantly by the very commands driving the probe — fighting it for z-order on + * a shared rectangle is a fight the harness cannot win, and every run that tried + * ended up recording the operator's window. Occlusion is per-display, so putting + * the probe window on its own display makes stacking irrelevant. On a + * single-display machine this is a no-op and the z-order checks still apply. + */ +async function isolateProbeWindowOnOwnDisplay(title, attempt = 0) { + const state = await readWindowState(); + if (state.displays.length < 2) return false; + const probe = state.windows.find( + (window) => window.owner === "cmux" && window.name === title && window.bounds.h > 200, + ); + if (!probe) return false; + + const occupancy = new Map(state.displays.map((display) => [display.index, 0])); + for (const window of state.windows) { + if (window.layer !== probe.layer) continue; + if (window === probe) continue; + if (window.bounds.w < 200 || window.bounds.h < 200) continue; + const display = displayContaining(state.displays, window.bounds); + if (display) occupancy.set(display.index, (occupancy.get(display.index) ?? 0) + 1); + } + const ranked = state.displays + .slice() + .sort((a, b) => (occupancy.get(a.index) ?? 0) - (occupancy.get(b.index) ?? 0)); + // Retries walk down the ranking: the emptiest display is a good guess, not a + // guarantee, and a full-screen window the heuristic under-counts should not + // wedge the harness onto a display it can never own. + const target = ranked[attempt % ranked.length]; + if (!target) return false; + const current = displayContaining(state.displays, probe.bounds); + if (current && current.index === target.index && attempt === 0) return true; + + try { + await execFileAsync("osascript", [ + "-e", + `tell application "System Events" to tell process "cmux" to tell (first window whose name is "${title}") + set position to {${target.bounds.x}, ${target.bounds.y + 25}} + set size to {${target.bounds.w}, ${target.bounds.h - 50}} + end tell`, + ]); + } catch { + return false; + } + await sleep(800); + return true; +} + +async function waitForProbeGeometry(probeWindow, timeoutMs = 25_000) { + const deadline = nowMs() + timeoutMs; + for (let attempt = 0; ; attempt += 1) { + await isolateProbeWindowOnOwnDisplay(probeWindow.title, attempt); + await focusProbeWindow(probeWindow.windowId, probeWindow.title); + await sleep(800); + await raiseProbeWindow(probeWindow.title); + const geometry = await probeWindowGeometry(probeWindow.title).catch(() => null); + if (geometry?.clear) return geometry; + if (nowMs() >= deadline) return geometry ?? null; + } +} + +// --------------------------------------------------------------------------- +// Recorder +// --------------------------------------------------------------------------- + +/** + * Map a CoreGraphics display index onto its avfoundation input index. + * + * ffmpeg names these "Capture screen N" following CGGetActiveDisplayList order, + * but the surrounding numbering also counts cameras, so the two indices are not + * interchangeable and must be looked up rather than assumed. + */ +async function screenDeviceIndexFor(displayIndex = 0) { + const output = await new Promise((resolveOutput) => { + const child = spawn("ffmpeg", ["-hide_banner", "-f", "avfoundation", "-list_devices", "true", "-i", ""]); + let buffer = ""; + child.stderr.on("data", (chunk) => { + buffer += chunk.toString(); + }); + child.on("close", () => resolveOutput(buffer)); + }); + const screens = new Map(); + for (const match of output.matchAll(/\[(\d+)\]\s+Capture screen (\d+)/g)) { + screens.set(Number(match[2]), match[1]); + } + const found = screens.get(Number(displayIndex)); + if (found) return found; + const fallback = screens.get(0); + if (fallback) { + process.stderr.write( + `[qa-video] warning: no avfoundation input for display ${displayIndex}; falling back to Capture screen 0\n`, + ); + return fallback; + } + throw new Error("ffmpeg reports no screen-capture inputs; check Screen Recording permission"); +} + +/** + * ffmpeg/avfoundation screen-region recorder. + * + * AIDEV-NOTE: ffmpeg is chosen over `screencapture -v` because it reports its + * own clock through `-progress`, which is what lets every probe timestamp be + * mapped onto the recording. `screencapture -v` gives no such signal, so its + * frames could only ever be aligned by eye. + * + * AIDEV-NOTE: capturing the window's own content with `screencapture -l + * ` was tried and REJECTED, despite being immune to occlusion, + * display placement and focus stealing. cmux renders its terminals with Metal, + * and window-content capture returns the window chrome with a blank content + * area — the sidebar and title render, the terminal text does not. Do not + * "improve" this back to window capture without checking a frame first. + * + * Because this records a REGION, isolation is not free: the probe window has to + * be the frontmost thing on that region, which is what the CoreGraphics + * occlusion check above is for, and what every mark's `frontmost` flag records. + */ +export class Recorder { + constructor({ + path, + captureFps, + scaleWidth, + crop, + deviceIndex, + spawnFn = spawn, + now = nowMs, + anchorTimeoutMs = 30_000, + }) { + this.path = path; + this.captureFps = captureFps; + this.scaleWidth = scaleWidth; + this.crop = crop; + this.deviceIndex = deviceIndex; + this.spawnFn = spawnFn; + this.now = now; + this.anchorTimeoutMs = anchorTimeoutMs; + this.child = null; + this.t0WallMs = null; + this.t0VideoS = null; + this.stderr = ""; + } + + buildFilters() { + const filters = []; + if (this.crop) { + const { x, y, width, height } = this.crop; + filters.push(`crop=${width}:${height}:${x}:${y}`); + } + if (this.scaleWidth > 0) filters.push(`scale=${this.scaleWidth}:-2`); + filters.push("format=yuv420p"); + return filters.join(","); + } + + async start() { + const args = [ + "-hide_banner", + "-loglevel", + "warning", + "-y", + "-f", + "avfoundation", + "-capture_cursor", + "1", + "-framerate", + String(this.captureFps), + "-i", + `${this.deviceIndex}:none`, + "-vf", + this.buildFilters(), + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "23", + "-progress", + "pipe:1", + this.path, + ]; + this.child = this.spawnFn("ffmpeg", args, { stdio: ["pipe", "pipe", "pipe"] }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + + // Wait for the first progress block that carries a real out_time_us. That + // instant is the anchor between wall clock and recording clock. + await new Promise((done, fail) => { + let buffer = ""; + const timer = setTimeout( + () => fail(new Error(`recorder never reported progress:\n${this.stderr}`)), + this.anchorTimeoutMs, + ); + const onExit = () => + fail(new Error(`recorder exited before producing frames:\n${this.stderr}`)); + this.child.once("close", onExit); + const onProgress = (chunk) => { + buffer += chunk.toString(); + const match = [...buffer.matchAll(/out_time_us=(\d+)/g)].find( + (candidate) => Number(candidate[1]) > 0, + ); + if (!match) return; + const micros = Number(match[1]); + if (!Number.isFinite(micros) || micros <= 0) return; + this.t0WallMs = this.now(); + this.t0VideoS = micros / 1_000_000; + clearTimeout(timer); + this.child.off("close", onExit); + this.child.stdout.off("data", onProgress); + done(); + }; + this.child.stdout.on("data", onProgress); + }); + } + + /** Seconds from the recording's anchor for a wall-clock instant. */ + secondsAt(wallMs) { + return wallToVideoSeconds( + { t0WallMs: this.t0WallMs, t0VideoS: this.t0VideoS }, + wallMs, + ); + } + + async stop() { + if (!this.child) return; + const exited = new Promise((done) => this.child.once("close", done)); + this.child.stdin.write("q\n"); + const timer = setTimeout(() => this.child.kill("SIGINT"), 5_000); + await exited; + clearTimeout(timer); + this.child = null; + } +} + +async function probeVideo(path) { + const { stdout } = await execFileAsync("ffprobe", [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=nb_frames,width,height", + "-show_entries", + "format=duration,size", + "-of", + "json", + path, + ]); + const parsed = JSON.parse(stdout); + const stream = parsed.streams?.[0] ?? {}; + return { + frames: Number(stream.nb_frames ?? 0), + width: Number(stream.width ?? 0), + height: Number(stream.height ?? 0), + durationS: Number(parsed.format?.duration ?? 0), + bytes: Number(parsed.format?.size ?? 0), + }; +} + +export function assertVideoUsable(info, path) { + if (info.bytes <= 0) throw new Error(`recording ${path} is empty`); + if (!(info.durationS > 0)) throw new Error(`recording ${path} has no duration`); + if (!(info.frames > 0)) throw new Error(`recording ${path} has no decodable frames`); +} + +/** Read actual ffmpeg frame PTS values from `-frame_pts 1` filenames. */ +export async function readExtractedFrameMapping( + outDir, + { relativeDir, start, fps }, +) { + const entries = await readdir(outDir, { withFileTypes: true }); + const frames = []; + for (const entry of entries) { + if (!entry.isFile()) continue; + const match = entry.name.match(/^f-(\d+)\.jpg$/); + if (!match) continue; + const info = await stat(join(outDir, entry.name)); + if (info.size <= 0) continue; + frames.push({ + pts: Number(match[1]), + frame: join(relativeDir, entry.name), + }); + } + frames.sort((a, b) => a.pts - b.pts); + return frames.map(({ pts, frame }) => ({ + frame, + timeS: Math.round((start + pts / fps) * 1000) / 1000, + })); +} + +/** Accurate-seek extraction whose filenames carry the actual filtered frame PTS. */ +async function extractFrames({ video, outDir, relativeDir, start, end, fps }) { + await mkdir(outDir, { recursive: true }); + await execFileAsync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + video, + "-ss", + String(start), + "-to", + String(end), + "-vf", + `fps=${fps}`, + "-frame_pts", + "1", + "-q:v", + "2", + join(outDir, "f-%010d.jpg"), + ]); + return readExtractedFrameMapping(outDir, { relativeDir, start, fps }); +} + +// --------------------------------------------------------------------------- +// MCP stdio client +// --------------------------------------------------------------------------- + +/** + * cmux exports the operator pane's identity into every child process. If the + * harness's MCP server inherits it, cmuxlayer resolves the CALLER as whatever + * agent is running the harness and starts guarding its surface — the first full + * run died on `Stable surface UUID ... refusing terminal I/O`. The harness is + * an outside observer and must carry no caller identity at all. + * + * AIDEV-NOTE: CMUX_SOCKET_PATH/CMUX_SOCKET are deliberately kept — those address + * the cmux daemon, they do not identify a caller. + */ +const CALLER_IDENTITY_ENV = [ + "CMUX_WORKSPACE_ID", + "CMUX_TAB_ID", + "CMUX_SURFACE_ID", + "CMUX_PANEL_ID", + "CMUX_TERMINAL_LIFECYCLE_ID", +]; + +export function sterileEnv(env = process.env) { + const copy = { ...env }; + for (const key of CALLER_IDENTITY_ENV) delete copy[key]; + return copy; +} + +class McpStdioClient { + constructor(command, args) { + this.nextId = 1; + this.pending = new Map(); + this.buffer = ""; + this.closed = false; + this.stderr = ""; + this.child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"], env: sterileEnv() }); + this.child.stdout.setEncoding("utf8"); + this.child.stderr.setEncoding("utf8"); + this.child.stdout.on("data", (chunk) => this.onStdout(chunk)); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk; + }); + this.child.on("close", () => { + this.closed = true; + for (const [, pending] of this.pending) pending.reject(new Error("MCP server exited")); + this.pending.clear(); + }); + } + + onStdout(chunk) { + this.buffer += chunk; + let newline = this.buffer.indexOf("\n"); + while (newline >= 0) { + const line = this.buffer.slice(0, newline).trim(); + this.buffer = this.buffer.slice(newline + 1); + if (line.length > 0) { + try { + this.onMessage(JSON.parse(line)); + } catch { + /* server logged a non-JSON line; ignore */ + } + } + newline = this.buffer.indexOf("\n"); + } + } + + onMessage(message) { + if (typeof message.id !== "number") return; + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.error) { + pending.reject(new Error(message.error.message ?? JSON.stringify(message.error))); + return; + } + pending.resolve(message.result ?? {}); + } + + request(method, params, timeoutMs = 180_000) { + if (this.closed) return Promise.reject(new Error("MCP server already closed")); + const id = this.nextId++; + return new Promise((done, fail) => { + const timer = setTimeout(() => { + this.pending.delete(id); + fail(new Error(`Timed out waiting for ${method} after ${timeoutMs}ms`)); + }, timeoutMs); + this.pending.set(id, { + resolve: (value) => { + clearTimeout(timer); + done(value); + }, + reject: (error) => { + clearTimeout(timer); + fail(error); + }, + }); + this.child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`); + }); + } + + async initialize() { + await this.request("initialize", { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "cmuxlayer-qa-video-harness", version: "0.1.0" }, + }); + this.child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })}\n`, + ); + } + + async callTool(name, args, timeoutMs) { + const result = await this.request("tools/call", { name, arguments: args }, timeoutMs); + const text = (result.content ?? []) + .filter((part) => part?.type === "text") + .map((part) => part.text) + .join("\n"); + return { + ok: result.isError !== true, + text, + structured: result.structuredContent ?? null, + raw: result, + }; + } + + close() { + try { + this.child.stdin.end(); + this.child.kill("SIGTERM"); + } catch { + /* already gone */ + } + } +} + +function resolveServerCommand(options) { + if (options.serverCommand) { + return { command: options.serverCommand, args: options.serverArgs }; + } + const dist = join(REPO_ROOT, "dist", "index.js"); + if (existsSync(dist)) return { command: process.execPath, args: [dist] }; + const tsx = join(REPO_ROOT, "node_modules", ".bin", "tsx"); + if (existsSync(tsx)) return { command: tsx, args: [join(REPO_ROOT, "src", "index.ts")] }; + throw new Error( + "No MCP server entrypoint found. Run `bun run build` (creates dist/index.js) or pass --server-command.", + ); +} + +// --------------------------------------------------------------------------- +// Run recording helpers +// --------------------------------------------------------------------------- + +class RunLog { + constructor(recorder, probeWindow) { + this.recorder = recorder; + this.probeWindow = probeWindow; + this.steps = []; + } + + /** + * Re-assert the raise before every probe. Raising once before the recorder + * starts is not enough — anything on the desktop can restack at any time, and + * a five-minute run has already been observed losing the probe window to the + * operator's own cmux window partway through. + */ + async enterStep(spec, context = {}) { + await raiseProbeWindow(this.probeWindow.title); + return this.step(spec, context); + } + + step(spec, context = {}) { + const step = { id: spec.id, title: spec.title, issues: spec.issues, context, marks: {}, calls: [], error: null }; + this.steps.push(step); + return step; + } + + /** + * Marks are wall-clock instants stamped onto the capture timeline. + * + * AIDEV-NOTE: every mark records whether the probe window was actually + * unoccluded at that instant. The recorder captures a screen REGION, so + * anything stacked over it silently replaces the evidence; a mark with + * `frontmost: false` is one the manifest refuses to ask a question about. + */ + async mark(step, name) { + const wallMs = nowMs(); + const entry = { wallMs, videoS: this.recorder.secondsAt(wallMs), at: new Date(wallMs).toISOString() }; + step.marks[name] = entry; + entry.frontmost = await probeWindowIsFrontmost(this.probeWindow.title); + return entry; + } + + /** Record a tool call verbatim alongside its position on the recording clock. */ + async call(step, client, tool, args, { timeoutMs, mark } = {}) { + const startedAt = nowMs(); + if (mark) await this.mark(step, mark); + let receipt = null; + let error = null; + try { + receipt = await client.callTool(tool, args, timeoutMs); + } catch (thrown) { + error = thrown instanceof Error ? thrown.message : String(thrown); + } + const finishedAt = nowMs(); + step.calls.push({ + tool, + args, + receipt, + error, + started_at: new Date(startedAt).toISOString(), + started_video_s: this.recorder.secondsAt(startedAt), + finished_video_s: this.recorder.secondsAt(finishedAt), + duration_ms: finishedAt - startedAt, + }); + return { receipt, error }; + } +} + +function structuredOf(result) { + return result?.receipt?.structured ?? {}; +} + +function agentIdOf(result) { + const structured = structuredOf(result); + return ( + structured.agent_id ?? + structured.agent?.agent_id ?? + structured.spawned?.agent_id ?? + null + ); +} + +function surfaceIdOf(result) { + const structured = structuredOf(result); + return structured.surface_id ?? structured.surface ?? structured.agent?.surface_id ?? null; +} + +async function pollAgentState(client, agentId, predicate, timeoutMs) { + const startedAt = nowMs(); + let last = null; + while (nowMs() - startedAt < timeoutMs) { + const result = await client.callTool("list_agents", { agent_ids: [agentId] }); + const agents = result.structured?.agents ?? []; + last = agents.find((agent) => (agent.agent_id ?? agent.id) === agentId) ?? null; + if (last && predicate(last)) return last; + await sleep(1_500); + } + return last; +} + +// --------------------------------------------------------------------------- +// Isolated probe window +// --------------------------------------------------------------------------- + +async function createProbeWindow(runId) { + const before = new Set(await listWindowIds()); + const created = await cmux(["new-window"]); + const match = created.match(/([0-9A-Fa-f-]{36})/); + if (!match) throw new Error(`could not parse window id from: ${created}`); + const windowId = match[1]; + if (before.has(windowId)) { + throw new Error(`new-window returned a pre-existing window (${windowId}); refusing to proceed`); + } + await sleep(1_000); + const listing = await cmux(["workspace", "list", "--window", windowId, "--json"], { json: true }); + const workspace = listing.workspaces?.[0]; + if (!workspace?.ref) throw new Error("probe window has no workspace"); + // The title is the harness's only handle on this window from outside cmux, so + // it has to be unique and it has to be set on the window itself, not just the + // workspace (System Events reads the window title). + // Sanitised because the title is interpolated into an AppleScript string literal. + const title = `QAV-${String(runId).replace(/[^A-Za-z0-9._-]/g, "-")}`; + await cmux(["rename-workspace", "--workspace", workspace.ref, "--window", windowId, title]); + await cmux(["rename-window", "--window", windowId, title]); + await focusProbeWindow(windowId, title); + await sleep(800); + return { + windowId, + windowRef: listing.window_ref ?? null, + workspaceRef: workspace.ref, + title, + preExisting: before, + }; +} + +export async function destroyProbeWindow( + probeWindow, + { cmuxFn = cmux, stderr = process.stderr } = {}, +) { + if (!probeWindow) return; + if (probeWindow.preExisting.has(probeWindow.windowId)) { + stderr.write(`Refusing to close pre-existing window ${probeWindow.windowId}\n`); + return; + } + try { + await cmuxFn(["close-window", "--window", probeWindow.windowId]); + } catch (error) { + stderr.write(`Teardown warning: ${error instanceof Error ? error.message : error}\n`); + } +} + +async function countSurfaces(workspaceRef, windowId) { + try { + const listing = await cmux( + ["list-pane-surfaces", "--workspace", workspaceRef, "--window", windowId, "--json"], + { json: true }, + ); + return (listing.surfaces ?? []).length; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Probes +// --------------------------------------------------------------------------- + +const BUSY_PROMPT = + "Do exactly this and nothing else: count slowly from 1 to 60, printing one number per line, waiting about two seconds between each line. Do not stop early."; +const FAST_PROMPT = "Reply with the single word ACKNOWLEDGED and then stop. Do nothing else."; + +async function runFullProbes({ client, log, options, probeWindow }) { + const spec = (id) => PROBE_SPECS.find((entry) => entry.id === id); + + // --- Agent A: the busy agent used by probes 1, 4 and 5. --- + const busySpawn = await log.enterStep({ id: "setup-busy-agent", title: "spawn the busy probe agent", issues: [] }); + const spawnA = await log.call(busySpawn, client, "spawn_agent", { + repo: options.repo, + cli: options.cli, + role: "implementor", + authority: "worker", + workspace: probeWindow.workspaceRef, + prompt: BUSY_PROMPT, + }, { timeoutMs: options.agentReadyTimeoutMs, mark: "spawn" }); + const agentA = agentIdOf(spawnA); + const surfaceA = surfaceIdOf(spawnA); + busySpawn.context = { agentId: agentA, surfaceId: surfaceA }; + if (!agentA) { + busySpawn.error = spawnA.error ?? "spawn_agent returned no agent_id"; + return; + } + const busyState = await pollAgentState(client, agentA, (agent) => agent.state === "working", 90_000); + // AIDEV-NOTE: recorded because a run has already been observed where the pane + // was visibly streaming while the registry still said "ready". If this is not + // "working", the busy-send probe did not test what it claims to test, and the + // report has to say so rather than quietly reporting on an idle agent. + busySpawn.context.registryStateWhenBusy = busyState?.state ?? null; + + const paneHintA = `the ${options.cli} agent pane (surface ${surfaceA ?? "?"}, agent ${agentA})`; + + // --- Probe 1: send_to a BUSY agent (#432/#484) --- + { + const nonce = makeNonce("BUSY"); + const step = await log.enterStep(spec("busy-send"), { + nonce, + paneHint: paneHintA, + agentId: agentA, + surfaceId: surfaceA, + registryStateAtSend: busySpawn.context.registryStateWhenBusy, + }); + await log.call(step, client, "send_to", { mode: "agent", agent_id: agentA, text: nonce }, { mark: "send", timeoutMs: 60_000 }); + await sleep(2_000); + await log.mark(step, "plus2"); + await sleep(1_500); + } + + // --- Probe 5: wait_for on a working agent (#473) --- + { + const step = await log.enterStep(spec("wait-for-working"), { paneHint: paneHintA, agentId: agentA }); + const result = await log.call( + step, + client, + "wait_for", + { agent_id: agentA, target_state: "done", timeout_ms: options.waitTimeoutMs }, + { mark: "call", timeoutMs: options.waitTimeoutMs + 30_000 }, + ); + await log.mark(step, "return"); + const blob = `${result.receipt?.text ?? ""} ${JSON.stringify(result.receipt?.structured ?? {})}`; + step.context.claimedAlreadyCompleted = /already\s+(completed|done)/i.test(blob); + step.context.waitDurationMs = step.calls.at(-1)?.duration_ms ?? null; + await sleep(1_500); + } + + // --- Probe 4: list_agents x3 over ~20s (#488) --- + { + const step = await log.enterStep(spec("list-closure-flap"), { paneHint: paneHintA, agentId: agentA }); + await log.mark(step, "span"); + const closures = []; + for (let attempt = 0; attempt < 3; attempt += 1) { + const result = await log.call(step, client, "list_agents", { agent_ids: [agentA], detail: "full" }, { timeoutMs: 60_000 }); + const agents = result.receipt?.structured?.agents ?? []; + const row = agents.find((agent) => (agent.agent_id ?? agent.id) === agentA) ?? {}; + closures.push({ closure: row.closure ?? null, state: row.state ?? null }); + if (attempt < 2) await sleep(10_000); + } + step.context.closures = closures; + step.context.closureFlapped = + new Set(closures.map((entry) => JSON.stringify(entry.closure))).size > 1; + step.context.stateFlapped = new Set(closures.map((entry) => entry.state)).size > 1; + } + + // --- Agent B: reaches a terminal registry state while its pane stays live. --- + const fastSpawn = await log.enterStep({ id: "setup-terminal-agent", title: "spawn the short-lived probe agent", issues: [] }); + const spawnB = await log.call(fastSpawn, client, "spawn_agent", { + repo: options.repo, + cli: options.cli, + role: "implementor", + authority: "worker", + workspace: probeWindow.workspaceRef, + prompt: FAST_PROMPT, + }, { timeoutMs: options.agentReadyTimeoutMs, mark: "spawn" }); + const agentB = agentIdOf(spawnB); + const surfaceB = surfaceIdOf(spawnB); + fastSpawn.context = { agentId: agentB, surfaceId: surfaceB }; + const paneHintB = `the short-lived ${options.cli} pane (surface ${surfaceB ?? "?"}, agent ${agentB})`; + + if (agentB) { + const settled = await pollAgentState( + client, + agentB, + (agent) => ["done", "error", "idle"].includes(agent.state), + 120_000, + ); + fastSpawn.context.settledState = settled?.state ?? null; + + // --- Probe 2: send_to a stale-terminal registry row (#484) --- + { + const nonce = makeNonce("STALE"); + const step = await log.enterStep(spec("stale-terminal-send"), { + nonce, + paneHint: paneHintB, + agentId: agentB, + registryState: settled?.state ?? null, + }); + await log.call(step, client, "send_to", { mode: "agent", agent_id: agentB, text: nonce }, { mark: "send", timeoutMs: 60_000 }); + await sleep(2_000); + await log.mark(step, "plus2"); + await sleep(1_500); + } + + // --- Probe 3: close_surface(scope:"agent") (#485) --- + { + const before = await countSurfaces(probeWindow.workspaceRef, probeWindow.windowId); + const step = await log.enterStep(spec("close-agent"), { + paneHint: paneHintB, + agentId: agentB, + surfaceCountBefore: before, + }); + await log.call(step, client, "close_surface", { scope: "agent", agent_id: agentB, force: true }, { mark: "close", timeoutMs: 60_000 }); + await sleep(3_000); + await log.mark(step, "plus3"); + step.context.surfaceCountAfter = await countSurfaces(probeWindow.workspaceRef, probeWindow.windowId); + await sleep(1_500); + } + } else { + fastSpawn.error = spawnB.error ?? "spawn_agent returned no agent_id"; + } + + // --- Probe 6: spawn while keystrokes are injected (#434/#440) --- + { + const junk = makeNonce("JUNK"); + const step = await log.enterStep(spec("spawn-under-keystrokes"), { + junk, + paneHint: "the newly launched agent pane (rightmost in the probe workspace)", + }); + await log.mark(step, "launch"); + const spawnPromise = log.call(step, client, "spawn_agent", { + repo: options.repo, + cli: options.cli, + role: "implementor", + authority: "worker", + workspace: probeWindow.workspaceRef, + prompt: "Reply with the single word LAUNCHED and then stop.", + }, { timeoutMs: options.agentReadyTimeoutMs }); + + const injectionDeadline = nowMs() + 5_000; + const injections = []; + while (nowMs() < injectionDeadline) { + try { + await cmux(["send", "--workspace", probeWindow.workspaceRef, "--window", probeWindow.windowId, junk]); + injections.push({ at: new Date().toISOString(), ok: true }); + } catch (error) { + injections.push({ at: new Date().toISOString(), ok: false, error: String(error) }); + } + await sleep(700); + } + step.context.injections = injections; + const spawnC = await spawnPromise; + step.context.agentId = agentIdOf(spawnC); + step.context.surfaceId = surfaceIdOf(spawnC); + await sleep(4_000); + await log.mark(step, "settle"); + await sleep(2_000); + } +} + +async function runDryRunProbe({ log, probeWindow }) { + const nonce = makeNonce("CLAPPER"); + const step = await log.enterStep(DRY_RUN_SPEC, { nonce }); + await sleep(1_000); + await log.mark(step, "clap"); + // A clapperboard: printing a known nonce at a known instant on the recording + // clock is what proves the wall-clock -> video-clock mapping is real. + await cmux([ + "send", + "--workspace", + probeWindow.workspaceRef, + "--window", + probeWindow.windowId, + `clear; printf '\\n\\n %s\\n\\n' ${nonce}\n`, + ]); + step.calls.push({ + tool: "cmux send (clapper)", + args: { nonce }, + receipt: { ok: true, structured: { ok: true } }, + error: null, + started_at: new Date().toISOString(), + }); + await sleep(4_000); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +export function installSignalHandlers( + probeWindow, + { + signalSource = process, + execFileFn = execFile, + exit = (code) => process.exit(code), + } = {}, +) { + const onSignal = () => { + try { + execFileFn("cmux", ["close-window", "--window", probeWindow.windowId], () => exit(130)); + } catch { + exit(130); + } + }; + signalSource.once("SIGINT", onSignal); + signalSource.once("SIGTERM", onSignal); + return () => { + signalSource.off("SIGINT", onSignal); + signalSource.off("SIGTERM", onSignal); + }; +} + +export function assertPreflightReady(preflight) { + if (preflight.extracted > 0) return; + throw new Error( + "Preflight produced no frames. Refusing to run the full harness against live panes.", + ); +} + +async function directorySizeBytes(root) { + let total = 0; + for (const entry of await readdir(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) total += await directorySizeBytes(path); + else if (entry.isFile()) total += (await stat(path)).size; + } + return total; +} + +export function formatBytes(bytes) { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KiB`; + if (bytes < 1024 ** 3) return `${(bytes / 1024 ** 2).toFixed(1)} MiB`; + return `${(bytes / 1024 ** 3).toFixed(2)} GiB`; +} + +/** Retain only the newest completed default-output runs. Partial runs are untouched. */ +export async function pruneRunDirectories(baseDir, { keep = DEFAULTS.keepRuns } = {}) { + let entries; + try { + entries = await readdir(baseDir, { withFileTypes: true }); + } catch (error) { + if (error?.code === "ENOENT") return []; + throw error; + } + const completed = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!existsSync(join(baseDir, entry.name, "run.json"))) continue; + const info = await stat(join(baseDir, entry.name)); + completed.push({ name: entry.name, mtimeMs: info.mtimeMs }); + } + completed.sort((a, b) => b.mtimeMs - a.mtimeMs || b.name.localeCompare(a.name)); + const pruned = completed.slice(keep).map((entry) => entry.name); + for (const name of pruned) { + await rm(join(baseDir, name), { recursive: true, force: true }); + } + return pruned; +} + +async function runOnce(options, { runId, root }) { + await mkdir(root, { recursive: true }); + const videoPath = join(root, "video.mov"); + + const probeWindow = await createProbeWindow(runId); + // A killed harness must not leak an isolated window onto the operator's + // desktop. This is the only teardown path that survives Ctrl-C / SIGTERM. + const removeSignalHandlers = installSignalHandlers(probeWindow); + let recorder = null; + let log = null; + let geometry = null; + let occlusionRisk = null; + const startedAt = new Date().toISOString(); + try { + geometry = await waitForProbeGeometry(probeWindow); + if (!geometry) { + throw new Error( + `Could not find the probe window "${probeWindow.title}" in the CoreGraphics window list, so there is no window to record.`, + ); + } + if (!geometry.clear) { + throw new Error( + `The probe window is covered by: ${geometry.occluders.join(", ")}. Recording now would capture those windows instead — ` + + "ambiguous evidence, and a privacy leak. Move or close them, or free up a second display for the probe window, and re-run.", + ); + } + const deviceIndex = await screenDeviceIndexFor(geometry.display.index); + recorder = new Recorder({ + path: videoPath, + captureFps: options.captureFps, + scaleWidth: options.scaleWidth, + crop: geometry.crop, + deviceIndex, + }); + await recorder.start(); + log = new RunLog(recorder, probeWindow); + + if (options.mode === "dry-run") { + await runDryRunProbe({ log, probeWindow }); + } else { + const server = resolveServerCommand(options); + const client = new McpStdioClient(server.command, server.args); + try { + await client.initialize(); + await runFullProbes({ client, log, options, probeWindow }); + } finally { + client.close(); + } + } + // The window can be restacked mid-run by anything on the desktop. Say so in + // the receipts rather than letting the frames read as trustworthy. + occlusionRisk = !(await probeWindowIsFrontmost(probeWindow.title)); + } finally { + removeSignalHandlers(); + if (recorder) await recorder.stop(); + if (!options.keepWindow) await destroyProbeWindow(probeWindow); + } + + const video = await probeVideo(videoPath); + assertVideoUsable(video, videoPath); + + const run = { + runId, + mode: options.mode, + startedAt, + finishedAt: new Date().toISOString(), + options: { ...options, serverArgs: options.serverArgs }, + window: { + windowId: probeWindow.windowId, + windowRef: probeWindow.windowRef, + workspaceRef: probeWindow.workspaceRef, + title: probeWindow.title, + cgWindowId: geometry.id, + bounds: geometry.bounds, + display: geometry.display, + }, + video: { + path: "video.mov", + fps: options.captureFps, + frameFps: options.frameFps, + t0WallMs: recorder.t0WallMs, + t0VideoS: recorder.t0VideoS, + occlusionRisk, + ...video, + }, + steps: log?.steps ?? [], + }; + + const manifest = buildAdjudicationManifest(run, { fps: options.frameFps }); + const plannedFrames = assertFrameBudget(manifest, options.maxFrames); + let extracted = 0; + for (const question of manifest.questions) { + if (!question.frame_window) continue; + const mapping = await extractFrames({ + video: videoPath, + outDir: join(root, question.frame_dir), + relativeDir: question.frame_dir, + start: question.frame_window.start, + end: question.frame_window.end, + fps: question.frame_window.fps, + }); + question.frames = mapping.map((entry) => entry.frame); + question.frame_times = mapping.map((entry) => entry.timeS); + if (mapping.length === 0) { + question.unadjudicable_reason = "frame extraction produced no images for this window"; + } + extracted += mapping.length; + } + manifest.extracted_frames = extracted; + manifest.planned_frames = plannedFrames; + const { questions, expectations } = splitAdjudicationManifest(manifest); + + await writeFile(join(root, "run.json"), `${JSON.stringify(run, null, 2)}\n`, "utf8"); + await writeFile(join(root, "questions.json"), `${JSON.stringify(questions, null, 2)}\n`, "utf8"); + await writeFile(join(root, "expectations.json"), `${JSON.stringify(expectations, null, 2)}\n`, "utf8"); + const artifactBytes = await directorySizeBytes(root); + return { run, manifest, root, extracted, plannedFrames, artifactBytes }; +} + + +async function main() { + assertOptIn(); + const options = parseArgs(process.argv.slice(2)); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const baseRunId = `${options.mode}-${stamp}`; + const baseRoot = options.root || join(REPO_ROOT, "results", "qa-video", baseRunId); + + if (options.mode === "full" && !options.skipPreflight) { + process.stdout.write("[qa-video] preflight: recorder + frame-extraction self-test\n"); + const preflight = await runOnce( + { ...options, mode: "dry-run", keepWindow: false }, + { runId: `${baseRunId}-preflight`, root: join(baseRoot, "preflight") }, + ); + assertPreflightReady(preflight); + process.stdout.write( + `[qa-video] preflight ok: ${preflight.run.video.frames} recorded frames, ${preflight.extracted} extracted.\n` + + `[qa-video] VISUAL confirmation is still required: adjudicate ${join(preflight.root, "questions.json")} before trusting the full run.\n`, + ); + } + + const result = await runOnce(options, { runId: baseRunId, root: baseRoot }); + const prunedRuns = options.root + ? [] + : await pruneRunDirectories(join(REPO_ROOT, "results", "qa-video"), { + keep: options.keepRuns, + }); + process.stdout.write( + [ + `[qa-video] run: ${result.run.runId}`, + `[qa-video] video: ${join(result.root, "video.mov")} (${result.run.video.durationS}s, ${result.run.video.frames} frames, ${result.run.video.width}x${result.run.video.height}, display ${result.run.window.display?.index})`, + `[qa-video] receipts: ${join(result.root, "run.json")}`, + `[qa-video] questions: ${join(result.root, "questions.json")} (${result.manifest.questions.length} questions, ${result.extracted}/${result.plannedFrames} frames)`, + `[qa-video] expectations: ${join(result.root, "expectations.json")}`, + `[qa-video] artifacts: ${formatBytes(result.artifactBytes)}; retention pruned ${prunedRuns.length} old run(s)`, + `[qa-video] next: adjudicate with Sonnet sub-agents, then render the report (docs/qa-video-harness.md)`, + "", + ].join("\n"), + ); +} + +/** Render a report from a manifest + a verdicts file. Kept here so the whole + * lane is one script: `qa-video-harness.mjs report `. */ +async function report(runDir, verdictsPath, outPath) { + const { buildReportMarkdown } = await import("./qa-video-lib.mjs"); + const run = JSON.parse(await readFile(join(runDir, "run.json"), "utf8")); + const questions = JSON.parse(await readFile(join(runDir, "questions.json"), "utf8")); + const expectations = JSON.parse(await readFile(join(runDir, "expectations.json"), "utf8")); + const manifest = combineAdjudicationArtifacts(questions, expectations); + const verdicts = JSON.parse(await readFile(verdictsPath, "utf8")); + const markdown = buildReportMarkdown(run, manifest, Array.isArray(verdicts) ? verdicts : verdicts.verdicts, { + now: new Date().toISOString(), + }); + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, markdown, "utf8"); + process.stdout.write(`[qa-video] report written: ${outPath}\n`); +} + +function isMainModule(metaUrl, argvEntry = process.argv[1]) { + return Boolean(argvEntry) && metaUrl === pathToFileURL(argvEntry).href; +} + +if (isMainModule(import.meta.url)) { + const [, , maybeSubcommand] = process.argv; + if (maybeSubcommand === "report") { + const [runDir, verdictsPath, outPath] = process.argv.slice(3); + if (!runDir || !verdictsPath || !outPath) { + process.stderr.write("Usage: qa-video-harness.mjs report \n"); + process.exit(2); + } + report(resolve(runDir), resolve(verdictsPath), resolve(outPath)).catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exit(1); + }); + } else { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exit(1); + }); + } +} diff --git a/scripts/qa-video-lib.mjs b/scripts/qa-video-lib.mjs new file mode 100644 index 00000000..e37c66be --- /dev/null +++ b/scripts/qa-video-lib.mjs @@ -0,0 +1,581 @@ +/** + * Pure helpers for the QA video harness (lane QA-V). + * + * AIDEV-NOTE: Everything in this file must stay side-effect free so the harness + * logic (frame planning, question generation, verdict reconciliation, report + * rendering) is unit-testable without a live cmux, a display, or ffmpeg. + * The impure runner lives in scripts/qa-video-harness.mjs. + */ + +/** Verdicts an adjudicating sub-agent may return. */ +export const VERDICTS = ["YES", "NO", "NOT_OBSERVABLE"]; + +/** Reconciliation outcomes between a tool receipt and the recorded frames. */ +export const OUTCOMES = ["AGREE", "CONTRADICT", "NOT_OBSERVABLE", "MISSING"]; + +/** + * Probe catalogue. Each entry names the fleet-reported repro it re-runs and the + * narrow questions a vision sub-agent must answer about it. `mark` selects which + * timestamp on the step the frame window is centred on. + */ +export const PROBE_SPECS = [ + { + id: "busy-send", + title: "send_to a BUSY agent", + issues: ["#432", "#484"], + questions: [ + { + suffix: "typed", + mark: "send", + // AIDEV-NOTE: the mark is the instant the tool CALL was made; the pixels + // lag it by the cmux round-trip plus a terminal repaint. A measured + // dry-run put that at ~1.7s, so this window is deliberately generous + // after the mark — missing the transition is the one failure that makes + // the whole frame budget worthless. + window: { before: 1.0, after: 6.0 }, + text: (ctx) => + `Did the exact text \`${ctx.nonce}\` appear in the composer / input line of the agent pane (${ctx.paneHint})?`, + expectedIfReceiptTrue: (step) => + receiptDelivered(step) ? "YES" : "NO", + }, + { + suffix: "submitted", + mark: "plus2", + window: { before: 0.5, after: 2.0 }, + text: (ctx) => + `Two seconds after the send, is \`${ctx.nonce}\` STILL sitting unsent in the composer of ${ctx.paneHint}? Answer YES if it is still in the composer (i.e. it was never submitted), NO if the composer is clear or the text has moved into the transcript above the composer (i.e. it was submitted).`, + // A receipt that claims submitted implies the composer is clear -> NO. + expectedIfReceiptTrue: (step) => + receiptSubmitted(step) ? "NO" : "YES", + }, + ], + }, + { + id: "stale-terminal-send", + title: "send_to a pane whose registry row is stale-terminal", + issues: ["#484"], + questions: [ + { + suffix: "typed", + mark: "send", + // AIDEV-NOTE: the mark is the instant the tool CALL was made; the pixels + // lag it by the cmux round-trip plus a terminal repaint. A measured + // dry-run put that at ~1.7s, so this window is deliberately generous + // after the mark — missing the transition is the one failure that makes + // the whole frame budget worthless. + window: { before: 1.0, after: 6.0 }, + text: (ctx) => + `The registry believes this agent is terminal (done/error) but its pane is still alive. Did the exact text \`${ctx.nonce}\` appear in the composer of ${ctx.paneHint}?`, + expectedIfReceiptTrue: (step) => + receiptDelivered(step) ? "YES" : "NO", + }, + { + suffix: "submitted", + mark: "plus2", + window: { before: 0.5, after: 2.0 }, + text: (ctx) => + `Two seconds later, is \`${ctx.nonce}\` still unsent in the composer of ${ctx.paneHint}? YES = still in the composer (never submitted). NO = composer clear or text moved into the transcript (submitted).`, + expectedIfReceiptTrue: (step) => + receiptSubmitted(step) ? "NO" : "YES", + }, + ], + }, + { + id: "close-agent", + title: 'close_surface(scope:"agent")', + issues: ["#485"], + questions: [ + { + suffix: "pane-gone", + mark: "plus3", + window: { before: 3.5, after: 1.5 }, + text: (ctx) => + `Before the close there were ${ctx.surfaceCountBefore ?? "N"} terminal pane(s)/tab(s) in this window. After the close, has the agent pane (${ctx.paneHint}) actually disappeared from the window? YES = the pane is gone. NO = the pane is still rendered.`, + expectedIfReceiptTrue: (step) => (receiptOk(step) ? "YES" : "NO"), + }, + ], + }, + { + id: "list-closure-flap", + title: "list_agents x3 over ~20s", + issues: ["#488"], + questions: [ + { + suffix: "pane-unchanged", + mark: "span", + window: { before: 0.5, after: 21.0 }, + text: () => + `Across this ~20 second span, did the agent pane change in any visible way that could justify its lifecycle/closure state changing — did it start or stop streaming, exit, show a new prompt, or disappear? YES = something visibly changed. NO = the pane looked static throughout.`, + // The receipt is honest only if a flapping closure field is matched by a + // visible change. If closure did NOT flap, frames are simply corroborating. + expectedIfReceiptTrue: (step) => + step?.context?.closureFlapped ? "YES" : "NO", + }, + ], + }, + { + id: "wait-for-working", + title: "wait_for on a working agent", + issues: ["#473"], + questions: [ + { + suffix: "still-working", + mark: "return", + window: { before: 2.0, after: 1.5 }, + text: (ctx) => + `At the instant wait_for returned, was the agent pane (${ctx.paneHint}) visibly STILL working — spinner animating, tokens streaming, or a "working/thinking/esc to interrupt" style status line on screen? YES = still working. NO = the pane is idle at a bare prompt.`, + // A receipt claiming "already completed" is only honest if the pane is idle. + expectedIfReceiptTrue: (step) => + step?.context?.claimedAlreadyCompleted ? "NO" : "YES", + }, + ], + }, + { + id: "spawn-under-keystrokes", + title: "spawn while keystrokes are injected", + issues: ["#434", "#440"], + questions: [ + { + suffix: "launcher-line-clean", + mark: "launch", + // A spawn takes far longer to reach the launcher line than a send does. + window: { before: 1.0, after: 12.0 }, + text: (ctx) => + `Keystrokes were injected into this pane while the launcher command was being typed. Look at the shell command line in ${ctx.paneHint}. Is the launcher invocation clean — i.e. does it read as an intact command with no stray injected characters (\`${ctx.junk}\`) spliced into it? YES = clean. NO = corrupted / interleaved.`, + expectedIfReceiptTrue: (step) => (receiptOk(step) ? "YES" : "NO"), + }, + { + suffix: "recovered", + mark: "settle", + window: { before: 2.0, after: 4.0 }, + text: (ctx) => + `A few seconds after launch, did the agent CLI in ${ctx.paneHint} actually come up (a real agent TUI is rendered), or is the pane sitting at a shell prompt / showing a command-not-found style error? YES = the agent CLI is up. NO = it did not launch.`, + expectedIfReceiptTrue: (step) => (receiptOk(step) ? "YES" : "NO"), + }, + ], + }, +]; + +/** The single trivial probe used by --dry-run to prove the whole pipeline. */ +export const DRY_RUN_SPEC = { + id: "clapper", + title: "recorder + frame-extraction self-test", + issues: [], + questions: [ + { + suffix: "visible", + mark: "clap", + window: { before: 1.0, after: 5.0 }, + text: (ctx) => + `Is the exact string \`${ctx.nonce}\` visible anywhere in this frame? YES = visible. NO = not visible.`, + expectedIfReceiptTrue: () => "YES", + }, + ], +}; + +function firstReceipt(step) { + const call = (step?.calls ?? []).find((entry) => entry?.receipt); + return call?.receipt ?? null; +} + +/** True when the receipt claims the text reached the pane at all. */ +export function receiptDelivered(step) { + const receipt = firstReceipt(step); + if (!receipt) return false; + const structured = receipt.structured ?? {}; + if (structured.delivered === true) return true; + return ["submitted", "queued", "queued_followup", "pending_verify"].includes( + String(structured.delivery ?? structured.delivery_state ?? ""), + ); +} + +/** True when the receipt claims the text was actually submitted (Enter landed). */ +export function receiptSubmitted(step) { + const receipt = firstReceipt(step); + if (!receipt) return false; + const structured = receipt.structured ?? {}; + return ( + String(structured.delivery ?? structured.delivery_state ?? "") === + "submitted" + ); +} + +/** True when the receipt reported success. */ +export function receiptOk(step) { + const receipt = firstReceipt(step); + if (!receipt) return false; + if (receipt.ok === false) return false; + if (receipt.structured?.ok === false) return false; + return receipt.ok === true || receipt.structured?.ok === true; +} + +/** Human-readable one-liner of what the receipts CLAIMED for a step. */ +export function describeReceiptClaim(step) { + const calls = step?.calls ?? []; + if (calls.length === 0) return "no tool call recorded"; + return calls + .map((call) => { + if (call.error) return `${call.tool} threw: ${call.error}`; + const structured = call.receipt?.structured ?? {}; + const bits = []; + if (typeof structured.ok === "boolean" || typeof call.receipt?.ok === "boolean") { + bits.push(`ok=${structured.ok ?? call.receipt?.ok}`); + } + for (const key of [ + "delivered", + "delivery", + "delivery_state", + "state", + "closure", + "closed", + "scope", + "agent_id", + ]) { + if (structured[key] !== undefined) bits.push(`${key}=${JSON.stringify(structured[key])}`); + } + return `${call.tool} -> ${bits.length > 0 ? bits.join(" ") : "(no structured fields)"}`; + }) + .join("; "); +} + +function round(value, digits = 3) { + const factor = 10 ** digits; + return Math.round(value * factor) / factor; +} + +/** + * Convert a wall-clock instant into a position on the recording clock. + * `video.t0WallMs` is the wall time at which the recorder reported `t0VideoS`. + */ +export function wallToVideoSeconds(video, wallMs) { + if (!video || typeof video.t0WallMs !== "number") { + throw new Error("video clock is missing t0WallMs"); + } + return round(video.t0VideoS + (wallMs - video.t0WallMs) / 1000); +} + +/** + * Dense frame plan for one question: samples at `fps` across the mark's window, + * clamped to the recording. Transitions are the hot spots, so the caller gives a + * tight window rather than sampling the whole video uniformly. + */ +export function planFrameWindow({ markSeconds, window: win, fps, durationS }) { + if (typeof markSeconds !== "number" || Number.isNaN(markSeconds)) return null; + const start = Math.max(0, round(markSeconds - win.before)); + const rawEnd = round(markSeconds + win.after); + const end = typeof durationS === "number" ? Math.min(rawEnd, round(durationS)) : rawEnd; + if (end <= start) return null; + const count = Math.max(1, Math.round((end - start) * fps)); + const times = []; + for (let index = 0; index < count; index += 1) { + times.push(round(start + index / fps)); + } + return { start, end, fps, count, times }; +} + +function specForStep(step) { + if (step?.id === DRY_RUN_SPEC.id) return DRY_RUN_SPEC; + return PROBE_SPECS.find((spec) => spec.id === step?.id) ?? null; +} + +/** + * Build the adjudication manifest: one narrow question per entry, each carrying + * the frames a Sonnet sub-agent should look at and nothing else. + */ +export function buildAdjudicationManifest(run, { fps = 10 } = {}) { + const durationS = run?.video?.durationS; + const questions = []; + for (const step of run?.steps ?? []) { + const spec = specForStep(step); + if (!spec) continue; + const claim = describeReceiptClaim(step); + for (const question of spec.questions) { + const mark = step.marks?.[question.mark]; + const id = `${step.id}.${question.suffix}`; + // A mark taken while the probe window was occluded recorded some OTHER + // window. Those frames cannot answer anything about the probe, and asking + // a sub-agent about them invites a confident wrong answer. + if (mark && mark.frontmost === false) { + questions.push({ + id, + step: step.id, + step_title: spec.title, + issues: spec.issues, + mark: question.mark, + mark_video_s: mark.videoS, + question: safeText(question, step), + receipt_claim: claim, + expected_if_receipt_true: safeExpected(question, step), + frames: [], + frame_times: [], + unadjudicable_reason: + "the probe window was occluded at this mark; the recording captured a different window", + }); + continue; + } + if (!mark || typeof mark.videoS !== "number") { + questions.push({ + id, + step: step.id, + step_title: spec.title, + issues: spec.issues, + mark: question.mark, + question: safeText(question, step), + receipt_claim: claim, + expected_if_receipt_true: safeExpected(question, step), + frames: [], + frame_times: [], + unadjudicable_reason: step.error + ? `step failed before the mark was reached: ${step.error}` + : `mark "${question.mark}" was never recorded`, + }); + continue; + } + const plan = planFrameWindow({ + markSeconds: mark.videoS, + window: question.window, + fps, + durationS, + }); + questions.push({ + id, + step: step.id, + step_title: spec.title, + issues: spec.issues, + mark: question.mark, + mark_video_s: mark.videoS, + question: safeText(question, step), + receipt_claim: claim, + expected_if_receipt_true: safeExpected(question, step), + frame_dir: `frames/${id}`, + frame_window: plan + ? { start: plan.start, end: plan.end, fps: plan.fps, count: plan.count } + : null, + frames: plan + ? plan.times.map((_, index) => `frames/${id}/f-${String(index + 1).padStart(4, "0")}.jpg`) + : [], + frame_times: plan ? plan.times : [], + unadjudicable_reason: plan ? null : "mark falls outside the recorded video", + }); + } + } + return { + run_id: run?.runId ?? null, + mode: run?.mode ?? null, + video: run?.video ?? null, + allowed_verdicts: VERDICTS, + questions, + }; +} + +/** + * Keep the adjudicator's payload structurally independent from receipt claims. + * The questions document may be handed to a vision worker; the expectations + * document is held back until report generation. + */ +export function splitAdjudicationManifest(manifest) { + const questions = { + ...manifest, + questions: (manifest?.questions ?? []).map( + ({ receipt_claim: _claim, expected_if_receipt_true: _expected, ...question }) => question, + ), + }; + const expectations = { + run_id: manifest?.run_id ?? null, + expectations: (manifest?.questions ?? []).map((question) => ({ + id: question.id, + receipt_claim: question.receipt_claim, + expected_if_receipt_true: question.expected_if_receipt_true, + })), + }; + return { questions, expectations }; +} + +/** Rejoin the two trusted inputs only inside report generation. */ +export function combineAdjudicationArtifacts(questions, expectations) { + if ((questions?.run_id ?? null) !== (expectations?.run_id ?? null)) { + throw new Error("questions and expectations belong to different QA video runs"); + } + const byId = new Map( + (expectations?.expectations ?? []).map((entry) => [entry.id, entry]), + ); + return { + ...questions, + questions: (questions?.questions ?? []).map((question) => { + const expectation = byId.get(question.id); + if (!expectation) { + throw new Error(`missing expectation for adjudication question ${question.id}`); + } + return { ...question, ...expectation }; + }), + }; +} + +/** Fail before extraction can create an unbounded frame set. */ +export function assertFrameBudget(manifest, maxFrames) { + if (!Number.isSafeInteger(maxFrames) || maxFrames <= 0) { + throw new Error("frame cap must be a positive integer"); + } + const planned = (manifest?.questions ?? []).reduce( + (total, question) => total + (question.frame_window?.count ?? 0), + 0, + ); + if (planned > maxFrames) { + throw new Error( + `planned extraction requires ${planned} frames, exceeding the ${maxFrames}-frame cap`, + ); + } + return planned; +} + +function safeText(question, step) { + try { + return question.text(step?.context ?? {}); + } catch (error) { + return `(question text failed to render: ${error instanceof Error ? error.message : String(error)})`; + } +} + +function safeExpected(question, step) { + try { + return question.expectedIfReceiptTrue(step); + } catch { + return null; + } +} + +/** + * Reconcile sub-agent verdicts against what the receipts claimed. + * A CONTRADICT is the product of this harness; a MISSING verdict is loud on + * purpose so a partial adjudication can never read as a clean run. + */ +export function reconcile(manifest, verdicts) { + const byId = new Map( + (verdicts ?? []).map((verdict) => [verdict.id, verdict]), + ); + const rows = manifest.questions.map((question) => { + const verdict = byId.get(question.id); + if (question.unadjudicable_reason && !verdict) { + return { + ...question, + verdict: "NOT_OBSERVABLE", + outcome: "NOT_OBSERVABLE", + note: question.unadjudicable_reason, + frame: null, + }; + } + if (!verdict) { + return { ...question, verdict: null, outcome: "MISSING", note: "no verdict returned", frame: null }; + } + const value = String(verdict.verdict ?? "").toUpperCase(); + if (!VERDICTS.includes(value)) { + return { + ...question, + verdict: value || null, + outcome: "MISSING", + note: `invalid verdict ${JSON.stringify(verdict.verdict)}`, + frame: verdict.frame ?? null, + }; + } + let outcome = "NOT_OBSERVABLE"; + if (value !== "NOT_OBSERVABLE") { + if (question.expected_if_receipt_true === null || question.expected_if_receipt_true === undefined) { + outcome = "NOT_OBSERVABLE"; + } else { + outcome = value === question.expected_if_receipt_true ? "AGREE" : "CONTRADICT"; + } + } + return { + ...question, + verdict: value, + outcome, + note: verdict.note ?? null, + frame: verdict.frame ?? null, + }; + }); + const totals = Object.fromEntries(OUTCOMES.map((outcome) => [outcome, 0])); + for (const row of rows) totals[row.outcome] += 1; + return { rows, totals }; +} + +function fence(value) { + return "```json\n" + JSON.stringify(value, null, 2) + "\n```"; +} + +/** Render the human-facing report. Contradictions lead; they are the product. */ +export function buildReportMarkdown(run, manifest, verdicts, { now } = {}) { + const { rows, totals } = reconcile(manifest, verdicts); + const stamp = now ?? run?.startedAt ?? "unknown"; + const lines = []; + lines.push(`# QA video report — ${run?.runId ?? "unknown run"}`); + lines.push(""); + lines.push( + `Ground truth for cmuxlayer claims taken from **outside** cmuxlayer: a screen recording of an isolated probe window, adjudicated frame-by-frame by vision sub-agents.`, + ); + lines.push(""); + lines.push(`- Run: \`${run?.runId ?? "?"}\` (mode: \`${run?.mode ?? "?"}\`) started ${stamp}`); + lines.push( + `- Video: \`${run?.video?.path ?? "?"}\` — ${run?.video?.durationS ?? "?"}s, ${run?.video?.fps ?? "?"} fps capture, ${run?.video?.frames ?? "?"} frames`, + ); + lines.push(`- Isolated window: \`${run?.window?.windowRef ?? "?"}\`, workspace \`${run?.window?.workspaceRef ?? "?"}\``); + lines.push( + `- Outcomes: **${totals.CONTRADICT} CONTRADICT**, ${totals.AGREE} AGREE, ${totals.NOT_OBSERVABLE} NOT OBSERVABLE, ${totals.MISSING} MISSING`, + ); + lines.push(""); + + const contradictions = rows.filter((row) => row.outcome === "CONTRADICT"); + lines.push("## Contradictions"); + lines.push(""); + if (contradictions.length === 0) { + lines.push("None. Every adjudicable receipt matched the frames."); + } else { + for (const row of contradictions) { + lines.push(`### ${row.id} ${row.issues.length > 0 ? `(${row.issues.join(", ")})` : ""}`); + lines.push(""); + lines.push(`- **Receipt claimed:** ${row.receipt_claim}`); + lines.push(`- **Frames show:** verdict \`${row.verdict}\` (receipt implies \`${row.expected_if_receipt_true}\`)`); + lines.push(`- **Evidence frame:** \`${row.frame ?? "(sub-agent returned none)"}\` at t≈${row.mark_video_s ?? "?"}s`); + if (row.note) lines.push(`- **Adjudicator note:** ${row.note}`); + lines.push(""); + } + } + lines.push(""); + + lines.push("## Per-probe detail"); + lines.push(""); + lines.push("| probe | issues | question | receipt claimed | frames show | outcome |"); + lines.push("| --- | --- | --- | --- | --- | --- |"); + for (const row of rows) { + lines.push( + `| \`${row.id}\` | ${row.issues.join(" ") || "—"} | ${escapeCell(row.question)} | ${escapeCell(row.receipt_claim)} | ${row.verdict ?? "—"} | **${row.outcome}** |`, + ); + } + lines.push(""); + + const notObservable = rows.filter((row) => row.outcome === "NOT_OBSERVABLE"); + if (notObservable.length > 0) { + lines.push("## Not observable from video"); + lines.push(""); + lines.push("These are honest gaps, not passes — the state they assert is off-screen."); + lines.push(""); + for (const row of notObservable) { + lines.push(`- \`${row.id}\` — ${row.note ?? row.unadjudicable_reason ?? "adjudicator could not tell from the frames"}`); + } + lines.push(""); + } + + lines.push("## Raw receipts"); + lines.push(""); + for (const step of run?.steps ?? []) { + lines.push(`### ${step.id}`); + lines.push(""); + if (step.error) lines.push(`Step error: \`${step.error}\``); + lines.push(fence(step.calls ?? [])); + lines.push(""); + } + return lines.join("\n"); +} + +function escapeCell(value) { + return String(value ?? "").replace(/\|/g, "\\|").replace(/\n/g, " "); +} diff --git a/scripts/qa-video-windows.py b/scripts/qa-video-windows.py new file mode 100755 index 00000000..6783bf8a --- /dev/null +++ b/scripts/qa-video-windows.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Emit cmux's on-screen windows, front-to-back, as JSON. + +AIDEV-NOTE: this exists because System Events / AXRaise proved unreliable for +this job — a freshly created cmux window is intermittently absent from the +accessibility window list entirely, which made the harness's isolation check +say "not frontmost" for a window that was plainly on top. CoreGraphics' +window list is the authority the window server itself uses: it needs no +Accessibility grant, it always sees the window, and it reports true front-to- +back order plus bounds, which is what "is anything covering the probe" needs. + +Output: {"windows": [{"owner","name","bounds":{x,y,w,h},"layer"}, ...]} in +front-to-back order, on-screen windows only. +""" + +import json +import sys + +try: + from Quartz import ( + CGDisplayBounds, + CGDisplayCopyDisplayMode, + CGDisplayModeGetPixelWidth, + CGDisplayModeGetWidth, + CGGetActiveDisplayList, + CGMainDisplayID, + CGWindowListCopyWindowInfo, + kCGNullWindowID, + kCGWindowListExcludeDesktopElements, + kCGWindowListOptionOnScreenOnly, + ) +except ImportError: + print(json.dumps({"error": "Quartz (pyobjc) unavailable"})) + sys.exit(3) + + +def displays() -> list: + """Active displays in CGGetActiveDisplayList order, with backing scale. + + avfoundation enumerates its "Capture screen N" inputs in this same order, + so the list index is the capture device index -- but the harness still + verifies that by comparing the recorded resolution against pixel_w/pixel_h + rather than trusting the correspondence. + """ + err, ids, count = CGGetActiveDisplayList(16, None, None) + if err: + return [] + main = CGMainDisplayID() + out = [] + for index, display_id in enumerate(ids[:count]): + bounds = CGDisplayBounds(display_id) + mode = CGDisplayCopyDisplayMode(display_id) + logical_w = CGDisplayModeGetWidth(mode) if mode else int(bounds.size.width) + pixel_w = CGDisplayModeGetPixelWidth(mode) if mode else logical_w + scale = (pixel_w / logical_w) if logical_w else 1 + out.append( + { + "index": index, + "id": int(display_id), + "main": int(display_id) == int(main), + "scale": round(scale, 4), + "bounds": { + "x": int(bounds.origin.x), + "y": int(bounds.origin.y), + "w": int(bounds.size.width), + "h": int(bounds.size.height), + }, + "pixel_w": int(pixel_w), + "pixel_h": int(round(bounds.size.height * scale)), + } + ) + return out + + +def main() -> int: + options = kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements + info = CGWindowListCopyWindowInfo(options, kCGNullWindowID) or [] + windows = [] + for entry in info: + bounds = entry.get("kCGWindowBounds") or {} + windows.append( + { + "id": int(entry.get("kCGWindowNumber", 0)), + "owner": entry.get("kCGWindowOwnerName") or "", + "name": entry.get("kCGWindowName") or "", + "layer": entry.get("kCGWindowLayer", 0), + "bounds": { + "x": int(bounds.get("X", 0)), + "y": int(bounds.get("Y", 0)), + "w": int(bounds.get("Width", 0)), + "h": int(bounds.get("Height", 0)), + }, + } + ) + print(json.dumps({"windows": windows, "displays": displays()})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/qa-video-harness.test.ts b/tests/qa-video-harness.test.ts new file mode 100644 index 00000000..a56bfa17 --- /dev/null +++ b/tests/qa-video-harness.test.ts @@ -0,0 +1,518 @@ +import { EventEmitter } from "node:events"; +import { mkdtemp, mkdir, readdir, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + DRY_RUN_SPEC, + OUTCOMES, + PROBE_SPECS, + VERDICTS, + assertFrameBudget, + buildAdjudicationManifest, + buildReportMarkdown, + combineAdjudicationArtifacts, + describeReceiptClaim, + planFrameWindow, + receiptOk, + receiptSubmitted, + reconcile, + splitAdjudicationManifest, + wallToVideoSeconds, +// @ts-expect-error -- plain .mjs helper module shared with the harness script +} from "../scripts/qa-video-lib.mjs"; + +import { + Recorder, + assertOptIn, + assertPreflightReady, + assertVideoUsable, + destroyProbeWindow, + displayContaining, + installSignalHandlers, + parseArgs, + probeWindowGeometryFromState, + pruneRunDirectories, + readExtractedFrameMapping, + rectsIntersect, + sterileEnv, +// @ts-expect-error -- executable .mjs module also exports testable runner seams +} from "../scripts/qa-video-harness.mjs"; + +function sendStep(delivery: string | undefined, overrides: Record = {}) { + return { + id: "busy-send", + marks: { send: { videoS: 12.4 }, plus2: { videoS: 14.4 } }, + context: { nonce: "QAV-BUSY-AB12", paneHint: "pane 2" }, + calls: [ + { + tool: "send_to", + args: {}, + receipt: { + ok: true, + text: "", + structured: delivery === undefined ? { ok: true } : { ok: true, delivered: true, delivery }, + }, + error: null, + }, + ], + ...overrides, + }; +} + +function runFixture(steps: unknown[]) { + return { + runId: "test-run", + mode: "full", + startedAt: "2026-08-19T00:00:00.000Z", + window: { windowRef: "window:2", workspaceRef: "workspace:13" }, + video: { path: "video.mov", fps: 15, durationS: 120, frames: 1800 }, + steps, + }; +} + +describe("qa-video probe catalogue", () => { + it("covers every fleet-reported repro the lane was chartered for", () => { + const issues = PROBE_SPECS.flatMap((spec: { issues: string[] }) => spec.issues); + for (const issue of ["#432", "#484", "#485", "#488", "#473", "#434", "#440"]) { + expect(issues).toContain(issue); + } + }); + + it("keeps the post-mark window wide enough for the measured call-to-pixel lag", () => { + // A mark is the instant of the tool CALL; the pixels lag it by the cmux + // round-trip plus a repaint. A dry-run measured that at ~1.7s for a plain + // `cmux send`, so any window centred on an action mark must reach past it. + const actionMarks = new Set(["send", "launch", "clap"]); + for (const spec of [...PROBE_SPECS, DRY_RUN_SPEC]) { + for (const question of spec.questions) { + if (!actionMarks.has(question.mark)) continue; + expect(question.window.after).toBeGreaterThanOrEqual(4); + } + } + }); + + it("asks only narrow, single-fact questions with a receipt-derived expectation", () => { + for (const spec of [...PROBE_SPECS, DRY_RUN_SPEC]) { + expect(spec.questions.length).toBeGreaterThan(0); + for (const question of spec.questions) { + expect(typeof question.text).toBe("function"); + expect(typeof question.expectedIfReceiptTrue).toBe("function"); + expect(question.window.before + question.window.after).toBeGreaterThan(0); + } + } + }); +}); + +describe("recording clock", () => { + it("maps wall clock onto the recording clock through the recorder anchor", () => { + const video = { t0WallMs: 1_000_000, t0VideoS: 0.5 }; + expect(wallToVideoSeconds(video, 1_000_000)).toBe(0.5); + expect(wallToVideoSeconds(video, 1_012_000)).toBe(12.5); + }); + + it("refuses to guess when the recorder never anchored", () => { + expect(() => wallToVideoSeconds({ t0VideoS: 0 }, 1)).toThrow(/t0WallMs/); + }); + + it("keeps the first recorder progress block as the stable clock anchor", async () => { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + stdin: { write: () => void }; + kill: () => void; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.stdin = { write: () => undefined }; + child.kill = () => undefined; + let wallMs = 10_000; + const recorder = new Recorder({ + path: "/tmp/qa-video-test.mov", + captureFps: 15, + scaleWidth: 0, + crop: null, + deviceIndex: "1", + spawnFn: () => child, + now: () => wallMs, + anchorTimeoutMs: 100, + }); + + const started = recorder.start(); + child.stdout.emit("data", "out_time_us=250000\nprogress=continue\n"); + await started; + expect({ t0WallMs: recorder.t0WallMs, t0VideoS: recorder.t0VideoS }).toEqual({ + t0WallMs: 10_000, + t0VideoS: 0.25, + }); + + wallMs = 20_000; + child.stdout.emit("data", "out_time_us=10250000\nprogress=continue\n"); + expect({ t0WallMs: recorder.t0WallMs, t0VideoS: recorder.t0VideoS }).toEqual({ + t0WallMs: 10_000, + t0VideoS: 0.25, + }); + expect(recorder.secondsAt(12_000)).toBe(2.25); + }); +}); + +describe("frame planning", () => { + it("samples densely around the mark rather than uniformly across the video", () => { + const plan = planFrameWindow({ + markSeconds: 12.4, + window: { before: 1, after: 2 }, + fps: 10, + durationS: 120, + }); + expect(plan?.start).toBe(11.4); + expect(plan?.end).toBe(14.4); + expect(plan?.count).toBe(30); + expect(plan?.times[0]).toBe(11.4); + expect(plan?.times.at(-1)).toBeCloseTo(14.3, 3); + }); + + it("clamps to the recording instead of planning frames that do not exist", () => { + const plan = planFrameWindow({ + markSeconds: 119.5, + window: { before: 1, after: 10 }, + fps: 10, + durationS: 120, + }); + expect(plan?.end).toBe(120); + }); + + it("returns null when the mark was never recorded", () => { + expect(planFrameWindow({ markSeconds: undefined, window: { before: 1, after: 1 }, fps: 10 })).toBeNull(); + }); +}); + +describe("receipt reading", () => { + it("distinguishes queued from submitted", () => { + expect(receiptSubmitted(sendStep("submitted"))).toBe(true); + expect(receiptSubmitted(sendStep("queued"))).toBe(false); + expect(receiptOk(sendStep("queued"))).toBe(true); + }); + + it("summarises the claim verbatim enough to be checkable", () => { + expect(describeReceiptClaim(sendStep("submitted"))).toContain('delivery="submitted"'); + expect(describeReceiptClaim({ id: "x", calls: [] })).toBe("no tool call recorded"); + expect( + describeReceiptClaim({ id: "x", calls: [{ tool: "send_to", error: "boom" }] }), + ).toContain("send_to threw: boom"); + }); +}); + +describe("adjudication manifest", () => { + it("emits one narrow question per fact, each carrying its own frames", () => { + const manifest = buildAdjudicationManifest(runFixture([sendStep("submitted")]), { fps: 10 }); + expect(manifest.questions.map((q: { id: string }) => q.id)).toEqual([ + "busy-send.typed", + "busy-send.submitted", + ]); + const typed = manifest.questions[0]; + expect(typed.frames.length).toBe(typed.frame_times.length); + expect(typed.frames[0]).toBe("frames/busy-send.typed/f-0001.jpg"); + expect(typed.question).toContain("QAV-BUSY-AB12"); + expect(manifest.allowed_verdicts).toEqual(VERDICTS); + }); + + it("derives the frame expectation from the receipt, so a lying receipt is what fails", () => { + const submitted = buildAdjudicationManifest(runFixture([sendStep("submitted")])); + const queued = buildAdjudicationManifest(runFixture([sendStep("queued")])); + // "submitted" implies the composer is clear two seconds later. + expect(submitted.questions[1].expected_if_receipt_true).toBe("NO"); + // "queued" implies the text is still sitting there. + expect(queued.questions[1].expected_if_receipt_true).toBe("YES"); + }); + + it("refuses to ask about frames recorded while the probe window was occluded", () => { + const occluded = { + ...sendStep("submitted"), + marks: { send: { videoS: 12.4, frontmost: false }, plus2: { videoS: 14.4, frontmost: true } }, + }; + const manifest = buildAdjudicationManifest(runFixture([occluded])); + expect(manifest.questions[0].frames).toEqual([]); + expect(manifest.questions[0].unadjudicable_reason).toContain("occluded"); + expect(manifest.questions[1].frames.length).toBeGreaterThan(0); + }); + + it("marks steps that never reached their mark as unadjudicable instead of silently dropping them", () => { + const broken = { ...sendStep("submitted"), marks: {}, error: "spawn failed" }; + const manifest = buildAdjudicationManifest(runFixture([broken])); + expect(manifest.questions[0].frames).toEqual([]); + expect(manifest.questions[0].unadjudicable_reason).toContain("spawn failed"); + }); +}); + +describe("adjudicator independence", () => { + it("keeps receipt claims structurally absent from the adjudicator questions file", () => { + const manifest = buildAdjudicationManifest(runFixture([sendStep("submitted")]), { fps: 10 }); + const { questions, expectations } = splitAdjudicationManifest(manifest); + const adjudicatorPayload = JSON.stringify(questions); + expect(adjudicatorPayload).not.toContain("receipt_claim"); + expect(adjudicatorPayload).not.toContain("expected_if_receipt_true"); + expect(adjudicatorPayload).not.toContain('delivery=\\"submitted\\"'); + expect(expectations.expectations[0]).toHaveProperty("receipt_claim"); + expect(combineAdjudicationArtifacts(questions, expectations)).toEqual(manifest); + }); + + it("rejects questions and expectations from different QA video runs", () => { + const runA = buildAdjudicationManifest({ ...runFixture([sendStep("submitted")]), runId: "run-a" }); + const runB = buildAdjudicationManifest({ ...runFixture([sendStep("submitted")]), runId: "run-b" }); + const { questions } = splitAdjudicationManifest(runA); + const { expectations } = splitAdjudicationManifest(runB); + + expect(() => combineAdjudicationArtifacts(questions, expectations)).toThrow( + /different QA video runs/, + ); + }); +}); + +describe("frame storage controls", () => { + it("rejects a run whose planned extraction exceeds the configured frame cap", () => { + const manifest = buildAdjudicationManifest(runFixture([sendStep("submitted")]), { fps: 10 }); + expect(() => assertFrameBudget(manifest, 10)).toThrow(/frame cap/i); + expect(() => assertFrameBudget(manifest, 100)).not.toThrow(); + }); + + it("reads timestamp gaps from frame PTS filenames instead of relabelling by array index", async () => { + const root = await mkdtemp(join(tmpdir(), "qa-video-frame-map-")); + try { + await writeFile(join(root, "f-0000000000.jpg"), "a"); + await writeFile(join(root, "f-0000000002.jpg"), "b"); + expect(await readExtractedFrameMapping(root, { relativeDir: "frames/q", start: 11.4, fps: 10 })).toEqual([ + { frame: "frames/q/f-0000000000.jpg", timeS: 11.4 }, + { frame: "frames/q/f-0000000002.jpg", timeS: 11.6 }, + ]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("prunes old completed runs while preserving the newest configured run and incomplete runs", async () => { + const root = await mkdtemp(join(tmpdir(), "qa-video-retention-")); + try { + const runs = [ + ["full-2026-08-18", 1], + ["full-2026-08-19", 2], + ["dry-run-2026-08-20", 3], + ] as const; + for (const [name, seconds] of runs) { + await mkdir(join(root, name)); + await writeFile(join(root, name, "run.json"), "{}\n"); + await utimes(join(root, name), seconds, seconds); + } + await mkdir(join(root, "full-in-progress")); + await writeFile(join(root, "full-in-progress", "video.mov"), "partial"); + await utimes(join(root, "full-in-progress"), 0, 0); + + expect(await pruneRunDirectories(root, { keep: 1 })).toEqual([ + "full-2026-08-19", + "full-2026-08-18", + ]); + expect((await readdir(root)).sort()).toEqual(["dry-run-2026-08-20", "full-in-progress"]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe("reconciliation", () => { + const manifest = buildAdjudicationManifest(runFixture([sendStep("submitted")])); + + it("calls a receipt-vs-frames mismatch a CONTRADICT", () => { + const { rows, totals } = reconcile(manifest, [ + { id: "busy-send.typed", verdict: "YES" }, + // Receipt said submitted, so the text should be gone; the frames show it stuck. + { id: "busy-send.submitted", verdict: "YES", frame: "frames/busy-send.submitted/f-0007.png" }, + ]); + expect(totals).toMatchObject({ AGREE: 1, CONTRADICT: 1 }); + expect(rows[1].outcome).toBe("CONTRADICT"); + }); + + it("keeps NOT_OBSERVABLE distinct from agreement", () => { + const { totals } = reconcile(manifest, [ + { id: "busy-send.typed", verdict: "NOT_OBSERVABLE", note: "pane off-screen" }, + { id: "busy-send.submitted", verdict: "NO" }, + ]); + expect(totals.NOT_OBSERVABLE).toBe(1); + expect(totals.AGREE).toBe(1); + expect(totals.CONTRADICT).toBe(0); + }); + + it("reports a partial adjudication as MISSING rather than as a pass", () => { + const { totals } = reconcile(manifest, [{ id: "busy-send.typed", verdict: "YES" }]); + expect(totals.MISSING).toBe(1); + expect(OUTCOMES).toContain("MISSING"); + }); + + it("rejects a verdict outside the allowed set", () => { + const { rows } = reconcile(manifest, [ + { id: "busy-send.typed", verdict: "probably" }, + { id: "busy-send.submitted", verdict: "NO" }, + ]); + expect(rows[0].outcome).toBe("MISSING"); + expect(rows[0].note).toContain("invalid verdict"); + }); +}); + +describe("report", () => { + it("leads with contradictions and still prints the raw receipts", () => { + const run = runFixture([sendStep("submitted")]); + const manifest = buildAdjudicationManifest(run); + const markdown = buildReportMarkdown(run, manifest, [ + { id: "busy-send.typed", verdict: "YES" }, + { id: "busy-send.submitted", verdict: "YES", frame: "frames/busy-send.submitted/f-0007.png" }, + ]); + expect(markdown).toContain("**1 CONTRADICT**"); + expect(markdown.indexOf("## Contradictions")).toBeLessThan(markdown.indexOf("## Per-probe detail")); + expect(markdown).toContain("frames/busy-send.submitted/f-0007.png"); + expect(markdown).toContain("## Raw receipts"); + expect(markdown).toContain("#432"); + }); + + it("says so out loud when nothing could be observed", () => { + const run = runFixture([{ ...sendStep("submitted"), marks: {}, error: "spawn failed" }]); + const manifest = buildAdjudicationManifest(run); + const markdown = buildReportMarkdown(run, manifest, []); + expect(markdown).toContain("## Not observable from video"); + expect(markdown).toContain("spawn failed"); + }); +}); + +describe("harness runner behaviour", () => { + it("refuses to run without an explicit live opt-in", () => { + expect(() => assertOptIn({})).toThrow(/Refusing to run the QA video harness/); + expect(() => assertOptIn({ CMUX_QA_VIDEO: "1" })).not.toThrow(); + }); + + it("refuses a full run when the recorder preflight produced no frames", () => { + expect(() => assertPreflightReady({ extracted: 0 })).toThrow( + /Refusing to run the full harness against live panes/, + ); + expect(() => assertPreflightReady({ extracted: 1 })).not.toThrow(); + }); + + it("never closes a window that existed before the run", async () => { + let closeCalls = 0; + await destroyProbeWindow( + { windowId: "window-1", preExisting: new Set(["window-1"]) }, + { cmuxFn: async () => { closeCalls += 1; }, stderr: { write: () => undefined } }, + ); + expect(closeCalls).toBe(0); + }); + + it("strips caller identity while preserving the cmux socket address", () => { + expect( + sterileEnv({ + CMUX_WORKSPACE_ID: "workspace-1", + CMUX_TAB_ID: "tab-1", + CMUX_SURFACE_ID: "surface-1", + CMUX_PANEL_ID: "panel-1", + CMUX_TERMINAL_LIFECYCLE_ID: "lifecycle-1", + CMUX_SOCKET_PATH: "/tmp/cmux.sock", + PATH: "/usr/bin", + }), + ).toEqual({ CMUX_SOCKET_PATH: "/tmp/cmux.sock", PATH: "/usr/bin" }); + }); + + it("selects the display containing the probe window centre", () => { + const displays = [ + { index: 0, main: true, bounds: { x: 0, y: 0, w: 100, h: 100 } }, + { index: 1, main: false, bounds: { x: 100, y: 0, w: 100, h: 100 } }, + ]; + expect(displayContaining(displays, { x: 120, y: 20, w: 40, h: 40 })?.index).toBe(1); + }); + + it("detects rectangle overlap without treating edge contact as occlusion", () => { + expect(rectsIntersect({ x: 0, y: 0, w: 10, h: 10 }, { x: 9, y: 9, w: 2, h: 2 })).toBe(true); + expect(rectsIntersect({ x: 0, y: 0, w: 10, h: 10 }, { x: 10, y: 0, w: 2, h: 2 })).toBe(false); + }); + + it("derives capture geometry and occlusion from a window-server snapshot", () => { + const state = { + displays: [{ index: 1, main: true, scale: 2, bounds: { x: 100, y: 50, w: 500, h: 400 } }], + windows: [ + { id: 9, owner: "Browser", name: "cover", layer: 0, bounds: { x: 140, y: 90, w: 50, h: 50 } }, + { id: 7, owner: "cmux", name: "QAV-test", layer: 0, bounds: { x: 120, y: 70, w: 100, h: 250 } }, + ], + }; + expect(probeWindowGeometryFromState(state, "QAV-test")).toMatchObject({ + id: 7, + clear: false, + occluders: ["Browser: cover"], + crop: { x: 40, y: 40, width: 200, height: 500 }, + display: { index: 1 }, + }); + }); + + it("closes the isolated window on SIGINT and SIGTERM", () => { + for (const signal of ["SIGINT", "SIGTERM"] as const) { + const signalSource = new EventEmitter(); + const calls: unknown[][] = []; + const exits: number[] = []; + const remove = installSignalHandlers( + { windowId: "window-2" }, + { + signalSource, + execFileFn: (...args: unknown[]) => { + calls.push(args); + const callback = args.at(-1); + if (typeof callback === "function") callback(); + }, + exit: (code: number) => { exits.push(code); }, + }, + ); + signalSource.emit(signal); + expect(calls[0]?.slice(0, 2)).toEqual(["cmux", ["close-window", "--window", "window-2"]]); + expect(exits).toEqual([130]); + remove(); + } + }); + + it("verifies the recording is real before deriving anything from it", () => { + expect(() => assertVideoUsable({ bytes: 1, durationS: 1, frames: 0 }, "video.mov")).toThrow( + /no decodable frames/, + ); + expect(() => assertVideoUsable({ bytes: 1, durationS: 1, frames: 1 }, "video.mov")).not.toThrow(); + }); + + it("parses runner options and validates every numeric capture setting", () => { + expect(parseArgs(["--capture-fps", "20", "--frame-fps", "5", "--scale-width", "1920"])).toMatchObject({ + captureFps: 20, + frameFps: 5, + scaleWidth: 1920, + }); + expect(() => parseArgs(["--scale-width", "abc"])).toThrow(/scaleWidth/); + }); + + it("records through ffmpeg avfoundation instead of blank window capture", async () => { + const child = new EventEmitter() as EventEmitter & { stdout: EventEmitter; stderr: EventEmitter }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + let invocation: { command: string; args: string[] } | null = null; + const recorder = new Recorder({ + path: "/tmp/qa-video-test.mov", + captureFps: 15, + scaleWidth: 0, + crop: { x: 1, y: 2, width: 3, height: 4 }, + deviceIndex: "2", + spawnFn: (command: string, args: string[]) => { + invocation = { command, args }; + return child; + }, + now: () => 1, + anchorTimeoutMs: 100, + }); + const started = recorder.start(); + child.stdout.emit("data", "out_time_us=1\n"); + await started; + expect(invocation).not.toBeNull(); + const ffmpegInvocation = invocation as unknown as { command: string; args: string[] }; + expect(ffmpegInvocation.command).toBe("ffmpeg"); + expect(ffmpegInvocation.args).toContain("avfoundation"); + expect(ffmpegInvocation.args.join(" ")).toContain("crop=3:4:1:2"); + expect(ffmpegInvocation.args.join(" ")).not.toContain("screencapture"); + }); +});