From 143baefc9eea0279e9570a48d83a5fb5f5354de5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 12:36:06 +0000 Subject: [PATCH 1/2] docs: align snapshot fallback and actions guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The snapshot guide claimed a zero-node XCTest result fails without ever switching to AX, but regular iOS capture has an explicit recursive-tree → query-sweep → private-AX recovery plan (ADR 0004, RunnerTests+SnapshotCapturePlan.swift). The public CLI also exposes `--actions`, `--force-full`, and `--timeout`, while both website reference pages published a three-flag snapshot usage line. - Add a schema-derived gate: `commands.md` must publish the exact usage `buildCommandUsage('snapshot', getCliCommandSchema('snapshot'))` produces, so the canonical invocation cannot drift from the command schema again. The flag list is never restated in the test. Proven red against the pre-fix `commands.md`. - Extract the fence walker both doc checks now share, and prove the new gate fails on a planted usage drift. - Publish the canonical snapshot usage in the command reference and describe `--actions` as iOS-simulator-only and planning-only. - Replace the "Backends (iOS)" list with an iOS capture behavior section written from ADR 0004 and the live capture plan: regular visible strategy with a bounded recovery ladder, raw diagnostic strategy preserving strict capture failures, and recovered/sparse/degraded output staying observable through quality warnings. Capture tiers are documented as internal, not as user-selectable backends. Custom-action discovery stays separate from invocation: the runner can read names but cannot trigger them (RunnerAXSnapshotBridge.h), so neither page implies otherwise. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014VzyMim5q4jbp1xDMm3jVo --- src/__tests__/command-doc-coverage.test.ts | 77 ++++++++++++++++++---- website/docs/docs/commands.md | 8 ++- website/docs/docs/snapshots.md | 55 +++++++++++++--- 3 files changed, 116 insertions(+), 24 deletions(-) diff --git a/src/__tests__/command-doc-coverage.test.ts b/src/__tests__/command-doc-coverage.test.ts index f8aae057e7..68f16e187b 100644 --- a/src/__tests__/command-doc-coverage.test.ts +++ b/src/__tests__/command-doc-coverage.test.ts @@ -3,6 +3,8 @@ import assert from 'node:assert/strict'; import { describe, test } from 'vitest'; import { PUBLIC_COMMANDS, isKnownCliCommandName } from '../command-catalog.ts'; import { cliCommandAlias } from '../commands/cli-command-aliases.ts'; +import { getCliCommandSchema } from '../cli-schema/command-schema.ts'; +import { buildCommandUsage } from '../cli-schema/usage.ts'; // Enumerates the public command surface (`PUBLIC_COMMANDS`, derived from the // command descriptor registry) against the human-facing command reference at @@ -29,31 +31,54 @@ const UNDOCUMENTED_PUBLIC_COMMAND_WAIVERS: readonly CommandDocWaiver[] = []; // companion binary). Empty today. const UNKNOWN_DOCUMENTED_COMMAND_WAIVERS: readonly CommandDocWaiver[] = []; -// Extracts the set of `agent-device ` command tokens documented inside -// fenced code blocks, mapped to the 1-based line where each first appears. Prose -// mentions outside code fences are ignored on purpose: only executable usage -// lines count as "documenting" a command, so a passing reference like -// "the effective agent-device state dir" is not read as an `agent-device state` -// command. -function extractDocumentedCommandTokens(markdown: string): Map { - const documented = new Map(); +// Yields the lines inside fenced code blocks with their 1-based line numbers. +// Prose outside code fences is skipped on purpose: only executable usage lines +// count as "documenting" a command, so a passing reference like "the effective +// agent-device state dir" is not read as an `agent-device state` command. +function* fencedLines(markdown: string): Generator<{ text: string; lineNumber: number }> { let insideFence = false; const lines = markdown.split('\n'); for (let index = 0; index < lines.length; index++) { - const line = lines[index] ?? ''; - if (/^\s*(?:```|~~~)/.test(line)) { + const text = lines[index] ?? ''; + if (/^\s*(?:```|~~~)/.test(text)) { insideFence = !insideFence; continue; } - if (!insideFence) continue; - const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(line)?.[1]; + if (insideFence) yield { text, lineNumber: index + 1 }; + } +} + +// Extracts the set of `agent-device ` command tokens documented inside +// fenced code blocks, mapped to the 1-based line where each first appears. +function extractDocumentedCommandTokens(markdown: string): Map { + const documented = new Map(); + for (const { text, lineNumber } of fencedLines(markdown)) { + const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(text)?.[1]; if (token !== undefined && !documented.has(token)) { - documented.set(token, index + 1); + documented.set(token, lineNumber); } } return documented; } +// Locates a verbatim executable usage line inside a code block. Command names +// are covered by the token checks above; this is the stricter per-command gate +// that keeps a published invocation identical to the schema that produces it. +function findUsageLine(markdown: string, usageLine: string): number | undefined { + for (const { text, lineNumber } of fencedLines(markdown)) { + if (text.trim() === usageLine) return lineNumber; + } + return undefined; +} + +// The canonical `snapshot` invocation, derived from the command schema rather +// than restated here: `snapshot` grew `--actions`, `--force-full`, and +// `--timeout` while the command reference still published a three-flag usage +// line, so the flag list must stay owned by the schema. +function canonicalSnapshotUsageLine(): string { + return `agent-device ${buildCommandUsage('snapshot', getCliCommandSchema('snapshot'))}`; +} + function isRegistryCommandToken(token: string): boolean { return isKnownCliCommandName(token) || cliCommandAlias(token) !== undefined; } @@ -111,6 +136,14 @@ function undocumentedMessage(missing: readonly string[]): string { ); } +function canonicalUsageMessage(usageLine: string): string { + return ( + `${COMMANDS_DOC_PATH} does not publish the canonical usage line \`${usageLine}\`. ` + + 'The command schema owns CLI syntax: update the documented code block to match it rather ' + + 'than editing this expectation.' + ); +} + function unknownMessage(unknown: readonly string[]): string { return ( `${COMMANDS_DOC_PATH} documents command token(s) absent from the command registry: ` + @@ -142,6 +175,15 @@ describe('command reference doc coverage', () => { assert.deepEqual(unknown, [], unknownMessage(unknown)); }); + test('commands.md publishes the canonical snapshot CLI usage', () => { + const usageLine = canonicalSnapshotUsageLine(); + assert.notEqual( + findUsageLine(markdown, usageLine), + undefined, + canonicalUsageMessage(usageLine), + ); + }); + test('no stale waivers', () => { assert.deepEqual( findStaleUndocumentedWaivers( @@ -216,6 +258,15 @@ describe('command reference doc coverage gate behavior', () => { assert.match(message, /retired-command/); }); + test('a drifted snapshot usage line fails, naming file and canonical usage', () => { + const usageLine = canonicalSnapshotUsageLine(); + const driftedDocs = markdown.replaceAll(usageLine, 'agent-device snapshot [-i]'); + assert.equal(findUsageLine(driftedDocs, usageLine), undefined); + const message = canonicalUsageMessage(usageLine); + assert.match(message, /website\/docs\/docs\/commands\.md/); + assert.ok(message.includes(usageLine), 'the failure names the usage line the schema produces'); + }); + test('a waiver suppresses an intentional forward omission', () => { const withNewCommand = [...publicCommands, 'deliberately-hidden']; const missing = findUndocumentedPublicCommands(withNewCommand, documentedCommands, [ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 05450307bd..38673129dd 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -316,7 +316,7 @@ agent-device close ## Snapshot and inspect ```bash -agent-device snapshot [--diff] [-i] [-d ] [-s ] [--raw] +agent-device snapshot [--diff] [-i] [-d ] [-s ] [--raw] [--actions] [--force-full] [--timeout ] agent-device diff snapshot [-i] [-d ] [-s ] [--raw] agent-device get text @e1 agent-device get attrs @e1 @@ -333,6 +333,12 @@ agent-device get attrs @e1 Android interactive window roots when available, so keyboard and system-overlay nodes can appear alongside the app root; `androidSnapshot.captureMode` and `androidSnapshot.windowCount` describe the capture. +- `--actions` names the custom accessibility affordances an element merged away (iOS + `UIAccessibilityCustomAction`, React Native `accessibilityActions`), so a card whose reply/options + controls are not separate elements still lists them. It is iOS-simulator-only and exists for + planning, not invocation: there is no API to trigger a named action, so reach the affordance + through the element's detail screen, the same control exposed as a labeled element elsewhere, or + coordinates from its rect. See [Snapshots](/docs/snapshots) for the full constraints. - `diff snapshot` compares the current snapshot with the previous session baseline and then updates baseline. - `snapshot --diff` is an alias for `diff snapshot`. - Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index efa41c68b4..406f1b1f7d 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -12,18 +12,37 @@ agent-device snapshot -i # Interactive elements only (recommende agent-device snapshot -d 3 # Limit depth to 3 levels agent-device snapshot -s "Contacts" # Scope to label/identifier agent-device snapshot -i -d 5 # Combine options +agent-device snapshot --actions # Name custom actions merged into elements (iOS simulator) agent-device diff snapshot # Preferred structural diff vs previous session baseline agent-device snapshot --diff # Alias for the same diff operation ``` -| Option | Description | -| ------------ | ---------------------------- | -| `-i` | Interactive-only output | -| `-d ` | Limit tree depth | -| `-s ` | Scope to label or identifier | +| Option | Description | +| ---------------- | ------------------------------------------------------------------------------- | +| `--diff` | Structural diff against the previous session baseline (alias for `diff snapshot`) | +| `-i` | Interactive-only output | +| `-d ` | Limit tree depth | +| `-s ` | Scope to label or identifier | +| `--raw` | Full provider tree instead of the visible-first agent view | +| `--actions` | Name the custom accessibility actions merged inside an element (iOS simulator) | +| `--force-full` | Re-emit the full tree even when it is unchanged since the previous snapshot | +| `--timeout ` | Maximum wall-clock time for the snapshot command | -Note: If XCTest returns 0 nodes (foreground app changed), agent-device fails explicitly. -It does not automatically switch to AX. +`--actions` constraints: + +- iOS simulators only. Physical iOS devices, macOS, and Android targets reject the flag. +- It names the affordances an element merged away (iOS `UIAccessibilityCustomAction`, React Native + `accessibilityActions`), so a card whose reply/options controls are not separate elements still + lists them. +- The names are for planning and discovery, not invocation. There is no API to trigger one: reach + the affordance through the element's detail screen, through the same control exposed as a labeled + element elsewhere, or by coordinates from its rect. +- Each merged element costs one accessibility round trip, so the pass is opt-in and bounded. When it + cannot read every candidate, the response says how many it read — an absent list on an unread + element is not evidence that it has none. +- Do not combine it with `--raw`. The raw diagnostic strategy stays tree-first, and only the private + accessibility backend can read custom actions, so `--raw --actions` returns a raw tree without + them. ## Efficient snapshot usage @@ -66,7 +85,23 @@ agent-device snapshot -i # [off-screen below] 2 interactive items: "All Contacts", "New List" ``` -## Backends (iOS): +## iOS capture behavior + +Capture tiers are internal. There is no flag that selects a backend; `--raw` chooses a strategy, and +the strategy owns which tiers it may use. -- `xctest` (default): full fidelity, fast, no Accessibility permission required. -- `ax`: fast accessibility tree, may miss details, requires Accessibility permission; simulator-only. +- Regular (non-`--raw`) capture uses the **regular visible strategy**: it starts with the recursive + XCTest tree, and when that returns **sparse** output for a screen XCTest cannot serialize, it can + recover through a query sweep and then, on simulators, a private accessibility backend. +- The ladder is bounded by the capture budget rather than retried indefinitely; when the budget is + spent, the best payload captured so far is returned. +- Recovery and degradation stay observable instead of being presented as an empty UI. A + **recovered** capture warns that it fell back to another backend and is safe to continue from; a + **sparse** capture reports that no backend could read the screen and points you at `screenshot` + as visual truth plus coordinate taps. Use `--json` and read `snapshotQuality` when you need the + state, backend, and reason behind **degraded** output. +- `--raw` uses the **raw diagnostic strategy**: it stays tree-first and preserves strict capture + failures, so a real XCTest accessibility serialization error surfaces as an error rather than as + an empty tree. +- Private-accessibility recovery and `--actions` reads are simulator-specific. Physical iOS devices + have no equivalent independent semantic backend; they bound the XCTest work with a probe instead. From aef59fdd8784afb05f7ff22536a1542efdc1da1d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 13:29:11 +0000 Subject: [PATCH 2/2] docs: state that --actions and --raw are rejected as a pair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pages described the combination as a silent no-op ("returns a raw tree without them"), which cannot happen: `customActionFlagsResponse` in src/daemon/request-router.ts rejects `snapshotCustomActions` + `snapshotRaw` with INVALID_ARGS at the shared request seam, before any session or device work, so CLI, Node client, and MCP all get the same answer. Pinned by src/daemon/__tests__/request-router-custom-action-flags.test.ts. The underlying reason was right and is kept — custom actions are only readable through the private-AX capture path, which the raw diagnostic strategy does not take — but the user-visible outcome is a rejection, not a degraded capture, so both pages now say to choose one flag or the other. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014VzyMim5q4jbp1xDMm3jVo --- website/docs/docs/commands.md | 4 +++- website/docs/docs/snapshots.md | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 38673129dd..7cc59b7a8c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -338,7 +338,9 @@ agent-device get attrs @e1 controls are not separate elements still lists them. It is iOS-simulator-only and exists for planning, not invocation: there is no API to trigger a named action, so reach the affordance through the element's detail screen, the same control exposed as a labeled element elsewhere, or - coordinates from its rect. See [Snapshots](/docs/snapshots) for the full constraints. + coordinates from its rect. It is mutually exclusive with `--raw`, which takes a capture path that + cannot carry custom actions: the pair is rejected as `INVALID_ARGS` before any device work. See + [Snapshots](/docs/snapshots) for the full constraints. - `diff snapshot` compares the current snapshot with the previous session baseline and then updates baseline. - `snapshot --diff` is an alias for `diff snapshot`. - Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 406f1b1f7d..61b743ce45 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -40,9 +40,9 @@ agent-device snapshot --diff # Alias for the same diff operation - Each merged element costs one accessibility round trip, so the pass is opt-in and bounded. When it cannot read every candidate, the response says how many it read — an absent list on an unread element is not evidence that it has none. -- Do not combine it with `--raw`. The raw diagnostic strategy stays tree-first, and only the private - accessibility backend can read custom actions, so `--raw --actions` returns a raw tree without - them. +- Mutually exclusive with `--raw`. Custom actions are only readable through the private-AX capture + path, which the raw diagnostic strategy does not take, so the pair is rejected as `INVALID_ARGS` + before any device work — on the CLI, the Node client, and MCP alike. Choose one or the other. ## Efficient snapshot usage