diff --git a/src/__tests__/command-doc-coverage.test.ts b/src/__tests__/command-doc-coverage.test.ts index f8aae057e7..68f16e187b 100644 --- a/src/__tests__/command-doc-coverage.test.ts +++ b/src/__tests__/command-doc-coverage.test.ts @@ -3,6 +3,8 @@ import assert from 'node:assert/strict'; import { describe, test } from 'vitest'; import { PUBLIC_COMMANDS, isKnownCliCommandName } from '../command-catalog.ts'; import { cliCommandAlias } from '../commands/cli-command-aliases.ts'; +import { getCliCommandSchema } from '../cli-schema/command-schema.ts'; +import { buildCommandUsage } from '../cli-schema/usage.ts'; // Enumerates the public command surface (`PUBLIC_COMMANDS`, derived from the // command descriptor registry) against the human-facing command reference at @@ -29,31 +31,54 @@ const UNDOCUMENTED_PUBLIC_COMMAND_WAIVERS: readonly CommandDocWaiver[] = []; // companion binary). Empty today. const UNKNOWN_DOCUMENTED_COMMAND_WAIVERS: readonly CommandDocWaiver[] = []; -// Extracts the set of `agent-device ` command tokens documented inside -// fenced code blocks, mapped to the 1-based line where each first appears. Prose -// mentions outside code fences are ignored on purpose: only executable usage -// lines count as "documenting" a command, so a passing reference like -// "the effective agent-device state dir" is not read as an `agent-device state` -// command. -function extractDocumentedCommandTokens(markdown: string): Map { - const documented = new Map(); +// Yields the lines inside fenced code blocks with their 1-based line numbers. +// Prose outside code fences is skipped on purpose: only executable usage lines +// count as "documenting" a command, so a passing reference like "the effective +// agent-device state dir" is not read as an `agent-device state` command. +function* fencedLines(markdown: string): Generator<{ text: string; lineNumber: number }> { let insideFence = false; const lines = markdown.split('\n'); for (let index = 0; index < lines.length; index++) { - const line = lines[index] ?? ''; - if (/^\s*(?:```|~~~)/.test(line)) { + const text = lines[index] ?? ''; + if (/^\s*(?:```|~~~)/.test(text)) { insideFence = !insideFence; continue; } - if (!insideFence) continue; - const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(line)?.[1]; + if (insideFence) yield { text, lineNumber: index + 1 }; + } +} + +// Extracts the set of `agent-device ` command tokens documented inside +// fenced code blocks, mapped to the 1-based line where each first appears. +function extractDocumentedCommandTokens(markdown: string): Map { + const documented = new Map(); + for (const { text, lineNumber } of fencedLines(markdown)) { + const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(text)?.[1]; if (token !== undefined && !documented.has(token)) { - documented.set(token, index + 1); + documented.set(token, lineNumber); } } return documented; } +// Locates a verbatim executable usage line inside a code block. Command names +// are covered by the token checks above; this is the stricter per-command gate +// that keeps a published invocation identical to the schema that produces it. +function findUsageLine(markdown: string, usageLine: string): number | undefined { + for (const { text, lineNumber } of fencedLines(markdown)) { + if (text.trim() === usageLine) return lineNumber; + } + return undefined; +} + +// The canonical `snapshot` invocation, derived from the command schema rather +// than restated here: `snapshot` grew `--actions`, `--force-full`, and +// `--timeout` while the command reference still published a three-flag usage +// line, so the flag list must stay owned by the schema. +function canonicalSnapshotUsageLine(): string { + return `agent-device ${buildCommandUsage('snapshot', getCliCommandSchema('snapshot'))}`; +} + function isRegistryCommandToken(token: string): boolean { return isKnownCliCommandName(token) || cliCommandAlias(token) !== undefined; } @@ -111,6 +136,14 @@ function undocumentedMessage(missing: readonly string[]): string { ); } +function canonicalUsageMessage(usageLine: string): string { + return ( + `${COMMANDS_DOC_PATH} does not publish the canonical usage line \`${usageLine}\`. ` + + 'The command schema owns CLI syntax: update the documented code block to match it rather ' + + 'than editing this expectation.' + ); +} + function unknownMessage(unknown: readonly string[]): string { return ( `${COMMANDS_DOC_PATH} documents command token(s) absent from the command registry: ` + @@ -142,6 +175,15 @@ describe('command reference doc coverage', () => { assert.deepEqual(unknown, [], unknownMessage(unknown)); }); + test('commands.md publishes the canonical snapshot CLI usage', () => { + const usageLine = canonicalSnapshotUsageLine(); + assert.notEqual( + findUsageLine(markdown, usageLine), + undefined, + canonicalUsageMessage(usageLine), + ); + }); + test('no stale waivers', () => { assert.deepEqual( findStaleUndocumentedWaivers( @@ -216,6 +258,15 @@ describe('command reference doc coverage gate behavior', () => { assert.match(message, /retired-command/); }); + test('a drifted snapshot usage line fails, naming file and canonical usage', () => { + const usageLine = canonicalSnapshotUsageLine(); + const driftedDocs = markdown.replaceAll(usageLine, 'agent-device snapshot [-i]'); + assert.equal(findUsageLine(driftedDocs, usageLine), undefined); + const message = canonicalUsageMessage(usageLine); + assert.match(message, /website\/docs\/docs\/commands\.md/); + assert.ok(message.includes(usageLine), 'the failure names the usage line the schema produces'); + }); + test('a waiver suppresses an intentional forward omission', () => { const withNewCommand = [...publicCommands, 'deliberately-hidden']; const missing = findUndocumentedPublicCommands(withNewCommand, documentedCommands, [ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 05450307bd..7cc59b7a8c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -316,7 +316,7 @@ agent-device close ## Snapshot and inspect ```bash -agent-device snapshot [--diff] [-i] [-d ] [-s ] [--raw] +agent-device snapshot [--diff] [-i] [-d ] [-s ] [--raw] [--actions] [--force-full] [--timeout ] agent-device diff snapshot [-i] [-d ] [-s ] [--raw] agent-device get text @e1 agent-device get attrs @e1 @@ -333,6 +333,14 @@ agent-device get attrs @e1 Android interactive window roots when available, so keyboard and system-overlay nodes can appear alongside the app root; `androidSnapshot.captureMode` and `androidSnapshot.windowCount` describe the capture. +- `--actions` names the custom accessibility affordances an element merged away (iOS + `UIAccessibilityCustomAction`, React Native `accessibilityActions`), so a card whose reply/options + controls are not separate elements still lists them. It is iOS-simulator-only and exists for + planning, not invocation: there is no API to trigger a named action, so reach the affordance + through the element's detail screen, the same control exposed as a labeled element elsewhere, or + coordinates from its rect. It is mutually exclusive with `--raw`, which takes a capture path that + cannot carry custom actions: the pair is rejected as `INVALID_ARGS` before any device work. See + [Snapshots](/docs/snapshots) for the full constraints. - `diff snapshot` compares the current snapshot with the previous session baseline and then updates baseline. - `snapshot --diff` is an alias for `diff snapshot`. - Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index efa41c68b4..61b743ce45 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -12,18 +12,37 @@ agent-device snapshot -i # Interactive elements only (recommende agent-device snapshot -d 3 # Limit depth to 3 levels agent-device snapshot -s "Contacts" # Scope to label/identifier agent-device snapshot -i -d 5 # Combine options +agent-device snapshot --actions # Name custom actions merged into elements (iOS simulator) agent-device diff snapshot # Preferred structural diff vs previous session baseline agent-device snapshot --diff # Alias for the same diff operation ``` -| Option | Description | -| ------------ | ---------------------------- | -| `-i` | Interactive-only output | -| `-d ` | Limit tree depth | -| `-s ` | Scope to label or identifier | +| Option | Description | +| ---------------- | ------------------------------------------------------------------------------- | +| `--diff` | Structural diff against the previous session baseline (alias for `diff snapshot`) | +| `-i` | Interactive-only output | +| `-d ` | Limit tree depth | +| `-s ` | Scope to label or identifier | +| `--raw` | Full provider tree instead of the visible-first agent view | +| `--actions` | Name the custom accessibility actions merged inside an element (iOS simulator) | +| `--force-full` | Re-emit the full tree even when it is unchanged since the previous snapshot | +| `--timeout ` | Maximum wall-clock time for the snapshot command | -Note: If XCTest returns 0 nodes (foreground app changed), agent-device fails explicitly. -It does not automatically switch to AX. +`--actions` constraints: + +- iOS simulators only. Physical iOS devices, macOS, and Android targets reject the flag. +- It names the affordances an element merged away (iOS `UIAccessibilityCustomAction`, React Native + `accessibilityActions`), so a card whose reply/options controls are not separate elements still + lists them. +- The names are for planning and discovery, not invocation. There is no API to trigger one: reach + the affordance through the element's detail screen, through the same control exposed as a labeled + element elsewhere, or by coordinates from its rect. +- Each merged element costs one accessibility round trip, so the pass is opt-in and bounded. When it + cannot read every candidate, the response says how many it read — an absent list on an unread + element is not evidence that it has none. +- Mutually exclusive with `--raw`. Custom actions are only readable through the private-AX capture + path, which the raw diagnostic strategy does not take, so the pair is rejected as `INVALID_ARGS` + before any device work — on the CLI, the Node client, and MCP alike. Choose one or the other. ## Efficient snapshot usage @@ -66,7 +85,23 @@ agent-device snapshot -i # [off-screen below] 2 interactive items: "All Contacts", "New List" ``` -## Backends (iOS): +## iOS capture behavior + +Capture tiers are internal. There is no flag that selects a backend; `--raw` chooses a strategy, and +the strategy owns which tiers it may use. -- `xctest` (default): full fidelity, fast, no Accessibility permission required. -- `ax`: fast accessibility tree, may miss details, requires Accessibility permission; simulator-only. +- Regular (non-`--raw`) capture uses the **regular visible strategy**: it starts with the recursive + XCTest tree, and when that returns **sparse** output for a screen XCTest cannot serialize, it can + recover through a query sweep and then, on simulators, a private accessibility backend. +- The ladder is bounded by the capture budget rather than retried indefinitely; when the budget is + spent, the best payload captured so far is returned. +- Recovery and degradation stay observable instead of being presented as an empty UI. A + **recovered** capture warns that it fell back to another backend and is safe to continue from; a + **sparse** capture reports that no backend could read the screen and points you at `screenshot` + as visual truth plus coordinate taps. Use `--json` and read `snapshotQuality` when you need the + state, backend, and reason behind **degraded** output. +- `--raw` uses the **raw diagnostic strategy**: it stays tree-first and preserves strict capture + failures, so a real XCTest accessibility serialization error surfaces as an error rather than as + an empty tree. +- Private-accessibility recovery and `--actions` reads are simulator-specific. Physical iOS devices + have no equivalent independent semantic backend; they bound the XCTest work with a probe instead.