Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 64 additions & 13 deletions src/__tests__/command-doc-coverage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import assert from 'node:assert/strict';
import { describe, test } from 'vitest';
import { PUBLIC_COMMANDS, isKnownCliCommandName } from '../command-catalog.ts';
import { cliCommandAlias } from '../commands/cli-command-aliases.ts';
import { getCliCommandSchema } from '../cli-schema/command-schema.ts';
import { buildCommandUsage } from '../cli-schema/usage.ts';

// Enumerates the public command surface (`PUBLIC_COMMANDS`, derived from the
// command descriptor registry) against the human-facing command reference at
Expand All @@ -29,31 +31,54 @@ const UNDOCUMENTED_PUBLIC_COMMAND_WAIVERS: readonly CommandDocWaiver[] = [];
// companion binary). Empty today.
const UNKNOWN_DOCUMENTED_COMMAND_WAIVERS: readonly CommandDocWaiver[] = [];

// Extracts the set of `agent-device <token>` command tokens documented inside
// fenced code blocks, mapped to the 1-based line where each first appears. Prose
// mentions outside code fences are ignored on purpose: only executable usage
// lines count as "documenting" a command, so a passing reference like
// "the effective agent-device state dir" is not read as an `agent-device state`
// command.
function extractDocumentedCommandTokens(markdown: string): Map<string, number> {
const documented = new Map<string, number>();
// Yields the lines inside fenced code blocks with their 1-based line numbers.
// Prose outside code fences is skipped on purpose: only executable usage lines
// count as "documenting" a command, so a passing reference like "the effective
// agent-device state dir" is not read as an `agent-device state` command.
function* fencedLines(markdown: string): Generator<{ text: string; lineNumber: number }> {
let insideFence = false;
const lines = markdown.split('\n');
for (let index = 0; index < lines.length; index++) {
const line = lines[index] ?? '';
if (/^\s*(?:```|~~~)/.test(line)) {
const text = lines[index] ?? '';
if (/^\s*(?:```|~~~)/.test(text)) {
insideFence = !insideFence;
continue;
}
if (!insideFence) continue;
const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(line)?.[1];
if (insideFence) yield { text, lineNumber: index + 1 };
}
}

// Extracts the set of `agent-device <token>` command tokens documented inside
// fenced code blocks, mapped to the 1-based line where each first appears.
function extractDocumentedCommandTokens(markdown: string): Map<string, number> {
const documented = new Map<string, number>();
for (const { text, lineNumber } of fencedLines(markdown)) {
const token = /^\s*agent-device\s+([a-z0-9-]+)/.exec(text)?.[1];
if (token !== undefined && !documented.has(token)) {
documented.set(token, index + 1);
documented.set(token, lineNumber);
}
}
return documented;
}

// Locates a verbatim executable usage line inside a code block. Command names
// are covered by the token checks above; this is the stricter per-command gate
// that keeps a published invocation identical to the schema that produces it.
function findUsageLine(markdown: string, usageLine: string): number | undefined {
for (const { text, lineNumber } of fencedLines(markdown)) {
if (text.trim() === usageLine) return lineNumber;
}
return undefined;
}

// The canonical `snapshot` invocation, derived from the command schema rather
// than restated here: `snapshot` grew `--actions`, `--force-full`, and
// `--timeout` while the command reference still published a three-flag usage
// line, so the flag list must stay owned by the schema.
function canonicalSnapshotUsageLine(): string {
return `agent-device ${buildCommandUsage('snapshot', getCliCommandSchema('snapshot'))}`;
}

function isRegistryCommandToken(token: string): boolean {
return isKnownCliCommandName(token) || cliCommandAlias(token) !== undefined;
}
Expand Down Expand Up @@ -111,6 +136,14 @@ function undocumentedMessage(missing: readonly string[]): string {
);
}

function canonicalUsageMessage(usageLine: string): string {
return (
`${COMMANDS_DOC_PATH} does not publish the canonical usage line \`${usageLine}\`. ` +
'The command schema owns CLI syntax: update the documented code block to match it rather ' +
'than editing this expectation.'
);
}

function unknownMessage(unknown: readonly string[]): string {
return (
`${COMMANDS_DOC_PATH} documents command token(s) absent from the command registry: ` +
Expand Down Expand Up @@ -142,6 +175,15 @@ describe('command reference doc coverage', () => {
assert.deepEqual(unknown, [], unknownMessage(unknown));
});

test('commands.md publishes the canonical snapshot CLI usage', () => {
const usageLine = canonicalSnapshotUsageLine();
assert.notEqual(
findUsageLine(markdown, usageLine),
undefined,
canonicalUsageMessage(usageLine),
);
});

test('no stale waivers', () => {
assert.deepEqual(
findStaleUndocumentedWaivers(
Expand Down Expand Up @@ -216,6 +258,15 @@ describe('command reference doc coverage gate behavior', () => {
assert.match(message, /retired-command/);
});

test('a drifted snapshot usage line fails, naming file and canonical usage', () => {
const usageLine = canonicalSnapshotUsageLine();
const driftedDocs = markdown.replaceAll(usageLine, 'agent-device snapshot [-i]');
assert.equal(findUsageLine(driftedDocs, usageLine), undefined);
const message = canonicalUsageMessage(usageLine);
assert.match(message, /website\/docs\/docs\/commands\.md/);
assert.ok(message.includes(usageLine), 'the failure names the usage line the schema produces');
});

test('a waiver suppresses an intentional forward omission', () => {
const withNewCommand = [...publicCommands, 'deliberately-hidden'];
const missing = findUndocumentedPublicCommands(withNewCommand, documentedCommands, [
Expand Down
10 changes: 9 additions & 1 deletion website/docs/docs/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ agent-device close
## Snapshot and inspect

```bash
agent-device snapshot [--diff] [-i] [-d <depth>] [-s <scope>] [--raw]
agent-device snapshot [--diff] [-i] [-d <depth>] [-s <scope>] [--raw] [--actions] [--force-full] [--timeout <ms>]
agent-device diff snapshot [-i] [-d <depth>] [-s <scope>] [--raw]
agent-device get text @e1
agent-device get attrs @e1
Expand All @@ -333,6 +333,14 @@ agent-device get attrs @e1
Android interactive window roots when available, so keyboard and system-overlay nodes can appear
alongside the app root; `androidSnapshot.captureMode` and `androidSnapshot.windowCount` describe
the capture.
- `--actions` names the custom accessibility affordances an element merged away (iOS
`UIAccessibilityCustomAction`, React Native `accessibilityActions`), so a card whose reply/options
controls are not separate elements still lists them. It is iOS-simulator-only and exists for
planning, not invocation: there is no API to trigger a named action, so reach the affordance
through the element's detail screen, the same control exposed as a labeled element elsewhere, or
coordinates from its rect. It is mutually exclusive with `--raw`, which takes a capture path that
cannot carry custom actions: the pair is rejected as `INVALID_ARGS` before any device work. See
[Snapshots](/docs/snapshots) for the full constraints.
- `diff snapshot` compares the current snapshot with the previous session baseline and then updates baseline.
- `snapshot --diff` is an alias for `diff snapshot`.
- Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree.
Expand Down
55 changes: 45 additions & 10 deletions website/docs/docs/snapshots.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,37 @@ agent-device snapshot -i # Interactive elements only (recommende
agent-device snapshot -d 3 # Limit depth to 3 levels
agent-device snapshot -s "Contacts" # Scope to label/identifier
agent-device snapshot -i -d 5 # Combine options
agent-device snapshot --actions # Name custom actions merged into elements (iOS simulator)
agent-device diff snapshot # Preferred structural diff vs previous session baseline
agent-device snapshot --diff # Alias for the same diff operation
```

| Option | Description |
| ------------ | ---------------------------- |
| `-i` | Interactive-only output |
| `-d <depth>` | Limit tree depth |
| `-s <scope>` | Scope to label or identifier |
| Option | Description |
| ---------------- | ------------------------------------------------------------------------------- |
| `--diff` | Structural diff against the previous session baseline (alias for `diff snapshot`) |
| `-i` | Interactive-only output |
| `-d <depth>` | Limit tree depth |
| `-s <scope>` | Scope to label or identifier |
| `--raw` | Full provider tree instead of the visible-first agent view |
| `--actions` | Name the custom accessibility actions merged inside an element (iOS simulator) |
| `--force-full` | Re-emit the full tree even when it is unchanged since the previous snapshot |
| `--timeout <ms>` | Maximum wall-clock time for the snapshot command |

Note: If XCTest returns 0 nodes (foreground app changed), agent-device fails explicitly.
It does not automatically switch to AX.
`--actions` constraints:

- iOS simulators only. Physical iOS devices, macOS, and Android targets reject the flag.
- It names the affordances an element merged away (iOS `UIAccessibilityCustomAction`, React Native
`accessibilityActions`), so a card whose reply/options controls are not separate elements still
lists them.
- The names are for planning and discovery, not invocation. There is no API to trigger one: reach
the affordance through the element's detail screen, through the same control exposed as a labeled
element elsewhere, or by coordinates from its rect.
- Each merged element costs one accessibility round trip, so the pass is opt-in and bounded. When it
cannot read every candidate, the response says how many it read — an absent list on an unread
element is not evidence that it has none.
- Mutually exclusive with `--raw`. Custom actions are only readable through the private-AX capture
path, which the raw diagnostic strategy does not take, so the pair is rejected as `INVALID_ARGS`
before any device work — on the CLI, the Node client, and MCP alike. Choose one or the other.

## Efficient snapshot usage

Expand Down Expand Up @@ -66,7 +85,23 @@ agent-device snapshot -i
# [off-screen below] 2 interactive items: "All Contacts", "New List"
```

## Backends (iOS):
## iOS capture behavior

Capture tiers are internal. There is no flag that selects a backend; `--raw` chooses a strategy, and
the strategy owns which tiers it may use.

- `xctest` (default): full fidelity, fast, no Accessibility permission required.
- `ax`: fast accessibility tree, may miss details, requires Accessibility permission; simulator-only.
- Regular (non-`--raw`) capture uses the **regular visible strategy**: it starts with the recursive
XCTest tree, and when that returns **sparse** output for a screen XCTest cannot serialize, it can
recover through a query sweep and then, on simulators, a private accessibility backend.
- The ladder is bounded by the capture budget rather than retried indefinitely; when the budget is
spent, the best payload captured so far is returned.
- Recovery and degradation stay observable instead of being presented as an empty UI. A
**recovered** capture warns that it fell back to another backend and is safe to continue from; a
**sparse** capture reports that no backend could read the screen and points you at `screenshot`
as visual truth plus coordinate taps. Use `--json` and read `snapshotQuality` when you need the
state, backend, and reason behind **degraded** output.
- `--raw` uses the **raw diagnostic strategy**: it stays tree-first and preserves strict capture
failures, so a real XCTest accessibility serialization error surfaces as an error rather than as
an empty tree.
- Private-accessibility recovery and `--actions` reads are simulator-specific. Physical iOS devices
have no equivalent independent semantic backend; they bound the XCTest work with a probe instead.
Loading