diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts index 19c547b789..53cd7d612d 100644 --- a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts @@ -14,45 +14,67 @@ import { captureThrows, scrollSnapshot, windowRoot } from './scroll-edge-state-f test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (bottom)', () => { assert.equal( - formatScrollEdgeMessage('down', 'bottom', 0, undefined, undefined), + formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 0 }), 'Already at bottom; no hidden content below detected', ); }); test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (top)', () => { assert.equal( - formatScrollEdgeMessage('up', 'top', 0, undefined, undefined), + formatScrollEdgeMessage({ direction: 'up', edge: 'top', passes: 0 }), 'Already at top; no hidden content above detected', ); }); test('formatScrollEdgeMessage: edge reached after N passes', () => { assert.equal( - formatScrollEdgeMessage('down', 'bottom', 4, undefined, undefined), + formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 4 }), 'Scrolled to bottom with 4 down passes', ); }); test('formatScrollEdgeMessage: no edge, pixel amount given', () => { assert.equal( - formatScrollEdgeMessage('down', undefined, 0, undefined, 250), + formatScrollEdgeMessage({ direction: 'down', passes: 0, pixels: 250 }), 'Scrolled down by 250px', ); }); test('formatScrollEdgeMessage: no edge, no pixels, symbolic amount given', () => { - assert.equal(formatScrollEdgeMessage('up', undefined, 0, 3, undefined), 'Scrolled up by 3'); + assert.equal( + formatScrollEdgeMessage({ direction: 'up', passes: 0, amount: 3 }), + 'Scrolled up by 3', + ); }); test('formatScrollEdgeMessage: no edge, no pixels, no amount falls back to bare direction', () => { + assert.equal(formatScrollEdgeMessage({ direction: 'left', passes: 0 }), 'Scrolled left'); +}); + +test('formatScrollEdgeMessage: pixels takes priority over amount when both are set', () => { assert.equal( - formatScrollEdgeMessage('left', undefined, 0, undefined, undefined), - 'Scrolled left', + formatScrollEdgeMessage({ direction: 'down', passes: 0, amount: 3, pixels: 250 }), + 'Scrolled down by 250px', ); }); -test('formatScrollEdgeMessage: pixels takes priority over amount when both are set', () => { - assert.equal(formatScrollEdgeMessage('down', undefined, 0, 3, 250), 'Scrolled down by 250px'); +/** + * One gesture saturates at the viewport axis minus its edge padding, so a large amount buys less + * travel than it names. The message reports what the planner honored rather than what was asked. + */ +test('an amount-based message names the honored travel when the planner reports it', () => { + assert.equal( + formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 3, honoredPixels: 640 }), + 'Scrolled down by 3 of the viewport (640px)', + ); + assert.equal( + formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 0.65 }), + 'Scrolled down by 0.65', + ); + assert.equal( + formatScrollEdgeMessage({ direction: 'down', passes: 1, pixels: 5000, honoredPixels: 640 }), + 'Scrolled down by 640px', + ); }); // --------------------------------------------------------------------------- @@ -279,7 +301,7 @@ test('runScrollEdgePasses: throws a COMMAND_FAILED AppError once the pass limit 'scroll bottom reached the safety limit before the snapshot showed the edge', ); assert.deepEqual(error.details, { - hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.', + hint: 'The scoped scroll container still reports hidden content. Run scroll --until to stop on the element you are after, or snapshot -i to inspect the current state.', }); return true; }, diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index f2d7e2a4d3..98c8c6e971 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -37,6 +37,18 @@ export async function captureScrollEdgeState(params: { } } +/** + * Is there hidden content left at this edge? The same question `runScrollEdgePasses` loops on, + * exposed for callers with their own stop condition (`scroll --until`) so both read one signal. + */ +export async function canScrollFurtherAtEdge( + nodes: readonly (RawSnapshotNode | SnapshotNode)[], + edge: ScrollEdge, +): Promise { + const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts'); + return analyzeScrollEdgeState(nodes, edge).canScroll; +} + export async function runScrollEdgePasses(params: { edge: ScrollEdge; captureState: (scope?: string) => Promise; @@ -56,7 +68,7 @@ export async function runScrollEdgePasses(params: { 'COMMAND_FAILED', `scroll ${edge} reached the safety limit before the snapshot showed the edge`, { - hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.', + hint: 'The scoped scroll container still reports hidden content. Run scroll --until to stop on the element you are after, or snapshot -i to inspect the current state.', }, ); } @@ -69,19 +81,31 @@ export async function runScrollEdgePasses(params: { return { passes, result }; } -export function formatScrollEdgeMessage( - direction: ScrollDirection, - edge: ScrollEdge | undefined, - passes: number, - amount: number | undefined, - pixels: number | undefined, -): string { +/** + * `honoredPixels` is the travel the gesture planner actually produced, which is not always the + * travel that was asked for: one gesture cannot cross more than the viewport axis minus its edge + * padding, so a large `amount` saturates. Naming the honored distance is what keeps + * `scroll down 3` from reporting a three-viewport scroll it never performed. + */ +export function formatScrollEdgeMessage(params: { + direction: ScrollDirection; + edge?: ScrollEdge | undefined; + passes: number; + amount?: number | undefined; + pixels?: number | undefined; + honoredPixels?: number | undefined; +}): string { + const { direction, edge, passes, amount, pixels, honoredPixels } = params; if (edge && passes === 0) { return `Already at ${edge}; no hidden content ${edge === 'bottom' ? 'below' : 'above'} detected`; } if (edge) return `Scrolled to ${edge} with ${passes} ${direction} passes`; - if (pixels !== undefined) return `Scrolled ${direction} by ${pixels}px`; - if (amount !== undefined) return `Scrolled ${direction} by ${amount}`; + if (pixels !== undefined) return `Scrolled ${direction} by ${honoredPixels ?? pixels}px`; + if (amount !== undefined) { + return honoredPixels === undefined + ? `Scrolled ${direction} by ${amount}` + : `Scrolled ${direction} by ${amount} of the viewport (${honoredPixels}px)`; + } return `Scrolled ${direction}`; } diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index 0764ee8895..c5ba3f1649 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -109,6 +109,8 @@ export type CliFlags = CloudProviderProfileFields & holdMs?: number; jitterPx?: number; pixels?: number; + /** Scroll: repeat passes until this selector is visible on screen. */ + until?: string; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 8206c6e769..5928b727a9 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -146,4 +146,6 @@ export type ScrollOptions = DeviceCommandBaseOptions & amount?: number; pixels?: number; durationMs?: number; + /** Repeat scroll passes until this selector is visible on screen, then stop. */ + until?: string; }; diff --git a/packages/contracts/src/client-request.ts b/packages/contracts/src/client-request.ts index a415a67c68..81f8b25321 100644 --- a/packages/contracts/src/client-request.ts +++ b/packages/contracts/src/client-request.ts @@ -39,6 +39,8 @@ export type CommandExecutionOptions = Partial & holdMs?: number; jitterPx?: number; pixels?: number; + /** Scroll: repeat passes until this selector is visible on screen. */ + until?: string; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 12aa3605ee..5f85b87a5c 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -249,12 +249,14 @@ export const gestureViewportRuntimeUse = defineUse({ required: ['gestureViewport /** `scroll ` executes one pass and needs nothing else. */ const scrollDirectionUse = defineUse({ required: ['scrollDirection'] }); /** - * `scroll top` / `scroll bottom` verify hidden content between passes, so the capture is part of - * the tier's requirement rather than something discovered mid-run — the retired leaf's + * Every scroll that verifies between passes: `scroll top`/`scroll bottom` read hidden content at + * the edge, and `scroll --until ` re-reads the tree to decide whether the target came + * into view. Both check against that capture rather than a stale session snapshot, so the capture + * is part of the tier's requirement rather than something discovered mid-run — the retired leaf's * "requires snapshot support to verify hidden content before scrolling" refusal, moved to - * admission. + * admission. One declaration, because the two tiers admit on identical facts. */ -const scrollEdgeUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); +const scrollVerifiedPassUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); const gestureUsesByTier = Object.freeze({ plan: gesturePlanUse, @@ -275,7 +277,10 @@ export const gestureRuntimePlanUses = Object.freeze([ export const swipeRuntimePlanUses = Object.freeze([gesturePlanUse] as const); /** Every use `scroll` can select between. */ -export const scrollRuntimePlanUses = Object.freeze([scrollDirectionUse, scrollEdgeUse] as const); +export const scrollRuntimePlanUses = Object.freeze([ + scrollDirectionUse, + scrollVerifiedPassUse, +] as const); type GesturePlanFor = Readonly<{ tier: Tier; @@ -337,15 +342,28 @@ function gesturePlan(tier: Tier): Gesture */ export type ScrollRuntimePlan = | Readonly<{ kind: 'direction'; use: typeof scrollDirectionUse }> - | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollEdgeUse }>; + | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollVerifiedPassUse }> + | Readonly<{ kind: 'until'; until: string; use: typeof scrollVerifiedPassUse }>; -/** `scroll top`/`scroll bottom` verify between passes; every other scroll executes one pass. */ +/** + * `scroll top`/`scroll bottom` verify between passes, `scroll --until` re-reads the tree between + * passes to check its selector, and every other scroll executes one pass. The edge directions carry + * their own stop condition, so pairing them with `--until` names two, which the caller rejects + * before this resolves. + */ export function resolveScrollRuntimePlan( - input: Readonly<{ edge?: 'top' | 'bottom' }>, + input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>, ): ScrollRuntimePlan { + if (input.until !== undefined) { + return Object.freeze({ + kind: 'until', + until: input.until, + use: scrollVerifiedPassUse, + } as const); + } return input.edge === undefined ? Object.freeze({ kind: 'direction', use: scrollDirectionUse } as const) - : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollEdgeUse } as const); + : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollVerifiedPassUse } as const); } const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index f0632d2a59..73b27385bb 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -47,6 +47,24 @@ export function assertExclusiveScrollDistanceInputs( } } +/** + * `top`/`bottom` are scroll-to-extreme requests that already carry a stop condition, so pairing one + * with `--until` names two and the request has no single meaning. Rejected at the surface rather + * than resolved by precedence, so neither stop condition can silently win. + */ +export function assertScrollUntilCompatible( + input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>, +): void { + if (input.until === undefined || input.edge === undefined) return; + throw new AppError( + 'INVALID_ARGS', + `scroll ${input.edge} already scrolls to the ${input.edge} edge and cannot take --until`, + { + hint: `Use scroll ${input.edge === 'bottom' ? 'down' : 'up'} --until to stop at the target, or scroll ${input.edge} to reach the edge.`, + }, + ); +} + export function normalizeScrollDurationMs( durationMs: number | undefined, options: { field?: string; invalidMessage?: string; max?: number } = {}, @@ -64,6 +82,13 @@ export function normalizeScrollDurationMs( return durationMs; } +/** The travel the planner produced, which saturates below a large requested amount. */ +export function honoredScrollPixels( + result: Record | undefined, +): number | undefined { + return typeof result?.pixels === 'number' ? result.pixels : undefined; +} + export function honoredScrollDurationMs( result: Record | undefined, ): number | undefined { @@ -84,7 +109,9 @@ export type ScrollCommandResult = { direction: ScrollDirection; /** Set for `top`/`bottom` requests: the extreme being scrolled to. */ edge?: 'top' | 'bottom'; - /** Edge scrolls only: how many scroll-and-check passes ran. */ + /** Set for `--until` requests: the selector the passes stopped on. */ + until?: string; + /** Edge and until scrolls only: how many scroll-and-check passes ran. */ passes?: number; amount?: number; pixels?: number; diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 6d09cfac42..cdc7ddc971 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -83,7 +83,13 @@ export type InPageSwipeGesturePlan = { referenceHeight: number; }; -const DEFAULT_SCROLL_AMOUNT = 0.6; +/** + * The finger-path fraction of the viewport axis one scroll covers when the caller names no + * distance. Exported because a backend with no viewport to measure against (the browser) scales its + * own default step by the ratio to this, and that ratio is only meaningful while both sides read + * the same number. + */ +export const DEFAULT_SCROLL_AMOUNT = 0.6; // Scroll gestures never touch the outer 10% of either axis. Modern app windows are edge-to-edge, // so the viewport includes the system bars: a swipe that starts inside the status bar (5.7% of a // Pixel 7's height, 6.9% of an iPhone's with a Dynamic Island) pulls the notification shade or diff --git a/packages/platform-linux/src/input-actions.ts b/packages/platform-linux/src/input-actions.ts index 1fb14402d4..8fe33c4b35 100644 --- a/packages/platform-linux/src/input-actions.ts +++ b/packages/platform-linux/src/input-actions.ts @@ -2,6 +2,7 @@ import { ensureInputTool } from './linux-env.ts'; import { resolveLinuxToolProvider, type LinuxPointerButton } from './tool-provider.ts'; import { sleep } from '@agent-device/host-kit/retry'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { DEFAULT_SCROLL_AMOUNT } from '@agent-device/contracts/scroll-gesture'; // ── Low-level wrappers ───────────────────────────────────────────────── @@ -227,8 +228,11 @@ export async function scrollLinux( ? Math.max(1, Math.round(options.pixels / 15)) : Math.max(1, Math.round(options.pixels / 40)); } else if (options?.amount != null) { - // amount is a fraction (0–1+) of the viewport; scale relative to default - scrollCount = Math.max(1, Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / 0.6))); + // amount is a fraction (0–1+) of the viewport; scale relative to the shared default + scrollCount = Math.max( + 1, + Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / DEFAULT_SCROLL_AMOUNT)), + ); } // xdotool: button 4=up, 5=down, 6=left, 7=right diff --git a/packages/platform-web/src/agent-browser-provider.test.ts b/packages/platform-web/src/agent-browser-provider.test.ts index d2629b2a14..da8f32b130 100644 --- a/packages/platform-web/src/agent-browser-provider.test.ts +++ b/packages/platform-web/src/agent-browser-provider.test.ts @@ -851,3 +851,28 @@ function createAudioProbeScriptPage(): AudioProbeScriptPage { }, }; } + +/** + * #2432: `amount` is a fraction of the viewport axis everywhere else, and the browser scrolls by + * CSS pixels. Passing it through raw made `scroll down 0.5` travel half a pixel. + */ +test('a relative scroll amount reaches agent-browser as pixels, not as the fraction itself', async () => { + await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { + const calls: AgentBrowserCall[] = []; + + await withCommandExecutorOverride(recordingExecutor(calls), async () => { + await provider.scroll('down', { amount: 0.6 }); + await provider.scroll('down', { amount: 1.2 }); + await provider.scroll('down', undefined); + }); + + assert.deepEqual( + calls.map((call) => call.args), + [ + ['scroll', 'down', '300', '--json', '--session', 'web-session'], + ['scroll', 'down', '600', '--json', '--session', 'web-session'], + ['scroll', 'down', '--json', '--session', 'web-session'], + ], + ); + }); +}); diff --git a/packages/platform-web/src/agent-browser-provider.ts b/packages/platform-web/src/agent-browser-provider.ts index 8e37fe543d..a80f9045c3 100644 --- a/packages/platform-web/src/agent-browser-provider.ts +++ b/packages/platform-web/src/agent-browser-provider.ts @@ -27,6 +27,7 @@ import { cleanupManagedAgentBrowserOrphansForProviderStartup, recordManagedAgentBrowserProcesses, } from './agent-browser-lifecycle.ts'; +import { DEFAULT_SCROLL_AMOUNT } from '@agent-device/contracts/scroll-gesture'; const AGENT_BROWSER = 'agent-browser'; const AGENT_BROWSER_TIMEOUT_MS = 30_000; @@ -125,7 +126,7 @@ async function runPacedScroll( direction: string, scrollOptions: { amount?: number; pixels?: number; durationMs?: number } | undefined, ): Promise { - const steps = buildPacedScrollSteps(scrollOptions); + const steps = buildPacedScrollSteps(resolveWebScrollDistance(scrollOptions)); for (const step of steps) { await runJson(buildScrollArgs(direction, step.distance)); if (step.delayAfterMs > 0) await sleep(step.delayAfterMs); @@ -137,34 +138,59 @@ type ScrollStep = { delayAfterMs: number; }; -function buildPacedScrollSteps( +/** agent-browser's own default wheel step, and the distance the default amount maps onto. */ +const WEB_DEFAULT_SCROLL_PIXELS = 300; + +type WebScrollDistance = { + distance?: number; + durationMs?: number; +}; + +/** + * The browser scrolls by CSS pixels, so a relative `amount` has to become one before it reaches + * agent-browser — feeding it through raw made `scroll down 0.5` travel half a pixel. + * + * There is no gesture viewport to measure against on this backend, so `amount` scales the default + * step the same way the Linux pointer backend scales its wheel clicks: the shared default amount + * maps to the default step, and everything else is proportional to it. + */ +function resolveWebScrollDistance( scrollOptions: { amount?: number; pixels?: number; durationMs?: number } | undefined, -): ScrollStep[] { - const requestedDistance = scrollOptions?.pixels ?? scrollOptions?.amount; +): WebScrollDistance { const durationMs = scrollOptions?.durationMs; + const timing = durationMs === undefined ? {} : { durationMs }; + if (scrollOptions?.pixels !== undefined) { + return { distance: scrollOptions.pixels, ...timing }; + } + if (scrollOptions?.amount !== undefined) { + return { + distance: Math.max( + 1, + Math.round((WEB_DEFAULT_SCROLL_PIXELS * scrollOptions.amount) / DEFAULT_SCROLL_AMOUNT), + ), + ...timing, + }; + } + return timing; +} + +function buildPacedScrollSteps(scrollDistance: WebScrollDistance): ScrollStep[] { + const { distance, durationMs } = scrollDistance; if (durationMs === undefined || durationMs <= 0) { - return [{ distance: requestedDistance, delayAfterMs: 0 }]; + return [{ distance, delayAfterMs: 0 }]; } const stepCount = Math.max(1, Math.min(20, Math.ceil(durationMs / 50))); const intervalMs = durationMs / Math.max(1, stepCount - 1); - return scrollStepDistances(scrollOptions, stepCount).map((distance, index) => ({ - distance, + return distributeIntegerDistance( + Math.round(distance ?? WEB_DEFAULT_SCROLL_PIXELS), + stepCount, + ).map((stepDistance, index) => ({ + distance: stepDistance, delayAfterMs: index < stepCount - 1 ? intervalMs : 0, })); } -function scrollStepDistances( - scrollOptions: { amount?: number; pixels?: number } | undefined, - stepCount: number, -): number[] { - const totalDistance = scrollOptions?.pixels ?? scrollOptions?.amount ?? 300; - if (scrollOptions?.amount !== undefined && scrollOptions.pixels === undefined) { - return Array.from({ length: stepCount }, () => totalDistance / stepCount); - } - return distributeIntegerDistance(Math.round(totalDistance), stepCount); -} - function distributeIntegerDistance(totalDistance: number, stepCount: number): number[] { const baseDistance = Math.floor(totalDistance / stepCount); const remainder = totalDistance - baseDistance * stepCount; diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 5f5839f582..ac2f811966 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -68,7 +68,7 @@ export function classifyAbsenceObservation( ...(firstMatch ? { firstMatch } : {}), }; } - const sparseQuality = sparseQualityForSnapshot(snapshot); + const sparseQuality = sparseCaptureQuality(snapshot); if (sparseQuality) { return { kind: 'sparse', @@ -86,7 +86,12 @@ export function classifyAbsenceObservation( return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } -function sparseQualityForSnapshot( +/** + * The one definition of "this capture is too sparse to trust": the backend's own verdict, then the + * legacy iOS shape that predates verdicts. Shared with `scroll --until`, which must not stop on a + * tree whose selectors are unreliable. + */ +export function sparseCaptureQuality( snapshot: Pick, ): SparseQuality | undefined { const quality = snapshot.snapshotQuality; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 611a7c134d..7c1d3f858b 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -160,6 +160,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['holdMs', 'press hold duration'], ['jitterPx', 'press jitter'], ['pixels', 'scroll distance'], + ['until', 'scroll-until-visible stop condition'], ['doubleTap', 'double tap gesture'], ['clickButton', 'desktop mouse button selection', ['button']], ['backMode', 'explicit app/system back behavior', ['mode']], diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index e8aafb0a48..a2447b3ed5 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -15,6 +15,8 @@ That starts the session and returns the initial interactive snapshot with `@refs Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. +Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until ` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something. + Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/src/__tests__/runtime-public.test.ts b/src/__tests__/runtime-public.test.ts index ae7942b5bc..a5fbc18ba6 100644 --- a/src/__tests__/runtime-public.test.ts +++ b/src/__tests__/runtime-public.test.ts @@ -251,7 +251,6 @@ test('internal backend, commands, and io modules are usable', () => { assert.equal(typeof commands.interactions.fill, 'function'); assert.equal(typeof commands.interactions.focus, 'function'); assert.equal(typeof commands.interactions.longPress, 'function'); - assert.equal(typeof commands.interactions.scroll, 'function'); assert.equal(typeof commands.interactions.gesture, 'function'); assert.equal(typeof commands.system.back, 'function'); assert.equal(typeof commands.system.home, 'function'); diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index c585131e01..1ff39fe8fe 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -148,10 +148,10 @@ Bootstrap: Snapshots and refs: snapshot reads visible state; snapshot -i gets current interactive refs only -- fast path before interaction. Default text is token-efficient; --raw/--json for full provider tree. - Legend: @e12 [button] label="Add to cart" enabled hittable -> press @e12. [off-screen below] -> scroll down (a hint, not a ref). + Legend: @e12 [button] label="Add to cart" enabled hittable -> press @e12. [off-screen below] -> scroll down --until (a hint, not a ref). Refs stay valid until you press/click/fill/type/scroll/back/wait-for-async-UI, or otherwise change app state; open/--relaunch clears the stored snapshot outright. Prefer --settle and its diff when it shows next target; refresh with snapshot -i only when you did not settle, it reported not settled, or output lacks what you need. A known selector/label after a mutation is often enough, since interaction commands refresh state internally. - Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing list target: scroll down/up then snapshot -i. TV/D-pad focus: help tv. + Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing target: scroll --until . TV/D-pad focus: help tv. Selectors: id="field-email", label="Allow", role=button label="Search" -- not bare role keys (button="Search"); no CSS selectors/--selector/--text/raw x-y when refs/selectors exist. @@ -252,10 +252,12 @@ Shapes: agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong agent-device gesture pan 200 420 0 -80 500 agent-device gesture pan 200 420 80 -40 700 --pointer-count 2 + agent-device scroll down --until 'id=submit' agent-device gesture fling right 200 420 180 agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. @@ -990,7 +992,7 @@ Rules: Findings must come from observed runtime behavior, not source reads. After each mutation, use the --settle diff as evidence when available; otherwise re-snapshot. Wait timeouts are integer milliseconds in the trailing positional: agent-device wait 'role=tab' 10000. Do not write duration suffixes such as 10s. - scroll takes a selector-less direction+amount form: agent-device scroll down 3. Use --settle to wait for the UI to go quiet and get the settled diff. + scroll takes a selector-less direction+amount form: agent-device scroll down 0.8. One gesture cannot travel further than 0.8 of the viewport axis, so a larger amount saturates rather than covering more ground; to cross several screens use agent-device scroll down --until or scroll bottom. Use --settle to wait for the UI to go quiet and get the settled diff. Keep commands in the report reproducible; use selectors or refs from fresh snapshots, not guessed coordinates. Prefer refs for exploration and selectors for deterministic replay. Use logs, network, screenshot --overlay-refs, trace, perf frames, perf memory, native profiles, or react-devtools only when they add evidence to a specific issue. diff --git a/src/commands/cli-grammar/flag-definitions-action.ts b/src/commands/cli-grammar/flag-definitions-action.ts index 15be134206..e11ff6b38e 100644 --- a/src/commands/cli-grammar/flag-definitions-action.ts +++ b/src/commands/cli-grammar/flag-definitions-action.ts @@ -115,6 +115,13 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--pixels ', usageDescription: 'Scroll: explicit gesture distance in pixels', }, + { + key: 'until', + names: ['--until'], + type: 'string', + usageLabel: '--until ', + usageDescription: 'Scroll: repeat passes until the selector is visible on screen', + }, { key: 'doubleTap', names: ['--double-tap'], diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index 1cc8c6c53c..f2c8dc5d91 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -89,6 +89,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { holdMs: options.holdMs, jitterPx: options.jitterPx, pixels: options.pixels, + until: options.until, doubleTap: options.doubleTap, verify: options.verify, settle: options.settle, diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index e0f21e93fb..ec3f622ed6 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -145,9 +145,9 @@ const interactionCliSchemas = { }, scroll: { usageOverride: 'scroll [amount]', - usageFlags: ['pixels', 'durationMs', 'settle'], + usageFlags: ['until', 'pixels', 'durationMs', 'settle'], positionalArgs: ['directionOrEdge', 'amount?'], - allowedFlags: ['pixels', 'durationMs', ...postActionObservationCliFlags('scroll')], + allowedFlags: ['pixels', 'durationMs', 'until', ...postActionObservationCliFlags('scroll')], }, } as const satisfies Record; diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 47e4a1346d..8f15ff32f4 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -33,7 +33,7 @@ import { targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import type { ScrollInputDirection } from './runtime/gestures.ts'; +import type { ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; export const interactionCliReaders = { click: (positionals, flags) => ({ @@ -101,6 +101,7 @@ export const interactionCliReaders = { amount: optionalCliNumber(positionals[1]), pixels: flags.pixels, durationMs: flags.durationMs, + until: flags.until, }), // The one observation-only reader in this file: `get` can be excluded from a // repair-armed heal by default, so it also takes the `--record` opt-in diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 5774ac1fab..2e78dd567b 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', @@ -143,6 +143,9 @@ const scrollFields = { direction: requiredField(enumField(SCROLL_INPUT_DIRECTIONS)), amount: numberField('Platform scroll amount.'), pixels: integerField('Pixel scroll amount.', { min: 0 }), + until: stringField( + 'Repeat scroll passes until this selector is visible on screen, then stop. Not valid with the top/bottom edge directions, which carry their own stop condition.', + ), durationMs: integerField('Scroll duration in milliseconds when the backend supports pacing.', { min: 0, max: SCROLL_DURATION_MAX_MS, diff --git a/src/commands/interaction/runtime/gestures.test.ts b/src/commands/interaction/runtime/gestures.test.ts index b7a6a7081b..97655ce402 100644 --- a/src/commands/interaction/runtime/gestures.test.ts +++ b/src/commands/interaction/runtime/gestures.test.ts @@ -5,7 +5,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { createInteractionDevice, dragTargetSnapshot, - runtimeScrollSnapshot, selectorSnapshot, } from './__tests__/test-utils/index.ts'; @@ -187,235 +186,6 @@ test('runtime longPress with settle drops the non-hittable hint when the diff pr assert.equal('hint' in result, false); }); -test('runtime scroll resolves selector targets before calling the backend primitive', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { scrolled: true }; - }, - }); - - const selectorResult = await device.interactions.scroll({ - session: 'default', - target: selector('label=Continue'), - direction: 'down', - pixels: 120, - durationMs: 50, - }); - const viewportResult = await device.interactions.scroll({ - direction: 'up', - amount: 0.5, - }); - - assert.equal(selectorResult.kind, 'selector'); - assert.equal(selectorResult.durationMs, undefined); - assert.equal(viewportResult.kind, 'viewport'); - assert.deepEqual(calls, [ - { - target: { kind: 'point', point: { x: 60, y: 40 } }, - options: { - direction: 'down', - pixels: 120, - durationMs: 50, - releaseBehavior: 'controlled', - }, - }, - { - target: { kind: 'viewport' }, - options: { direction: 'up', amount: 0.5, releaseBehavior: 'controlled' }, - }, - ]); -}); - -test('runtime scroll reports duration only when the backend honored it', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), - }); - - const result = await device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 50, - }); - - assert.equal(result.durationMs, 50); - assert.deepEqual(result.backendResult, { durationMs: 50 }); -}); - -test('runtime scroll rejects duration above the shared cap', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 10_001, - }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => { - throw new Error('snapshot unavailable'); - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - /Failed to verify scroll bottom state/, - ); - - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom does not scroll when no hidden content is below', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: unknown[] = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotScopes.push(options?.scope); - return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 1); - assert.equal(result.backendResult?.pass, 1); - assert.deepEqual(calls, [ - { - target: { kind: 'viewport' }, - options: { direction: 'down', releaseBehavior: 'inertial' }, - }, - ]); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - }), - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.passes, 2); - assert.equal(calls.length, 2); -}); - -test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotCount += 1; - if (options?.scope) throw new Error('scoped snapshot failed'); - return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; - }, - scroll: async () => ({}), - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => { - for (const surface of ['desktop', 'menubar'] as const) { - const device = createInteractionDevice(selectorSnapshot(), { - platform: 'macos', - sessionMetadata: { surface }, - scroll: async () => { - throw new Error(`${surface} scroll should be rejected before backend call`); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - target: { kind: 'viewport' }, - session: 'default', - }), - new RegExp(`scroll is not supported on macOS ${surface}`), - ); - } -}); - test('runtime multi-touch planning prefers backend viewport geometry without a snapshot capture', async () => { let capturedPlan: unknown; const device = createInteractionDevice(selectorSnapshot(), { diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index da17ed61e3..8e8e5fd846 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -4,36 +4,20 @@ import type { LongPressCommandResult, ResolutionDisclosure, } from '@agent-device/contracts/interaction'; -import type { ScrollDirection, ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; import { buildDragGesturePlan, singlePointerPlanEndpoints, } from '@agent-device/contracts/gesture-plan'; -import { - assertExclusiveScrollDistanceInputs, - honoredScrollDurationMs, - normalizeScrollDurationMs, - resolveScrollExecutionOptions, -} from '@agent-device/contracts/scroll-command'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import { - captureScrollEdgeState, - formatScrollEdgeMessage, - runScrollEdgePasses, - type ScrollEdge, - type ScrollEdgeState, - type ScrollEdgeTarget, -} from '@agent-device/capture-kit/scroll-edge-state'; import { successText } from '@agent-device/kernel/success-text'; import { requireIntInRange } from '@agent-device/kernel/validation'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type BackendResultEnvelope, - type BackendResultVariant, type RuntimeCommand, } from '../../runtime-types.ts'; import { @@ -125,48 +109,6 @@ export type HoverCommandOptions = CommandContext & { export type { HoverCommandResult }; -export type GestureDirection = ScrollDirection; -// The input vocabulary lives in contracts/scroll-gesture.ts beside the other scroll vocabularies, -// so the public API can declare `ScrollOptions` without depending on this command runtime. -export { type ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; - -export type ScrollTarget = - | InteractionTarget - | { - kind: 'viewport'; - }; - -export type ScrollCommandOptions = CommandContext & { - target?: ScrollTarget; - direction: ScrollInputDirection; - amount?: number; - pixels?: number; - durationMs?: number; -}; - -export type ScrollCommandResult = - | BackendResultVariant<{ - kind: 'viewport'; - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - }> - | BackendResultVariant< - ResolvedInteractionTarget & { - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - } - >; - -type ResolvedScrollTarget = { kind: 'viewport' } | ResolvedInteractionTarget; - export const focusCommand: RuntimeCommand = async ( runtime, options, @@ -405,182 +347,9 @@ function recordedDragTarget(target: ResolvedInteractionTarget): DragRecordingTar }; } -export const scrollCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => { - if (!runtime.backend.scroll) { - throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); - } - const target = resolveScrollDirection(options.direction); - const amount = normalizeOptionalPositiveNumber(options.amount, 'scroll amount'); - const pixels = normalizeOptionalPositiveInteger(options.pixels, 'scroll pixels'); - const durationMs = normalizeScrollDurationMs(options.durationMs); - assertExclusiveScrollDistanceInputs( - { amount, pixels }, - 'scroll accepts either amount or pixels, not both', - ); - - const resolved = await resolveScrollTarget(runtime, options); - const backendTarget = - resolved.kind === 'viewport' - ? { kind: 'viewport' as const } - : { kind: 'point' as const, point: requireResolvedPoint(resolved) }; - const scrollBackend = runtime.backend.scroll; - const executionOptions = resolveScrollExecutionOptions( - { - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - ...(durationMs !== undefined ? { durationMs } : {}), - }, - target.edge, - ); - const runScroll = async () => - await scrollBackend(toBackendContext(runtime, options), backendTarget, { - direction: target.direction, - ...executionOptions, - }); - let backendResult: Awaited>> | undefined; - let completedPasses = 0; - if (target.edge) { - const edge = target.edge; - const edgeTarget = buildScrollEdgeTarget(resolved); - const edgeResult = await runScrollEdgePasses({ - edge, - captureState: async (scope) => - await captureRuntimeScrollEdgeState(runtime, options, edge, edgeTarget, scope), - scroll: runScroll, - }); - backendResult = edgeResult.result; - completedPasses = edgeResult.passes; - } else { - backendResult = await runScroll(); - completedPasses = 1; - } - const formattedBackendResult = toBackendResult(backendResult); - const reportedDurationMs = honoredScrollDurationMs(formattedBackendResult); - return { - ...resolved, - direction: target.direction, - ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - ...(reportedDurationMs !== undefined ? { durationMs: reportedDurationMs } : {}), - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText( - formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), - ), - }; -}; - -async function resolveScrollTarget( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, -): Promise { - const target = options.target ?? { kind: 'viewport' as const }; - if (target.kind === 'viewport') { - await assertSupportedInteractionSurface(runtime, options, 'scroll'); - return { kind: 'viewport' }; - } - return await resolveInteractionTarget( - runtime, - { ...options, target }, - { - action: 'scroll', - requireInteractive: false, - pipeline: SELECTOR_PIPELINE_POLICIES.resolvedTarget, - }, - ); -} - -function resolveScrollDirection(direction: ScrollInputDirection): { - direction: GestureDirection; - edge?: 'top' | 'bottom'; -} { - if (direction === 'bottom') return { direction: 'down', edge: 'bottom' }; - if (direction === 'top') return { direction: 'up', edge: 'top' }; - return { direction: requireDirection(direction, 'scroll direction') }; -} - -function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget { - return resolved.kind === 'viewport' - ? {} - : { - point: resolved.point, - nodeIndex: 'node' in resolved ? resolved.node?.index : undefined, - }; -} - function requireResolvedPoint(result: { point?: Point }): Point { if (!result.point) { throw new AppError('COMMAND_FAILED', 'Interaction target resolved without coordinates'); } return result.point; } - -async function captureRuntimeScrollEdgeState( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, - edge: ScrollEdge, - target: ScrollEdgeTarget, - scope?: string, -): Promise { - if (!runtime.backend.captureSnapshot) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - ); - } - const { captureSnapshot } = runtime.backend; - return await captureScrollEdgeState({ - edge, - target, - scope, - captureNodes: async (snapshotScope) => { - const result = await captureSnapshot(toBackendContext(runtime, options), { - scope: snapshotScope, - }); - return result.snapshot?.nodes ?? result.nodes ?? []; - }, - }); -} - -function requireDirection( - direction: GestureDirection | undefined, - field: string, -): GestureDirection { - switch (direction) { - case 'up': - case 'down': - case 'left': - case 'right': - return direction; - default: - throw new AppError('INVALID_ARGS', `${field} must be up, down, left, or right`); - } -} - -function normalizeOptionalPositiveNumber( - value: number | undefined, - field: string, -): number | undefined { - return value === undefined ? undefined : normalizePositiveNumber(value, field); -} - -function normalizePositiveNumber(value: number, field: string): number { - if (!Number.isFinite(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive number`); - } - return value; -} - -function normalizeOptionalPositiveInteger( - value: number | undefined, - field: string, -): number | undefined { - if (value === undefined) return undefined; - if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive integer`); - } - return value; -} diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index dd50012a28..297d8b51d6 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -7,7 +7,6 @@ import { hoverCommand, longPressCommand, pressCommand, - scrollCommand, type ClickCommandOptions, type FillCommandOptions, type FillCommandResult, @@ -20,8 +19,6 @@ import { type LongPressCommandResult, type PressCommandOptions, type PressCommandResult, - type ScrollCommandOptions, - type ScrollCommandResult, } from './interactions.ts'; import { findCommand, @@ -79,7 +76,6 @@ export type InteractionCommands = { focus: RuntimeCommand; longPress: RuntimeCommand; hover: RuntimeCommand; - scroll: RuntimeCommand; gesture: RuntimeCommand; /** * #1638: the observation half of `--settle` for mutations that resolve no @@ -143,7 +139,6 @@ export type BoundInteractionCommands = { target: InteractionTarget, options?: Omit, ) => Promise; - scroll: BoundRuntimeCommand; gesture: BoundRuntimeCommand; settleObservation: BoundRuntimeCommand; }; @@ -167,7 +162,6 @@ export const interactionCommands: InteractionCommands = { focus: focusCommand, longPress: longPressCommand, hover: hoverCommand, - scroll: scrollCommand, gesture: gestureCommand, settleObservation: settleObservationCommand, }; @@ -198,7 +192,6 @@ export function bindInteractionCommands(runtime: AgentDeviceRuntime): BoundInter longPress: (target, options = {}) => interactionCommands.longPress(runtime, { ...options, target }), hover: (target, options = {}) => interactionCommands.hover(runtime, { ...options, target }), - scroll: (options) => interactionCommands.scroll(runtime, options), gesture: (options) => interactionCommands.gesture(runtime, options), settleObservation: (options) => interactionCommands.settleObservation(runtime, options), }; diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index a0ca7b795b..5f5c18afe9 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -26,7 +26,7 @@ import { type InteractionTarget, } from './resolution.ts'; -export { focusCommand, hoverCommand, longPressCommand, scrollCommand } from './gestures.ts'; +export { focusCommand, hoverCommand, longPressCommand } from './gestures.ts'; export type { FocusCommandOptions, FocusCommandResult, @@ -34,8 +34,6 @@ export type { HoverCommandResult, LongPressCommandOptions, LongPressCommandResult, - ScrollCommandOptions, - ScrollCommandResult, } from './gestures.ts'; export type { InteractionTarget } from './resolution.ts'; diff --git a/src/commands/interaction/runtime/resolution.test.ts b/src/commands/interaction/runtime/resolution.test.ts index 4d599252c6..6c08c2e138 100644 --- a/src/commands/interaction/runtime/resolution.test.ts +++ b/src/commands/interaction/runtime/resolution.test.ts @@ -187,10 +187,10 @@ test('runtime press names a direction for a partial clip whose center is off-scr assert.equal(details?.reason, 'offscreen_selector'); assert.equal(details?.scrollDirection, 'down'); assert.match(String(details?.hint), /scroll down/i); - // #1366 recovery must be bounded: a single large (fling) scroll overshoots, - // so the hint steers to small steps / a bounded gesture pan. - assert.match(String(details?.hint), /small steps/i); - assert.match(String(details?.hint), /gesture pan/i); + // #1366 recovery must be bounded. `--until` is what bounds it now: it checks the same + // selector between passes, so the hint names one command rather than a manual step loop. + assert.match(String(details?.hint), /scroll down --until 'label=Cash'/); + assert.match(String(details?.hint), /stops on the target/i); return true; }, ); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index b52a0452d8..1230ad1621 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -881,11 +881,11 @@ async function assertVisibleSelectorTarget( // A selector re-resolves against a fresh snapshot on every attempt, so the // recovery is: move the named direction, then retry THIS selector — no // separate snapshot step, and no @ref (a scroll expires the ref frame, - // #1366). Naming the direction stops the wrong-way / retry-the-same-ref loop; - // bounded steps stop the overshoot loop — a single large scroll (fling - // momentum on iOS) can sail past the target, so a short gesture pan lands it. + // #1366). `--until` is that whole loop as one command: it checks the same + // selector between passes, which is also what keeps a large step from + // overshooting, so the hint no longer has to trade distance for accuracy. hint: (direction) => - `${scrollRevealClause(direction)} in small steps, retrying ${action} with the same selector after each (it re-resolves against a fresh snapshot). A single large scroll can overshoot the target; a short bounded gesture pan lands it more reliably. If it is inside a closed drawer or another tab, open that container first.`, + `${scrollRevealClause(direction, selector)} then retry ${action} with the same selector. --until checks the selector between passes, so it stops on the target rather than sailing past it. If it is inside a closed drawer or another tab, open that container first.`, }); } @@ -902,19 +902,29 @@ async function assertVisibleRefTarget( details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) }, // The scroll that reveals the target expires the ref frame (#1366, ADR // 0014), so retrying this @ref would be rejected next. Steer to a selector, - // which re-resolves against a fresh snapshot and bypasses the ref-frame guard. + // which re-resolves against a fresh snapshot and bypasses the ref-frame guard + // — and which `--until` can then check between passes. hint: (direction) => - `${scrollRevealClause(direction)} in small steps (a single large scroll can overshoot; a short bounded gesture pan lands it more reliably), then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, + `${scrollRevealClause(direction, null)} then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, }); } -// Shared lead-in for both off-screen hints. Names the concrete `scroll ` -// when the geometry gives one, and falls back to the generic phrasing when the -// target is off more than one edge in a way that has no single reveal. Callers -// append the bounded-steps guidance: a single large scroll (fling momentum on -// iOS) can sail past the target, so small bounded moves are what actually land. -function scrollRevealClause(direction: OffscreenScrollDirection | null): string { - return direction ? `Scroll ${direction} toward it` : 'Scroll toward it'; +/** + * Shared lead-in for both off-screen hints: the one command that reveals the target. + * + * When the geometry names a direction AND the caller has a selector to check, this is a complete + * `scroll --until ` — one request that stops on the target instead of the + * scroll-then-look-again loop the hint used to prescribe. Without a selector to check (an @ref + * refusal) or without a single reveal direction (off more than one edge), it degrades to naming + * the move and leaves the stop condition to the caller's own next step. + */ +function scrollRevealClause( + direction: OffscreenScrollDirection | null, + selector: string | null, +): string { + if (!direction) return 'Scroll toward it,'; + if (!selector) return `Scroll ${direction} toward it,`; + return `Run scroll ${direction} --until '${selector}' to bring it on screen,`; } /** diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 34e6c420e4..0c143e4ccd 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -49,6 +49,7 @@ export const DISPATCH_CONTEXT_FLAG_KEYS = [ 'holdMs', 'jitterPx', 'pixels', + 'until', 'doubleTap', 'backMode', 'pauseMs', diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index 46cdc0fdce..631d04d4d6 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -338,3 +338,79 @@ test('the edge plan proves its capture statically and the direction plan cannot expectTypeOf().toEqualTypeOf<'scrollDirection'>(); expectTypeOf>().toEqualTypeOf<'scrollDirection'>(); }); + +/** A row walked into the viewport, so the executor's pass count is observable. */ +function untilNodes(targetY: number, hiddenBelow: boolean) { + return [ + { + index: 1, + depth: 0, + type: 'ScrollView', + label: 'Form', + ...(hiddenBelow ? { hiddenContentBelow: true } : {}), + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 2, + depth: 1, + parentIndex: 1, + type: 'TextField', + label: 'Email', + rect: { x: 0, y: targetY, width: 400, height: 40 }, + }, + ]; +} + +/** + * Route-level only: the executor's result envelope, the parse rejection, and admission. The loop's + * own behavior — arrival, end-of-content, the pass budget, and every capture refusal — is covered + * against the module in `scroll-until.test.ts` rather than duplicated through this harness. + */ +test('bound scroll --until reports the passes it spent and the selector it stopped on', async () => { + const scrolls: string[] = []; + const frames = [untilNodes(2400, true), untilNodes(1200, true), untilNodes(300, true)]; + const result = await runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ nodes: frames[Math.min(scrolls.length, frames.length - 1)] }), + scroll: async (direction) => { + scrolls.push(direction); + return { pixels: 480 }; + }, + }, + ); + + assert.equal(result.until, 'label=Email'); + assert.equal(result.direction, 'down'); + assert.equal(result.passes, 2); + assert.deepEqual(scrolls, ['down', 'down']); + assert.match(String(result.message), /Scrolled down 2 passes until label=Email was visible/); +}); + +test('bound scroll rejects --until on an edge direction before any device work', async () => { + await assert.rejects( + () => + runScroll( + ['bottom'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ nodes: untilNodes(300, true) }), + scroll: async () => { + throw new Error('scroll should be rejected before the backend call'); + }, + }, + ), + /cannot take --until/, + ); +}); + +test('bound scroll --until is refused at admission when the owner declares no capture', async () => { + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals: ['down'], + context: { until: 'label=Email' } as DaemonCommandContext, + ...bindings({ scroll: async () => ({}) }), + }); + assert.equal(resolved.ok, false); +}); diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts index df107f34a4..c63642246f 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts @@ -261,13 +261,12 @@ test('press @ref fails fast when the target is off-screen', async () => { if (response && !response.ok) { expect(response.error.code).toBe('COMMAND_FAILED'); expect(response.error.message).toMatch(/off-screen/i); - // #1366: the hint names the concrete scroll direction, steers to a - // selector-based retry (a @ref would be rejected as expired after the scroll), - // and prescribes bounded movement (a large fling scroll overshoots). + // #1366: the hint names the concrete scroll direction and steers to a selector-based retry + // (a @ref would be rejected as expired after the scroll). A ref refusal has no selector to + // check between passes, so this hint names the move and leaves the stop condition to the + // selector retry it prescribes. expect(response.error.hint).toMatch(/scroll down/i); expect(response.error.hint).toMatch(/selector/i); - expect(response.error.hint).toMatch(/small steps/i); - expect(response.error.hint).toMatch(/gesture pan/i); expect(response.error.details?.reason).toBe('offscreen_ref'); expect(response.error.details?.scrollDirection).toBe('down'); } diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 39c4d66825..6e52e556db 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -1,6 +1,8 @@ import { assertExclusiveScrollDistanceInputs, + assertScrollUntilCompatible, honoredScrollDurationMs, + honoredScrollPixels, normalizeScrollDurationMs, resolveScrollExecutionOptions, type ResolvedScrollExecutionOptions, @@ -22,6 +24,8 @@ import { type ScrollEdge, type ScrollEdgeState, } from '@agent-device/capture-kit/scroll-edge-state'; +import { formatScrollUntilMessage, runScrollUntilVisible } from './scroll-until.ts'; +import { publicPlatformString } from '@agent-device/kernel/device'; import { withSuccessText } from '@agent-device/kernel/success-text'; import type { DaemonCommandContext } from './context.ts'; import { errorResponse } from './response.ts'; @@ -43,6 +47,7 @@ type BoundScrollDirection = BoundDeviceRuntime< Extract['use'] >; type BoundScrollEdge = BoundDeviceRuntime['use']>; +type BoundScrollUntil = BoundDeviceRuntime['use']>; /** `scroll bottom` scrolls down to the edge; `scroll top` scrolls up to it. */ function parseScrollTarget(input: string): ScrollTarget { @@ -82,12 +87,18 @@ export async function resolveBoundScrollRuntime( const amount = params.positionals[1] ? Number(params.positionals[1]) : undefined; const pixels = params.context.pixels; const durationMs = params.context.durationMs; + const until = params.context.until; if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); assertScrollCommandInputs(amount, pixels, durationMs); const target = parseScrollTarget(directionInput); + const stopCondition = { + ...(target.edge === undefined ? {} : { edge: target.edge }), + ...(until === undefined ? {} : { until }), + }; + assertScrollUntilCompatible(stopCondition); const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); - const plan = resolveScrollRuntimePlan(target.edge === undefined ? {} : { edge: target.edge }); + const plan = resolveScrollRuntimePlan(stopCondition); const admission = { command: 'scroll', device: params.device, @@ -108,20 +119,51 @@ export async function resolveBoundScrollRuntime( ...admission, // The retired leaf refused an unsupported edge scroll by naming what the edge needs, so // the capture requirement keeps saying so rather than collapsing into "not supported". - unavailableResponse: (unavailable) => scrollEdgeUnsupported(edge, unavailable.hint), + unavailableResponse: (unavailable) => + scrollCaptureUnsupported( + `scroll ${edge}, which verifies hidden content before scrolling,`, + unavailable.hint, + ), use: plan.use, }, async (runtime, dispatchContext) => await executeEdgeScroll(runtime, edge, target, options, dispatchContext), ); } + case 'until': { + const selector = plan.until; + return await resolveBoundGenericRuntime( + { + ...admission, + unavailableResponse: (unavailable) => + scrollCaptureUnsupported( + 'scroll --until, which checks whether the selector became visible,', + unavailable.hint, + ), + use: plan.use, + }, + async (runtime, dispatchContext) => + await executeUntilScroll( + runtime, + params.device, + selector, + target, + options, + dispatchContext, + ), + ); + } } } -function scrollEdgeUnsupported(edge: ScrollEdge, hint: string | undefined) { +/** + * Both verifying tiers refuse the same way and differ only in what they would have checked, so the + * refusal names that rather than collapsing into "not supported" — the shape the retired leaf had. + */ +function scrollCaptureUnsupported(subject: string, hint: string | undefined) { return errorResponse( 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, + `${subject} requires snapshot support`, undefined, hint === undefined ? undefined : { hint }, ); @@ -158,6 +200,39 @@ async function executeEdgeScroll( return scrollResult(target, options, edgeResult.passes, edgeResult.result ?? {}); } +/** Repeats the pass until the selector is on screen; every failure shape is owned by the loop. */ +async function executeUntilScroll( + runtime: BoundScrollUntil, + device: DeviceInfo, + selector: string, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): Promise> { + const untilResult = await runScrollUntilVisible({ + selector, + direction: target.direction, + platform: publicPlatformString(device), + capture: async () => + await runtime.operations.captureSnapshot({ + options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, + execution: runtimeExecutionFromContext(context), + }), + scroll: async () => await scrollOnce(runtime, target, options, context), + }); + return withSuccessText( + { + direction: target.direction, + until: selector, + passes: untilResult.passes, + ...(options.amount !== undefined ? { amount: options.amount } : {}), + ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), + ...(untilResult.result ?? {}), + }, + formatScrollUntilMessage(target.direction, selector, untilResult.passes), + ); +} + async function captureEdgeState( runtime: BoundScrollEdge, edge: ScrollEdge, @@ -207,13 +282,14 @@ function scrollResult( ...(durationMs !== undefined ? { durationMs } : {}), ...interactionResult, }, - formatScrollEdgeMessage( - target.direction, - target.edge, - completedPasses, - options.amount, - options.pixels, - ), + formatScrollEdgeMessage({ + direction: target.direction, + edge: target.edge, + passes: completedPasses, + amount: options.amount, + pixels: options.pixels, + honoredPixels: honoredScrollPixels(interactionResult), + }), ); } diff --git a/src/daemon/scroll-until.test.ts b/src/daemon/scroll-until.test.ts new file mode 100644 index 0000000000..057fad0afe --- /dev/null +++ b/src/daemon/scroll-until.test.ts @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; +import { + SCROLL_UNTIL_PASS_LIMIT, + formatScrollUntilMessage, + runScrollUntilVisible, +} from './scroll-until.ts'; + +/** The provenance every `SnapshotResult` carries; the fields under test are the rest. */ +function capture(fields: Partial): SnapshotResult { + return { backend: 'xctest', producer: 'runner', ...fields } as SnapshotResult; +} + +const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; +const SPARSE = { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' } as const; + +/** A scrollable whose single row sits at `rowY`; below 800 is off-screen with content beneath. */ +function tree(rowY: number, label = 'Email'): SnapshotNode[] { + return [ + { index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode, + { index: 1, parentIndex: 0, ref: 'e2', type: 'ScrollView', rect: VIEWPORT } as SnapshotNode, + { + index: 2, + parentIndex: 1, + ref: 'e3', + type: 'TextField', + label, + rect: { x: 0, y: rowY, width: 400, height: 40 }, + } as SnapshotNode, + ]; +} + +async function run(params: { + captures: SnapshotResult[]; + selector?: string; + passLimit?: number; + onScroll?: () => void; +}) { + let index = 0; + return await runScrollUntilVisible({ + selector: params.selector ?? 'label=Email', + direction: 'down', + platform: 'ios', + ...(params.passLimit === undefined ? {} : { passLimit: params.passLimit }), + capture: async () => params.captures[Math.min(index++, params.captures.length - 1)]!, + scroll: async () => { + params.onScroll?.(); + return { pixels: 480 }; + }, + }); +} + +test('an already visible target costs one capture and no gesture', async () => { + let scrolls = 0; + const result = await run({ + captures: [capture({ nodes: tree(200) })], + onScroll: () => (scrolls += 1), + }); + assert.equal(result.passes, 0); + assert.equal(scrolls, 0); + assert.equal(result.result, undefined); +}); + +test('passes repeat until the selector is on screen, and the last gesture is reported', async () => { + let scrolls = 0; + const result = await run({ + captures: [tree(2400), tree(1600), tree(200)].map((nodes) => capture({ nodes })), + onScroll: () => (scrolls += 1), + }); + assert.equal(result.passes, 2); + assert.equal(scrolls, 2); + assert.deepEqual(result.result, { pixels: 480 }); +}); + +/** + * A target below the fold is present but not visible. Stopping on presence would leave the caller + * with a row it cannot act on, which is the whole reason the check asks about the viewport. + */ +test('a present but scrolled-out target does not end the loop', async () => { + await assert.rejects( + () => run({ captures: [capture({ nodes: tree(2400) })], passLimit: 1 }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_pass_limit'); + return true; + }, + ); +}); + +test('running out of content stops before the pass budget does', async () => { + let scrolls = 0; + await assert.rejects( + () => + run({ + // The row is on screen, so nothing is hidden below and the selector matches nothing. + captures: [capture({ nodes: tree(200, 'Other') })], + onScroll: () => (scrolls += 1), + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_edge_reached'); + assert.match(String(error.details?.hint), /scroll the opposite direction/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScrollUntilVisible({ + selector: 'label=Missing', + direction: 'right', + platform: 'ios', + passLimit: 3, + capture: async () => capture({ nodes: tree(200) }), + scroll: async () => { + scrolls += 1; + return {}; + }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_pass_limit'); + assert.equal(error.details?.passes, 3); + return true; + }, + ); + assert.equal(scrolls, 3); +}); + +test('the default budget is the shared constant', async () => { + await assert.rejects( + () => + runScrollUntilVisible({ + selector: 'label=Missing', + direction: 'right', + platform: 'ios', + capture: async () => capture({ nodes: tree(200) }), + scroll: async () => ({}), + }), + (error: unknown) => + error instanceof AppError && error.details?.passes === SCROLL_UNTIL_PASS_LIMIT, + ); +}); + +/** + * The defect this pins: coercing an unreadable capture to an empty tree makes the edge analyzer + * report "no room below", so a failed read used to be reported as end-of-content. Each case counts + * gestures, so the refusal is proven to land before matching, edge analysis or scrolling. + */ +test('an unreadable capture is refused rather than read as end-of-content', async () => { + for (const frame of [capture({}), capture({ nodes: [] })]) { + let scrolls = 0; + await assert.rejects( + () => run({ captures: [frame], onScroll: () => (scrolls += 1) }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'no-capture'); + return true; + }, + ); + assert.equal(scrolls, 0); + } +}); + +/** + * A tree the backend calls sparse is one whose selectors are not trustworthy, so it cannot answer + * the question either way. It carries content below the fold, so an edge verdict would be wrong too. + */ +test('a sparse capture is refused before matching, edge analysis or scrolling', async () => { + let scrolls = 0; + await assert.rejects( + () => + run({ + captures: [capture({ nodes: tree(2400), quality: SPARSE })], + onScroll: () => (scrolls += 1), + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(String(error.message), /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('the legacy iOS application-root-only shape is refused', async () => { + await assert.rejects( + () => + run({ + captures: [ + capture({ + backend: 'xctest', + nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], + }), + ], + }), + (error: unknown) => + error instanceof AppError && error.details?.captureRefusal === 'sparse-tree', + ); +}); + +/** + * A tree the backend vouches for is readable, and so is one whose tail was truncated: truncation + * drops content, it does not make the capture untrustworthy. + */ +test('a populated capture is not refused, healthy or recovered', async () => { + for (const state of ['healthy', 'recovered'] as const) { + const result = await run({ + captures: [capture({ nodes: tree(200), quality: { state, backend: 'tree' } })], + }); + assert.equal(result.passes, 0); + } +}); + +test('the success message distinguishes an already visible target from a scrolled one', () => { + assert.equal( + formatScrollUntilMessage('down', 'id=email', 0), + 'id=email was already visible; no down scroll needed', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 1), + 'Scrolled down 1 pass until id=email was visible', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 3), + 'Scrolled down 3 passes until id=email was visible', + ); +}); diff --git a/src/daemon/scroll-until.ts b/src/daemon/scroll-until.ts new file mode 100644 index 0000000000..0358734b7c --- /dev/null +++ b/src/daemon/scroll-until.ts @@ -0,0 +1,209 @@ +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; +import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; +import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { evaluateIsPredicate } from '@agent-device/selectors'; +import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; +import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; +import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; +import { + canScrollFurtherAtEdge, + type ScrollEdge, +} from '@agent-device/capture-kit/scroll-edge-state'; + +/** + * Everything `scroll --until ` needs beyond the ordinary scroll: when a pass has arrived, + * when the capture cannot answer that at all, and how the two failures read. + * + * One module beside the route that runs it. `scroll` reaches a device in exactly one place (ADR + * 0019, `scroll-runtime.ts`), so there is no second caller to keep in agreement and no reason for + * this to be a package surface. + */ + +/** + * How many gestures one `scroll --until` may spend before it gives up. A pass costs a capture plus + * a gesture, so this is the request's whole cost ceiling, not a retry budget: 12 passes at the + * honored 0.8-viewport maximum cover roughly ten screens, which is past the point where a list is + * better reached by `scroll bottom` or a search field. + */ +export const SCROLL_UNTIL_PASS_LIMIT = 12; + +/** Why a capture cannot answer the `--until` question at all. Never an outcome about the content. */ +type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; + +/** + * Scrolls until the selector matches a node that is on screen. + * + * Each pass reads the tree once and that read answers three questions in order: is the capture + * usable, has the target arrived, and is there anywhere left to go. Refusing an unusable capture + * first is what keeps a failed read from being reported as end-of-content — coercing it to an empty + * tree makes the edge analyzer say "no room below". + * + * The end-of-content signal is the one `scroll top`/`scroll bottom` already trust, so both stop in + * the same place. Horizontal scrolls have no such analyzer and are bounded by the pass budget alone. + * The first capture happens before the first gesture, so an already-visible target costs no scroll. + */ +export async function runScrollUntilVisible(params: { + selector: string; + direction: ScrollDirection; + platform: Platform | PublicPlatform; + passLimit?: number; + capture: () => Promise; + scroll: () => Promise; +}): Promise<{ passes: number; result?: TResult }> { + const { selector, direction, platform, capture, scroll } = params; + const passLimit = params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT; + const edge = verticalEdgeFor(direction); + let passes = 0; + let result: TResult | undefined; + + while (true) { + const captured = await capture(); + const refusal = captureRefusal(captured); + if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); + const nodes = (captured.nodes ?? []) as SnapshotNode[]; + if (await isSelectorVisible(nodes, selector, platform)) { + return { passes, ...(result === undefined ? {} : { result }) }; + } + if (edge && !(await canScrollFurtherAtEdge(nodes, edge))) { + throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); + } + if (passes >= passLimit) { + throw scrollUntilNotFoundError(direction, selector, 'pass-limit', passes); + } + result = await scroll(); + passes += 1; + } +} + +export function formatScrollUntilMessage( + direction: ScrollDirection, + selector: string, + passes: number, +): string { + if (passes === 0) return `${selector} was already visible; no ${direction} scroll needed`; + return `Scrolled ${direction} ${passes} ${passes === 1 ? 'pass' : 'passes'} until ${selector} was visible`; +} + +/** + * Does this selector match a node that is visible right now? + * + * Two questions, not one: the `wait` pipeline row answers presence and ignores off-screen, then + * `is visible`'s own predicate answers the rest. Borrowing that predicate rather than a narrower + * geometry check is what keeps `scroll --until X` from stopping on a node that `is visible X` would + * then reject — it carries the Android `visibleToUser` rule, non-positive rects, the hittable + * fallback and anchor resolution too. SOME match, not the first: a list whose rows share a selector + * can hold an off-screen twin above the fold. + */ +async function isSelectorVisible( + nodes: SnapshotNode[], + selector: string, + platform: Platform | PublicPlatform, +): Promise { + const outcome = await resolveSelectorPipeline(SELECTOR_PIPELINE_POLICIES.wait, nodes, selector, { + platform, + }); + const matched = + outcome.kind === 'target' || outcome.kind === 'ambiguous' + ? outcome.matchedNodes + : outcome.kind === 'occluded' + ? [outcome.node] + : []; + return matched.some( + (node) => evaluateIsPredicate({ predicate: 'visible', node, nodes, platform }).pass, + ); +} + +/** + * Sparseness is the same question absence assertions ask, answered by the same helper rather than a + * second definition of readable. Truncation is deliberately NOT refused: a truncated tree is real + * and readable with its tail missing, and refusing it would fail large screens where the target is + * plainly in view. + */ +function captureRefusal(result: SnapshotResult): ScrollUntilCaptureRefusal | undefined { + const nodes = result.nodes; + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } + const sparse = sparseCaptureQuality({ + backend: result.backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(result.quality ? { snapshotQuality: result.quality } : {}), + }); + if (sparse) { + return { + reason: 'sparse-tree', + detail: sparse.reason ?? 'the capture backend reported a sparse tree', + }; + } + return undefined; +} + +/** The content ran out, or the budget did. Separate messages: the corrective action differs. */ +function scrollUntilNotFoundError( + direction: ScrollDirection, + selector: string, + outcome: 'edge-reached' | 'pass-limit', + passes: number, +): AppError { + const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`; + if (outcome === 'edge-reached') { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} reached the end of the scrollable content after ${spent} without ${selector} becoming visible`, + { + reason: 'scroll_until_edge_reached', + selector, + direction, + passes, + hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, + }, + ); + } + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, + { + reason: 'scroll_until_pass_limit', + selector, + direction, + passes, + hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, + }, + ); +} + +function scrollUntilCaptureError( + direction: ScrollDirection, + selector: string, + refusal: ScrollUntilCaptureRefusal, +): AppError { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} --until ${selector} could not read the screen: ${refusal.detail}`, + { + reason: 'scroll_until_capture_unreadable', + selector, + direction, + captureRefusal: refusal.reason, + hint: + refusal.reason === 'no-capture' + ? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.' + : 'The accessibility tree came back sparse, so its refs and selectors are not trustworthy. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.', + }, + ); +} + +/** + * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by its + * pass budget alone rather than by a signal that would always report "no room". + */ +function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined { + if (direction === 'down') return 'bottom'; + if (direction === 'up') return 'top'; + return undefined; +} diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 2f1561cace..a32b85bfd0 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -531,7 +531,8 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { { direction: enumSchema(['up', 'down', 'left', 'right']), edge: enumSchema(['top', 'bottom']), - passes: numberSchema('Edge scrolls only: how many scroll-and-check passes ran.'), + until: stringSchema('Until scrolls only: the selector the passes stopped on.'), + passes: numberSchema('Edge and until scrolls only: how many scroll-and-check passes ran.'), amount: numberSchema(), pixels: numberSchema(), durationMs: numberSchema(), diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index de84ae789c..e94dbd6050 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -19,7 +19,7 @@ export const MCP_SERVER_INSTRUCTIONS = `agent-device drives iOS, Android, tvOS, Start: known app -> call open {app, foreground: true} at once; do not probe with devices, apps, appstate, snapshot, or screenshot first. open returns the initial interactive snapshot with @refs. Unknown app id: devices, then apps, then open the discovered id; never invent ids. Existing session: continue from its state, do not reopen. -Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. +Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. Targets: copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts new file mode 100644 index 0000000000..b951518c6a --- /dev/null +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_UNTIL_PASS_LIMIT } from '../../../src/daemon/scroll-until.ts'; +import { createAndroidSettingsWorld } from './android-world.ts'; +import { withProviderScenarioResource } from './harness.ts'; + +/** + * `scroll --until ` through the real daemon, provider admission, and capture path. + * + * The row climbs one screen per capture, so it is off-screen for the first captures and on screen + * from the third: the loop's stop condition is observed rather than asserted. Keyed on captures + * rather than on injected gestures because the Android gesture path runs through the persistent + * helper, not an adb shell command the world can count. + */ +/** Two screens below the fold, climbing one screen per capture: visible on the third capture. */ +const ARRIVAL_PASSES = 2; + +function climbingRow(): () => number { + let captures = 0; + return () => { + const top = Math.max(200, 1400 - captures * 600); + captures += 1; + return top; + }; +} + +function climbingHierarchy(targetTop: () => number): () => string { + return () => { + // Read the stateful position ONCE: calling it per bound advanced the row twice per capture and + // produced an inverted rectangle on the first one. + const top = targetTop(); + return [ + '', + '', + ' ', + ' ', + ` `, + ' ', + '', + ].join('\n'); + }; +} + +test('Provider-backed integration scroll --until stops on the capture that brings the target on screen', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: climbingHierarchy(climbingRow()) }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + const result = await client.interactions.scroll({ + direction: 'down', + until: 'text=Terms', + ...world.selection, + }); + + assert.equal(result.until, 'text=Terms'); + assert.equal(result.direction, 'down'); + // An exact count is what proves repeated scrolling on valid geometry, rather than a lucky + // first capture or a budget burned to the limit. + assert.equal(result.passes, ARRIVAL_PASSES); + assert.ok(ARRIVAL_PASSES < SCROLL_UNTIL_PASS_LIMIT); + assert.match( + String(result.message), + new RegExp(`Scrolled down ${ARRIVAL_PASSES} passes until text=Terms was visible`), + ); + }, + ); +}); + +test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => { + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + // Nothing below the fold and nothing hidden: the content cannot move further. + snapshotXml: climbingHierarchy(() => 200), + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + await assert.rejects( + () => + client.interactions.scroll({ + direction: 'down', + until: 'text=NeverPresent', + ...world.selection, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.match(String(error.message), /without text=NeverPresent becoming visible/); + return true; + }, + ); + }, + ); +}); + +test('Provider-backed integration scroll rejects --until on the edge directions', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: climbingHierarchy(() => 200) }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + await assert.rejects( + () => + client.interactions.scroll({ + direction: 'bottom', + until: 'text=Terms', + ...world.selection, + }), + /cannot take --until/, + ); + }, + ); +});