From 595ae205cf858ac3f3b0ddb152e398f874f77296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 9 Sep 2026 18:39:12 +0200 Subject: [PATCH 01/13] refactor(interaction): extract the scroll command runtime out of gestures.ts --- src/commands/interaction/interactions.ts | 2 +- .../interaction/runtime/gestures.test.ts | 230 ---------------- src/commands/interaction/runtime/gestures.ts | 233 +---------------- .../interaction/runtime/interactions.ts | 6 +- .../interaction/runtime/scroll.test.ts | 238 +++++++++++++++++ src/commands/interaction/runtime/scroll.ts | 247 ++++++++++++++++++ 6 files changed, 490 insertions(+), 466 deletions(-) create mode 100644 src/commands/interaction/runtime/scroll.test.ts create mode 100644 src/commands/interaction/runtime/scroll.ts diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 47e4a1346d..3144d4633f 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -33,7 +33,7 @@ import { targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import type { ScrollInputDirection } from './runtime/gestures.ts'; +import type { ScrollInputDirection } from './runtime/scroll.ts'; export const interactionCliReaders = { click: (positionals, flags) => ({ diff --git a/src/commands/interaction/runtime/gestures.test.ts b/src/commands/interaction/runtime/gestures.test.ts index b7a6a7081b..97655ce402 100644 --- a/src/commands/interaction/runtime/gestures.test.ts +++ b/src/commands/interaction/runtime/gestures.test.ts @@ -5,7 +5,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { createInteractionDevice, dragTargetSnapshot, - runtimeScrollSnapshot, selectorSnapshot, } from './__tests__/test-utils/index.ts'; @@ -187,235 +186,6 @@ test('runtime longPress with settle drops the non-hittable hint when the diff pr assert.equal('hint' in result, false); }); -test('runtime scroll resolves selector targets before calling the backend primitive', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { scrolled: true }; - }, - }); - - const selectorResult = await device.interactions.scroll({ - session: 'default', - target: selector('label=Continue'), - direction: 'down', - pixels: 120, - durationMs: 50, - }); - const viewportResult = await device.interactions.scroll({ - direction: 'up', - amount: 0.5, - }); - - assert.equal(selectorResult.kind, 'selector'); - assert.equal(selectorResult.durationMs, undefined); - assert.equal(viewportResult.kind, 'viewport'); - assert.deepEqual(calls, [ - { - target: { kind: 'point', point: { x: 60, y: 40 } }, - options: { - direction: 'down', - pixels: 120, - durationMs: 50, - releaseBehavior: 'controlled', - }, - }, - { - target: { kind: 'viewport' }, - options: { direction: 'up', amount: 0.5, releaseBehavior: 'controlled' }, - }, - ]); -}); - -test('runtime scroll reports duration only when the backend honored it', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), - }); - - const result = await device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 50, - }); - - assert.equal(result.durationMs, 50); - assert.deepEqual(result.backendResult, { durationMs: 50 }); -}); - -test('runtime scroll rejects duration above the shared cap', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 10_001, - }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => { - throw new Error('snapshot unavailable'); - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - /Failed to verify scroll bottom state/, - ); - - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom does not scroll when no hidden content is below', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: unknown[] = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotScopes.push(options?.scope); - return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 1); - assert.equal(result.backendResult?.pass, 1); - assert.deepEqual(calls, [ - { - target: { kind: 'viewport' }, - options: { direction: 'down', releaseBehavior: 'inertial' }, - }, - ]); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - }), - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.passes, 2); - assert.equal(calls.length, 2); -}); - -test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotCount += 1; - if (options?.scope) throw new Error('scoped snapshot failed'); - return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; - }, - scroll: async () => ({}), - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => { - for (const surface of ['desktop', 'menubar'] as const) { - const device = createInteractionDevice(selectorSnapshot(), { - platform: 'macos', - sessionMetadata: { surface }, - scroll: async () => { - throw new Error(`${surface} scroll should be rejected before backend call`); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - target: { kind: 'viewport' }, - session: 'default', - }), - new RegExp(`scroll is not supported on macOS ${surface}`), - ); - } -}); - test('runtime multi-touch planning prefers backend viewport geometry without a snapshot capture', async () => { let capturedPlan: unknown; const device = createInteractionDevice(selectorSnapshot(), { diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index da17ed61e3..9e17a3f18b 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -4,36 +4,20 @@ import type { LongPressCommandResult, ResolutionDisclosure, } from '@agent-device/contracts/interaction'; -import type { ScrollDirection, ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; import { buildDragGesturePlan, singlePointerPlanEndpoints, } from '@agent-device/contracts/gesture-plan'; -import { - assertExclusiveScrollDistanceInputs, - honoredScrollDurationMs, - normalizeScrollDurationMs, - resolveScrollExecutionOptions, -} from '@agent-device/contracts/scroll-command'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import { - captureScrollEdgeState, - formatScrollEdgeMessage, - runScrollEdgePasses, - type ScrollEdge, - type ScrollEdgeState, - type ScrollEdgeTarget, -} from '@agent-device/capture-kit/scroll-edge-state'; import { successText } from '@agent-device/kernel/success-text'; import { requireIntInRange } from '@agent-device/kernel/validation'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type BackendResultEnvelope, - type BackendResultVariant, type RuntimeCommand, } from '../../runtime-types.ts'; import { @@ -125,48 +109,6 @@ export type HoverCommandOptions = CommandContext & { export type { HoverCommandResult }; -export type GestureDirection = ScrollDirection; -// The input vocabulary lives in contracts/scroll-gesture.ts beside the other scroll vocabularies, -// so the public API can declare `ScrollOptions` without depending on this command runtime. -export { type ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; - -export type ScrollTarget = - | InteractionTarget - | { - kind: 'viewport'; - }; - -export type ScrollCommandOptions = CommandContext & { - target?: ScrollTarget; - direction: ScrollInputDirection; - amount?: number; - pixels?: number; - durationMs?: number; -}; - -export type ScrollCommandResult = - | BackendResultVariant<{ - kind: 'viewport'; - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - }> - | BackendResultVariant< - ResolvedInteractionTarget & { - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - } - >; - -type ResolvedScrollTarget = { kind: 'viewport' } | ResolvedInteractionTarget; - export const focusCommand: RuntimeCommand = async ( runtime, options, @@ -405,182 +347,9 @@ function recordedDragTarget(target: ResolvedInteractionTarget): DragRecordingTar }; } -export const scrollCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => { - if (!runtime.backend.scroll) { - throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); - } - const target = resolveScrollDirection(options.direction); - const amount = normalizeOptionalPositiveNumber(options.amount, 'scroll amount'); - const pixels = normalizeOptionalPositiveInteger(options.pixels, 'scroll pixels'); - const durationMs = normalizeScrollDurationMs(options.durationMs); - assertExclusiveScrollDistanceInputs( - { amount, pixels }, - 'scroll accepts either amount or pixels, not both', - ); - - const resolved = await resolveScrollTarget(runtime, options); - const backendTarget = - resolved.kind === 'viewport' - ? { kind: 'viewport' as const } - : { kind: 'point' as const, point: requireResolvedPoint(resolved) }; - const scrollBackend = runtime.backend.scroll; - const executionOptions = resolveScrollExecutionOptions( - { - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - ...(durationMs !== undefined ? { durationMs } : {}), - }, - target.edge, - ); - const runScroll = async () => - await scrollBackend(toBackendContext(runtime, options), backendTarget, { - direction: target.direction, - ...executionOptions, - }); - let backendResult: Awaited>> | undefined; - let completedPasses = 0; - if (target.edge) { - const edge = target.edge; - const edgeTarget = buildScrollEdgeTarget(resolved); - const edgeResult = await runScrollEdgePasses({ - edge, - captureState: async (scope) => - await captureRuntimeScrollEdgeState(runtime, options, edge, edgeTarget, scope), - scroll: runScroll, - }); - backendResult = edgeResult.result; - completedPasses = edgeResult.passes; - } else { - backendResult = await runScroll(); - completedPasses = 1; - } - const formattedBackendResult = toBackendResult(backendResult); - const reportedDurationMs = honoredScrollDurationMs(formattedBackendResult); - return { - ...resolved, - direction: target.direction, - ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - ...(reportedDurationMs !== undefined ? { durationMs: reportedDurationMs } : {}), - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText( - formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), - ), - }; -}; - -async function resolveScrollTarget( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, -): Promise { - const target = options.target ?? { kind: 'viewport' as const }; - if (target.kind === 'viewport') { - await assertSupportedInteractionSurface(runtime, options, 'scroll'); - return { kind: 'viewport' }; - } - return await resolveInteractionTarget( - runtime, - { ...options, target }, - { - action: 'scroll', - requireInteractive: false, - pipeline: SELECTOR_PIPELINE_POLICIES.resolvedTarget, - }, - ); -} - -function resolveScrollDirection(direction: ScrollInputDirection): { - direction: GestureDirection; - edge?: 'top' | 'bottom'; -} { - if (direction === 'bottom') return { direction: 'down', edge: 'bottom' }; - if (direction === 'top') return { direction: 'up', edge: 'top' }; - return { direction: requireDirection(direction, 'scroll direction') }; -} - -function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget { - return resolved.kind === 'viewport' - ? {} - : { - point: resolved.point, - nodeIndex: 'node' in resolved ? resolved.node?.index : undefined, - }; -} - -function requireResolvedPoint(result: { point?: Point }): Point { +export function requireResolvedPoint(result: { point?: Point }): Point { if (!result.point) { throw new AppError('COMMAND_FAILED', 'Interaction target resolved without coordinates'); } return result.point; } - -async function captureRuntimeScrollEdgeState( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, - edge: ScrollEdge, - target: ScrollEdgeTarget, - scope?: string, -): Promise { - if (!runtime.backend.captureSnapshot) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - ); - } - const { captureSnapshot } = runtime.backend; - return await captureScrollEdgeState({ - edge, - target, - scope, - captureNodes: async (snapshotScope) => { - const result = await captureSnapshot(toBackendContext(runtime, options), { - scope: snapshotScope, - }); - return result.snapshot?.nodes ?? result.nodes ?? []; - }, - }); -} - -function requireDirection( - direction: GestureDirection | undefined, - field: string, -): GestureDirection { - switch (direction) { - case 'up': - case 'down': - case 'left': - case 'right': - return direction; - default: - throw new AppError('INVALID_ARGS', `${field} must be up, down, left, or right`); - } -} - -function normalizeOptionalPositiveNumber( - value: number | undefined, - field: string, -): number | undefined { - return value === undefined ? undefined : normalizePositiveNumber(value, field); -} - -function normalizePositiveNumber(value: number, field: string): number { - if (!Number.isFinite(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive number`); - } - return value; -} - -function normalizeOptionalPositiveInteger( - value: number | undefined, - field: string, -): number | undefined { - if (value === undefined) return undefined; - if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive integer`); - } - return value; -} diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index a0ca7b795b..c3829d6ed0 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -26,7 +26,7 @@ import { type InteractionTarget, } from './resolution.ts'; -export { focusCommand, hoverCommand, longPressCommand, scrollCommand } from './gestures.ts'; +export { focusCommand, hoverCommand, longPressCommand } from './gestures.ts'; export type { FocusCommandOptions, FocusCommandResult, @@ -34,9 +34,9 @@ export type { HoverCommandResult, LongPressCommandOptions, LongPressCommandResult, - ScrollCommandOptions, - ScrollCommandResult, } from './gestures.ts'; +export { scrollCommand } from './scroll.ts'; +export type { ScrollCommandOptions, ScrollCommandResult } from './scroll.ts'; export type { InteractionTarget } from './resolution.ts'; export type PressCommandOptions = CommandContext & diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts new file mode 100644 index 0000000000..fefd017928 --- /dev/null +++ b/src/commands/interaction/runtime/scroll.test.ts @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { selector } from './selector-read-utils.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createInteractionDevice, + runtimeScrollSnapshot, + selectorSnapshot, +} from './__tests__/test-utils/index.ts'; + +test('runtime scroll resolves selector targets before calling the backend primitive', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { scrolled: true }; + }, + }); + + const selectorResult = await device.interactions.scroll({ + session: 'default', + target: selector('label=Continue'), + direction: 'down', + pixels: 120, + durationMs: 50, + }); + const viewportResult = await device.interactions.scroll({ + direction: 'up', + amount: 0.5, + }); + + assert.equal(selectorResult.kind, 'selector'); + assert.equal(selectorResult.durationMs, undefined); + assert.equal(viewportResult.kind, 'viewport'); + assert.deepEqual(calls, [ + { + target: { kind: 'point', point: { x: 60, y: 40 } }, + options: { + direction: 'down', + pixels: 120, + durationMs: 50, + releaseBehavior: 'controlled', + }, + }, + { + target: { kind: 'viewport' }, + options: { direction: 'up', amount: 0.5, releaseBehavior: 'controlled' }, + }, + ]); +}); + +test('runtime scroll reports duration only when the backend honored it', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), + }); + + const result = await device.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 50, + }); + + assert.equal(result.durationMs, 50); + assert.deepEqual(result.backendResult, { durationMs: 50 }); +}); + +test('runtime scroll rejects duration above the shared cap', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async () => { + throw new Error('scroll should be rejected before backend call'); + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 10_001, + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /durationMs.*at most 10000/i.test(error.message), + ); +}); + +test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => { + throw new Error('snapshot unavailable'); + }, + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'bottom', + }), + /Failed to verify scroll bottom state/, + ); + + assert.equal(calls.length, 0); +}); + +test('runtime scroll bottom does not scroll when no hidden content is below', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.kind, 'viewport'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 0); + assert.equal(calls.length, 0); +}); + +test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { + const calls: unknown[] = []; + const snapshotScopes: unknown[] = []; + const snapshots = [ + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + ]; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async (_context, options) => { + snapshotScopes.push(options?.scope); + return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; + }, + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.kind, 'viewport'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 1); + assert.equal(result.backendResult?.pass, 1); + assert.deepEqual(calls, [ + { + target: { kind: 'viewport' }, + options: { direction: 'down', releaseBehavior: 'inertial' }, + }, + ]); + assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); +}); + +test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { + const calls: unknown[] = []; + const snapshots = [ + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), + ]; + let snapshotIndex = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ + snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], + }), + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.passes, 2); + assert.equal(calls.length, 2); +}); + +test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { + let snapshotCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async (_context, options) => { + snapshotCount += 1; + if (options?.scope) throw new Error('scoped snapshot failed'); + return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; + }, + scroll: async () => ({}), + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'bottom', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /scoped container/i.test(error.message) && + error.details?.scope === 'Messages', + ); + assert.equal(snapshotCount, 2); +}); + +test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => { + for (const surface of ['desktop', 'menubar'] as const) { + const device = createInteractionDevice(selectorSnapshot(), { + platform: 'macos', + sessionMetadata: { surface }, + scroll: async () => { + throw new Error(`${surface} scroll should be rejected before backend call`); + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'down', + target: { kind: 'viewport' }, + session: 'default', + }), + new RegExp(`scroll is not supported on macOS ${surface}`), + ); + } +}); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts new file mode 100644 index 0000000000..76241bb664 --- /dev/null +++ b/src/commands/interaction/runtime/scroll.ts @@ -0,0 +1,247 @@ +import { + assertExclusiveScrollDistanceInputs, + honoredScrollDurationMs, + normalizeScrollDurationMs, + resolveScrollExecutionOptions, +} from '@agent-device/contracts/scroll-command'; +import type { ScrollDirection, ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; +import { + captureScrollEdgeState, + formatScrollEdgeMessage, + runScrollEdgePasses, + type ScrollEdge, + type ScrollEdgeState, + type ScrollEdgeTarget, +} from '@agent-device/capture-kit/scroll-edge-state'; +import { AppError } from '@agent-device/kernel/errors'; +import { successText } from '@agent-device/kernel/success-text'; +import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; +import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; +import { toBackendContext } from '../../runtime-common.ts'; +import { + toBackendResult, + type BackendResultVariant, + type RuntimeCommand, +} from '../../runtime-types.ts'; +import { requireResolvedPoint } from './gestures.ts'; +import { + assertSupportedInteractionSurface, + resolveInteractionTarget, + type InteractionTarget, + type ResolvedInteractionTarget, +} from './resolution.ts'; + +export type GestureDirection = ScrollDirection; +// The input vocabulary lives in contracts/scroll-gesture.ts beside the other scroll vocabularies, +// so the public API can declare `ScrollOptions` without depending on this command runtime. +export { type ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; + +export type ScrollTarget = + | InteractionTarget + | { + kind: 'viewport'; + }; + +export type ScrollCommandOptions = CommandContext & { + target?: ScrollTarget; + direction: ScrollInputDirection; + amount?: number; + pixels?: number; + durationMs?: number; +}; + +export type ScrollCommandResult = + | BackendResultVariant<{ + kind: 'viewport'; + direction: GestureDirection; + edge?: 'top' | 'bottom'; + passes?: number; + amount?: number; + pixels?: number; + durationMs?: number; + }> + | BackendResultVariant< + ResolvedInteractionTarget & { + direction: GestureDirection; + edge?: 'top' | 'bottom'; + passes?: number; + amount?: number; + pixels?: number; + durationMs?: number; + } + >; + +type ResolvedScrollTarget = { kind: 'viewport' } | ResolvedInteractionTarget; + +export const scrollCommand: RuntimeCommand = async ( + runtime, + options, +): Promise => { + if (!runtime.backend.scroll) { + throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); + } + const target = resolveScrollDirection(options.direction); + const amount = normalizeOptionalPositiveNumber(options.amount, 'scroll amount'); + const pixels = normalizeOptionalPositiveInteger(options.pixels, 'scroll pixels'); + const durationMs = normalizeScrollDurationMs(options.durationMs); + assertExclusiveScrollDistanceInputs( + { amount, pixels }, + 'scroll accepts either amount or pixels, not both', + ); + + const resolved = await resolveScrollTarget(runtime, options); + const backendTarget = + resolved.kind === 'viewport' + ? { kind: 'viewport' as const } + : { kind: 'point' as const, point: requireResolvedPoint(resolved) }; + const scrollBackend = runtime.backend.scroll; + const executionOptions = resolveScrollExecutionOptions( + { + ...(amount !== undefined ? { amount } : {}), + ...(pixels !== undefined ? { pixels } : {}), + ...(durationMs !== undefined ? { durationMs } : {}), + }, + target.edge, + ); + const runScroll = async () => + await scrollBackend(toBackendContext(runtime, options), backendTarget, { + direction: target.direction, + ...executionOptions, + }); + let backendResult: Awaited>> | undefined; + let completedPasses = 0; + if (target.edge) { + const edge = target.edge; + const edgeTarget = buildScrollEdgeTarget(resolved); + const edgeResult = await runScrollEdgePasses({ + edge, + captureState: async (scope) => + await captureRuntimeScrollEdgeState(runtime, options, edge, edgeTarget, scope), + scroll: runScroll, + }); + backendResult = edgeResult.result; + completedPasses = edgeResult.passes; + } else { + backendResult = await runScroll(); + completedPasses = 1; + } + const formattedBackendResult = toBackendResult(backendResult); + const reportedDurationMs = honoredScrollDurationMs(formattedBackendResult); + return { + ...resolved, + direction: target.direction, + ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), + ...(amount !== undefined ? { amount } : {}), + ...(pixels !== undefined ? { pixels } : {}), + ...(reportedDurationMs !== undefined ? { durationMs: reportedDurationMs } : {}), + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText( + formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), + ), + }; +}; + +async function resolveScrollTarget( + runtime: AgentDeviceRuntime, + options: ScrollCommandOptions, +): Promise { + const target = options.target ?? { kind: 'viewport' as const }; + if (target.kind === 'viewport') { + await assertSupportedInteractionSurface(runtime, options, 'scroll'); + return { kind: 'viewport' }; + } + return await resolveInteractionTarget( + runtime, + { ...options, target }, + { + action: 'scroll', + requireInteractive: false, + pipeline: SELECTOR_PIPELINE_POLICIES.resolvedTarget, + }, + ); +} + +function resolveScrollDirection(direction: ScrollInputDirection): { + direction: GestureDirection; + edge?: 'top' | 'bottom'; +} { + if (direction === 'bottom') return { direction: 'down', edge: 'bottom' }; + if (direction === 'top') return { direction: 'up', edge: 'top' }; + return { direction: requireDirection(direction, 'scroll direction') }; +} + +function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget { + return resolved.kind === 'viewport' + ? {} + : { + point: resolved.point, + nodeIndex: 'node' in resolved ? resolved.node?.index : undefined, + }; +} + +async function captureRuntimeScrollEdgeState( + runtime: AgentDeviceRuntime, + options: ScrollCommandOptions, + edge: ScrollEdge, + target: ScrollEdgeTarget, + scope?: string, +): Promise { + if (!runtime.backend.captureSnapshot) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, + ); + } + const { captureSnapshot } = runtime.backend; + return await captureScrollEdgeState({ + edge, + target, + scope, + captureNodes: async (snapshotScope) => { + const result = await captureSnapshot(toBackendContext(runtime, options), { + scope: snapshotScope, + }); + return result.snapshot?.nodes ?? result.nodes ?? []; + }, + }); +} + +function requireDirection( + direction: GestureDirection | undefined, + field: string, +): GestureDirection { + switch (direction) { + case 'up': + case 'down': + case 'left': + case 'right': + return direction; + default: + throw new AppError('INVALID_ARGS', `${field} must be up, down, left, or right`); + } +} + +function normalizeOptionalPositiveNumber( + value: number | undefined, + field: string, +): number | undefined { + return value === undefined ? undefined : normalizePositiveNumber(value, field); +} + +function normalizePositiveNumber(value: number, field: string): number { + if (!Number.isFinite(value) || value <= 0) { + throw new AppError('INVALID_ARGS', `${field} must be a positive number`); + } + return value; +} + +function normalizeOptionalPositiveInteger( + value: number | undefined, + field: string, +): number | undefined { + if (value === undefined) return undefined; + if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) { + throw new AppError('INVALID_ARGS', `${field} must be a positive integer`); + } + return value; +} From 636f3202ebfc4c017fd381b362df77dff9c6e20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 9 Sep 2026 22:16:58 +0200 Subject: [PATCH 02/13] feat(scroll): add --until , report honored travel, fix web amount units --- packages/capture-kit/package.json | 4 + ...roll-edge-state-pass-orchestration.test.ts | 19 ++ .../__tests__/scroll-until-visible.test.ts | 152 +++++++++++ .../src/snapshot/scroll-edge-state.ts | 15 +- .../src/snapshot/scroll-until-visible.ts | 119 +++++++++ packages/contracts/src/cli-flags.ts | 2 + packages/contracts/src/client-gesture.ts | 2 + packages/contracts/src/client-request.ts | 2 + .../src/platform-runtime-operations.ts | 36 ++- packages/contracts/src/scroll-command.ts | 18 ++ packages/contracts/src/scroll-gesture.ts | 8 +- .../src/agent-browser-provider.test.ts | 25 ++ .../src/agent-browser-provider.ts | 62 +++-- packages/selectors/package.json | 6 +- .../selectors/src/scroll-until-match.test.ts | 87 +++++++ packages/selectors/src/scroll-until-match.ts | 50 ++++ scripts/integration-progress-model.ts | 1 + scripts/layering/package-boundaries.test.ts | 2 + skills/agent-device/SKILL.md | 2 + src/cli-schema/cli-help.ts | 8 +- .../cli-grammar/flag-definitions-action.ts | 7 + src/commands/command-flags.ts | 1 + src/commands/interaction/index.ts | 4 +- src/commands/interaction/interactions.ts | 1 + src/commands/interaction/metadata.ts | 5 +- .../interaction/runtime/resolution.test.ts | 8 +- .../interaction/runtime/resolution.ts | 36 ++- .../interaction/runtime/scroll.test.ts | 96 +++++++ src/commands/interaction/runtime/scroll.ts | 246 +++++++++++++++--- src/core/dispatch-context.ts | 1 + src/daemon/__tests__/scroll-runtime.test.ts | 89 +++++++ .../interaction-touch-runtime.test.ts | 9 +- src/daemon/scroll-runtime.ts | 117 ++++++++- src/mcp/command-output-schemas.ts | 3 +- src/mcp/server-guide.ts | 2 +- 35 files changed, 1142 insertions(+), 103 deletions(-) create mode 100644 packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts create mode 100644 packages/capture-kit/src/snapshot/scroll-until-visible.ts create mode 100644 packages/selectors/src/scroll-until-match.test.ts create mode 100644 packages/selectors/src/scroll-until-match.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 4a997111cf..c5a61f9401 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -118,6 +118,10 @@ "types": "./src/snapshot/scroll-edge-state.ts", "default": "./src/snapshot/scroll-edge-state.ts" }, + "./scroll-until-visible": { + "types": "./src/snapshot/scroll-until-visible.ts", + "default": "./src/snapshot/scroll-until-visible.ts" + }, "./snapshot-chrome": { "types": "./src/snapshot-chrome.ts", "default": "./src/snapshot-chrome.ts" diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts index 19c547b789..f005969301 100644 --- a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts @@ -55,6 +55,25 @@ test('formatScrollEdgeMessage: pixels takes priority over amount when both are s assert.equal(formatScrollEdgeMessage('down', undefined, 0, 3, 250), 'Scrolled down by 250px'); }); +/** + * One gesture saturates at the viewport axis minus its edge padding, so a large amount buys less + * travel than it names. The message reports what the planner honored rather than what was asked. + */ +test('an amount-based message names the honored travel when the planner reports it', () => { + assert.equal( + formatScrollEdgeMessage('down', undefined, 1, 3, undefined, 640), + 'Scrolled down by 3 of the viewport (640px)', + ); + assert.equal( + formatScrollEdgeMessage('down', undefined, 1, 0.65, undefined, undefined), + 'Scrolled down by 0.65', + ); + assert.equal( + formatScrollEdgeMessage('down', undefined, 1, undefined, 5000, 640), + 'Scrolled down by 640px', + ); +}); + // --------------------------------------------------------------------------- // captureScrollEdgeState: retry-without-scope on an empty scoped capture // --------------------------------------------------------------------------- diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts new file mode 100644 index 0000000000..1529e81f61 --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts @@ -0,0 +1,152 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + SCROLL_UNTIL_PASS_LIMIT, + formatScrollUntilMessage, + runScrollUntilVisiblePasses, + scrollUntilNotFoundError, +} from '../scroll-until-visible.ts'; + +/** A tree the edge analyzer reads as "more content below": a scrollable with a clipped child. */ +function scrollableTree(childY: number): SnapshotNode[] { + return [ + { + index: 0, + ref: 'e1', + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + } as SnapshotNode, + { + index: 1, + parentIndex: 0, + ref: 'e2', + type: 'ScrollView', + rect: { x: 0, y: 0, width: 400, height: 800 }, + } as SnapshotNode, + { + index: 2, + parentIndex: 1, + ref: 'e3', + type: 'TextField', + rect: { x: 0, y: childY, width: 400, height: 40 }, + } as SnapshotNode, + ]; +} + +test('a target that is already visible costs one capture and zero scrolls', async () => { + let scrolls = 0; + const outcome = await runScrollUntilVisiblePasses({ + edge: 'bottom', + captureNodes: async () => scrollableTree(100), + isVisibleMatch: () => true, + scroll: async () => { + scrolls += 1; + return { scrolled: true }; + }, + }); + assert.equal(outcome.outcome, 'matched'); + assert.equal(outcome.passes, 0); + assert.equal(scrolls, 0); +}); + +test('passes repeat until the injected predicate reports the target on screen', async () => { + let scrolls = 0; + const outcome = await runScrollUntilVisiblePasses({ + edge: 'bottom', + captureNodes: async () => scrollableTree(2000), + isVisibleMatch: () => scrolls >= 3, + scroll: async () => { + scrolls += 1; + return { pixels: 250 }; + }, + }); + assert.equal(outcome.outcome, 'matched'); + assert.equal(outcome.passes, 3); + assert.deepEqual(outcome.result, { pixels: 250 }); +}); + +test('running out of content stops the loop before the pass budget does', async () => { + let scrolls = 0; + const outcome = await runScrollUntilVisiblePasses({ + edge: 'bottom', + // No child below the fold: the edge analyzer reports nothing hidden underneath. + captureNodes: async () => scrollableTree(100), + isVisibleMatch: () => false, + scroll: async () => { + scrolls += 1; + return {}; + }, + }); + assert.equal(outcome.outcome, 'edge-reached'); + assert.equal(scrolls, 0); +}); + +test('a horizontal scroll has no edge signal and is bounded by the pass budget alone', async () => { + let scrolls = 0; + const outcome = await runScrollUntilVisiblePasses({ + passLimit: 4, + captureNodes: async () => scrollableTree(100), + isVisibleMatch: () => false, + scroll: async () => { + scrolls += 1; + return {}; + }, + }); + assert.equal(outcome.outcome, 'pass-limit'); + assert.equal(outcome.passes, 4); + assert.equal(scrolls, 4); +}); + +test('the default pass budget is the shared constant', async () => { + const outcome = await runScrollUntilVisiblePasses({ + captureNodes: async () => scrollableTree(100), + isVisibleMatch: () => false, + scroll: async () => ({}), + }); + assert.equal(outcome.passes, SCROLL_UNTIL_PASS_LIMIT); +}); + +test('an empty capture never counts as a match', async () => { + const outcome = await runScrollUntilVisiblePasses({ + passLimit: 1, + captureNodes: async () => [], + isVisibleMatch: (nodes) => nodes.length > 0, + scroll: async () => ({}), + }); + assert.equal(outcome.outcome, 'pass-limit'); +}); + +test('the success message distinguishes an already-visible target from a scrolled one', () => { + assert.equal( + formatScrollUntilMessage('down', 'id=email', 0), + 'id=email was already visible; no down scroll needed', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 1), + 'Scrolled down 1 pass until id=email was visible', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 3), + 'Scrolled down 3 passes until id=email was visible', + ); +}); + +test('the two failures carry distinct typed reasons and distinct corrective hints', () => { + const edge = scrollUntilNotFoundError({ + direction: 'down', + selector: 'id=email', + outcome: 'edge-reached', + passes: 2, + }); + const budget = scrollUntilNotFoundError({ + direction: 'down', + selector: 'id=email', + outcome: 'pass-limit', + passes: 12, + }); + assert.equal(edge.details?.reason, 'scroll_until_edge_reached'); + assert.equal(budget.details?.reason, 'scroll_until_pass_limit'); + assert.match(String(edge.details?.hint), /scroll the opposite direction/); + assert.match(String(budget.details?.hint), /Raise the step with an amount/); +}); diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index f2d7e2a4d3..0c2a57104f 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -69,19 +69,30 @@ export async function runScrollEdgePasses(params: { return { passes, result }; } +/** + * `honoredPixels` is the travel the gesture planner actually produced, which is not always the + * travel that was asked for: one gesture cannot cross more than the viewport axis minus its edge + * padding, so a large `amount` saturates. Naming the honored distance is what keeps + * `scroll down 3` from reporting a three-viewport scroll it never performed. + */ export function formatScrollEdgeMessage( direction: ScrollDirection, edge: ScrollEdge | undefined, passes: number, amount: number | undefined, pixels: number | undefined, + honoredPixels?: number, ): string { if (edge && passes === 0) { return `Already at ${edge}; no hidden content ${edge === 'bottom' ? 'below' : 'above'} detected`; } if (edge) return `Scrolled to ${edge} with ${passes} ${direction} passes`; - if (pixels !== undefined) return `Scrolled ${direction} by ${pixels}px`; - if (amount !== undefined) return `Scrolled ${direction} by ${amount}`; + if (pixels !== undefined) return `Scrolled ${direction} by ${honoredPixels ?? pixels}px`; + if (amount !== undefined) { + return honoredPixels === undefined + ? `Scrolled ${direction} by ${amount}` + : `Scrolled ${direction} by ${amount} of the viewport (${honoredPixels}px)`; + } return `Scrolled ${direction}`; } diff --git a/packages/capture-kit/src/snapshot/scroll-until-visible.ts b/packages/capture-kit/src/snapshot/scroll-until-visible.ts new file mode 100644 index 0000000000..b29579182b --- /dev/null +++ b/packages/capture-kit/src/snapshot/scroll-until-visible.ts @@ -0,0 +1,119 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { ScrollEdge } from './scroll-edge-state.ts'; + +/** + * How many gestures one `scroll --until` may spend before it gives up. A pass costs a capture plus + * a gesture, so this is the request's whole cost ceiling, not a retry budget: 12 passes at the + * honored 0.8-viewport maximum cover roughly ten screens of content, which is past the point where + * a list is better reached by `scroll bottom` or a search field. + */ +export const SCROLL_UNTIL_PASS_LIMIT = 12; + +/** + * Why the loop stopped. `matched` is the only success; the other two are the two distinguishable + * ways a target never came into view, and callers report them differently because the corrective + * action differs — an exhausted list needs a different direction, an exhausted budget needs a + * bigger step or a narrower selector. + */ +export type ScrollUntilVisibleOutcome = 'matched' | 'edge-reached' | 'pass-limit'; + +export type ScrollUntilVisibleResult = { + passes: number; + outcome: ScrollUntilVisibleOutcome; + result?: TResult; +}; + +type CapturedNodes = readonly (RawSnapshotNode | SnapshotNode)[]; + +/** + * Scrolls until an injected predicate says the target is on screen. + * + * The predicate is injected rather than resolved here because the two callers (the daemon's generic + * scroll route and the in-process command runtime) reach selector matching through different + * layers; keeping the loop predicate-shaped is what lets both share one definition of when to stop. + * + * `edge` is the end-of-content signal, and it is the SAME signal `scroll top`/`scroll bottom` + * already trust (`analyzeScrollEdgeState`), so a list that reports no room below stops this loop + * exactly where an edge scroll would stop. Horizontal scrolls have no such analyzer and are bounded + * by `passLimit` alone. + * + * The first capture happens before the first gesture: a target that is already visible costs one + * capture and zero scrolls. + */ +export async function runScrollUntilVisiblePasses(params: { + edge?: ScrollEdge; + passLimit?: number; + captureNodes: () => Promise; + isVisibleMatch: (nodes: CapturedNodes) => Promise | boolean; + scroll: () => Promise; +}): Promise> { + const { edge, captureNodes, isVisibleMatch, scroll } = params; + const passLimit = params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT; + let passes = 0; + let result: TResult | undefined; + const stop = (outcome: ScrollUntilVisibleOutcome): ScrollUntilVisibleResult => ({ + passes, + outcome, + ...(result === undefined ? {} : { result }), + }); + + while (true) { + const nodes = await captureNodes(); + if (await isVisibleMatch(nodes)) return stop('matched'); + if (edge && !(await canScrollFurther(nodes, edge))) return stop('edge-reached'); + if (passes >= passLimit) return stop('pass-limit'); + result = await scroll(); + passes += 1; + } +} + +async function canScrollFurther(nodes: CapturedNodes, edge: ScrollEdge): Promise { + const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts'); + return analyzeScrollEdgeState(nodes, edge).canScroll; +} + +export function formatScrollUntilMessage( + direction: ScrollDirection, + selector: string, + passes: number, +): string { + if (passes === 0) return `${selector} was already visible; no ${direction} scroll needed`; + return `Scrolled ${direction} ${passes} ${passes === 1 ? 'pass' : 'passes'} until ${selector} was visible`; +} + +/** + * The two ways the loop can end without the target on screen. They are separate messages because + * the corrective action differs: content that ran out needs a different direction or a target that + * is not on this screen at all, while an exhausted budget needs a bigger step or a selector that + * matches something nearer. + */ +export function scrollUntilNotFoundError(params: { + direction: ScrollDirection; + selector: string; + outcome: Exclude; + passes: number; +}): AppError { + const { direction, selector, outcome, passes } = params; + if (outcome === 'edge-reached') { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`, + { + reason: 'scroll_until_edge_reached', + details: { selector, direction, passes }, + hint: `Nothing further lies ${direction} of here. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, + }, + ); + } + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, + { + reason: 'scroll_until_pass_limit', + details: { selector, direction, passes }, + hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, + }, + ); +} diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index 0764ee8895..c5ba3f1649 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -109,6 +109,8 @@ export type CliFlags = CloudProviderProfileFields & holdMs?: number; jitterPx?: number; pixels?: number; + /** Scroll: repeat passes until this selector is visible on screen. */ + until?: string; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 8206c6e769..5928b727a9 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -146,4 +146,6 @@ export type ScrollOptions = DeviceCommandBaseOptions & amount?: number; pixels?: number; durationMs?: number; + /** Repeat scroll passes until this selector is visible on screen, then stop. */ + until?: string; }; diff --git a/packages/contracts/src/client-request.ts b/packages/contracts/src/client-request.ts index a415a67c68..81f8b25321 100644 --- a/packages/contracts/src/client-request.ts +++ b/packages/contracts/src/client-request.ts @@ -39,6 +39,8 @@ export type CommandExecutionOptions = Partial & holdMs?: number; jitterPx?: number; pixels?: number; + /** Scroll: repeat passes until this selector is visible on screen. */ + until?: string; doubleTap?: boolean; verify?: boolean; settle?: boolean; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 12aa3605ee..5f85b87a5c 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -249,12 +249,14 @@ export const gestureViewportRuntimeUse = defineUse({ required: ['gestureViewport /** `scroll ` executes one pass and needs nothing else. */ const scrollDirectionUse = defineUse({ required: ['scrollDirection'] }); /** - * `scroll top` / `scroll bottom` verify hidden content between passes, so the capture is part of - * the tier's requirement rather than something discovered mid-run — the retired leaf's + * Every scroll that verifies between passes: `scroll top`/`scroll bottom` read hidden content at + * the edge, and `scroll --until ` re-reads the tree to decide whether the target came + * into view. Both check against that capture rather than a stale session snapshot, so the capture + * is part of the tier's requirement rather than something discovered mid-run — the retired leaf's * "requires snapshot support to verify hidden content before scrolling" refusal, moved to - * admission. + * admission. One declaration, because the two tiers admit on identical facts. */ -const scrollEdgeUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); +const scrollVerifiedPassUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); const gestureUsesByTier = Object.freeze({ plan: gesturePlanUse, @@ -275,7 +277,10 @@ export const gestureRuntimePlanUses = Object.freeze([ export const swipeRuntimePlanUses = Object.freeze([gesturePlanUse] as const); /** Every use `scroll` can select between. */ -export const scrollRuntimePlanUses = Object.freeze([scrollDirectionUse, scrollEdgeUse] as const); +export const scrollRuntimePlanUses = Object.freeze([ + scrollDirectionUse, + scrollVerifiedPassUse, +] as const); type GesturePlanFor = Readonly<{ tier: Tier; @@ -337,15 +342,28 @@ function gesturePlan(tier: Tier): Gesture */ export type ScrollRuntimePlan = | Readonly<{ kind: 'direction'; use: typeof scrollDirectionUse }> - | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollEdgeUse }>; + | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollVerifiedPassUse }> + | Readonly<{ kind: 'until'; until: string; use: typeof scrollVerifiedPassUse }>; -/** `scroll top`/`scroll bottom` verify between passes; every other scroll executes one pass. */ +/** + * `scroll top`/`scroll bottom` verify between passes, `scroll --until` re-reads the tree between + * passes to check its selector, and every other scroll executes one pass. The edge directions carry + * their own stop condition, so pairing them with `--until` names two, which the caller rejects + * before this resolves. + */ export function resolveScrollRuntimePlan( - input: Readonly<{ edge?: 'top' | 'bottom' }>, + input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>, ): ScrollRuntimePlan { + if (input.until !== undefined) { + return Object.freeze({ + kind: 'until', + until: input.until, + use: scrollVerifiedPassUse, + } as const); + } return input.edge === undefined ? Object.freeze({ kind: 'direction', use: scrollDirectionUse } as const) - : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollEdgeUse } as const); + : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollVerifiedPassUse } as const); } const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index f0632d2a59..50db871878 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -47,6 +47,24 @@ export function assertExclusiveScrollDistanceInputs( } } +/** + * `top`/`bottom` are scroll-to-extreme requests that already carry a stop condition, so pairing one + * with `--until` names two and the request has no single meaning. Rejected at the surface rather + * than resolved by precedence, so neither stop condition can silently win. + */ +export function assertScrollUntilCompatible( + input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>, +): void { + if (input.until === undefined || input.edge === undefined) return; + throw new AppError( + 'INVALID_ARGS', + `scroll ${input.edge} already scrolls to the ${input.edge} edge and cannot take --until`, + { + hint: `Use scroll ${input.edge === 'bottom' ? 'down' : 'up'} --until to stop at the target, or scroll ${input.edge} to reach the edge.`, + }, + ); +} + export function normalizeScrollDurationMs( durationMs: number | undefined, options: { field?: string; invalidMessage?: string; max?: number } = {}, diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 6d09cfac42..cdc7ddc971 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -83,7 +83,13 @@ export type InPageSwipeGesturePlan = { referenceHeight: number; }; -const DEFAULT_SCROLL_AMOUNT = 0.6; +/** + * The finger-path fraction of the viewport axis one scroll covers when the caller names no + * distance. Exported because a backend with no viewport to measure against (the browser) scales its + * own default step by the ratio to this, and that ratio is only meaningful while both sides read + * the same number. + */ +export const DEFAULT_SCROLL_AMOUNT = 0.6; // Scroll gestures never touch the outer 10% of either axis. Modern app windows are edge-to-edge, // so the viewport includes the system bars: a swipe that starts inside the status bar (5.7% of a // Pixel 7's height, 6.9% of an iPhone's with a Dynamic Island) pulls the notification shade or diff --git a/packages/platform-web/src/agent-browser-provider.test.ts b/packages/platform-web/src/agent-browser-provider.test.ts index d2629b2a14..da8f32b130 100644 --- a/packages/platform-web/src/agent-browser-provider.test.ts +++ b/packages/platform-web/src/agent-browser-provider.test.ts @@ -851,3 +851,28 @@ function createAudioProbeScriptPage(): AudioProbeScriptPage { }, }; } + +/** + * #2432: `amount` is a fraction of the viewport axis everywhere else, and the browser scrolls by + * CSS pixels. Passing it through raw made `scroll down 0.5` travel half a pixel. + */ +test('a relative scroll amount reaches agent-browser as pixels, not as the fraction itself', async () => { + await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { + const calls: AgentBrowserCall[] = []; + + await withCommandExecutorOverride(recordingExecutor(calls), async () => { + await provider.scroll('down', { amount: 0.6 }); + await provider.scroll('down', { amount: 1.2 }); + await provider.scroll('down', undefined); + }); + + assert.deepEqual( + calls.map((call) => call.args), + [ + ['scroll', 'down', '300', '--json', '--session', 'web-session'], + ['scroll', 'down', '600', '--json', '--session', 'web-session'], + ['scroll', 'down', '--json', '--session', 'web-session'], + ], + ); + }); +}); diff --git a/packages/platform-web/src/agent-browser-provider.ts b/packages/platform-web/src/agent-browser-provider.ts index 8e37fe543d..a80f9045c3 100644 --- a/packages/platform-web/src/agent-browser-provider.ts +++ b/packages/platform-web/src/agent-browser-provider.ts @@ -27,6 +27,7 @@ import { cleanupManagedAgentBrowserOrphansForProviderStartup, recordManagedAgentBrowserProcesses, } from './agent-browser-lifecycle.ts'; +import { DEFAULT_SCROLL_AMOUNT } from '@agent-device/contracts/scroll-gesture'; const AGENT_BROWSER = 'agent-browser'; const AGENT_BROWSER_TIMEOUT_MS = 30_000; @@ -125,7 +126,7 @@ async function runPacedScroll( direction: string, scrollOptions: { amount?: number; pixels?: number; durationMs?: number } | undefined, ): Promise { - const steps = buildPacedScrollSteps(scrollOptions); + const steps = buildPacedScrollSteps(resolveWebScrollDistance(scrollOptions)); for (const step of steps) { await runJson(buildScrollArgs(direction, step.distance)); if (step.delayAfterMs > 0) await sleep(step.delayAfterMs); @@ -137,34 +138,59 @@ type ScrollStep = { delayAfterMs: number; }; -function buildPacedScrollSteps( +/** agent-browser's own default wheel step, and the distance the default amount maps onto. */ +const WEB_DEFAULT_SCROLL_PIXELS = 300; + +type WebScrollDistance = { + distance?: number; + durationMs?: number; +}; + +/** + * The browser scrolls by CSS pixels, so a relative `amount` has to become one before it reaches + * agent-browser — feeding it through raw made `scroll down 0.5` travel half a pixel. + * + * There is no gesture viewport to measure against on this backend, so `amount` scales the default + * step the same way the Linux pointer backend scales its wheel clicks: the shared default amount + * maps to the default step, and everything else is proportional to it. + */ +function resolveWebScrollDistance( scrollOptions: { amount?: number; pixels?: number; durationMs?: number } | undefined, -): ScrollStep[] { - const requestedDistance = scrollOptions?.pixels ?? scrollOptions?.amount; +): WebScrollDistance { const durationMs = scrollOptions?.durationMs; + const timing = durationMs === undefined ? {} : { durationMs }; + if (scrollOptions?.pixels !== undefined) { + return { distance: scrollOptions.pixels, ...timing }; + } + if (scrollOptions?.amount !== undefined) { + return { + distance: Math.max( + 1, + Math.round((WEB_DEFAULT_SCROLL_PIXELS * scrollOptions.amount) / DEFAULT_SCROLL_AMOUNT), + ), + ...timing, + }; + } + return timing; +} + +function buildPacedScrollSteps(scrollDistance: WebScrollDistance): ScrollStep[] { + const { distance, durationMs } = scrollDistance; if (durationMs === undefined || durationMs <= 0) { - return [{ distance: requestedDistance, delayAfterMs: 0 }]; + return [{ distance, delayAfterMs: 0 }]; } const stepCount = Math.max(1, Math.min(20, Math.ceil(durationMs / 50))); const intervalMs = durationMs / Math.max(1, stepCount - 1); - return scrollStepDistances(scrollOptions, stepCount).map((distance, index) => ({ - distance, + return distributeIntegerDistance( + Math.round(distance ?? WEB_DEFAULT_SCROLL_PIXELS), + stepCount, + ).map((stepDistance, index) => ({ + distance: stepDistance, delayAfterMs: index < stepCount - 1 ? intervalMs : 0, })); } -function scrollStepDistances( - scrollOptions: { amount?: number; pixels?: number } | undefined, - stepCount: number, -): number[] { - const totalDistance = scrollOptions?.pixels ?? scrollOptions?.amount ?? 300; - if (scrollOptions?.amount !== undefined && scrollOptions.pixels === undefined) { - return Array.from({ length: stepCount }, () => totalDistance / stepCount); - } - return distributeIntegerDistance(Math.round(totalDistance), stepCount); -} - function distributeIntegerDistance(totalDistance: number, stepCount: number): number[] { const baseDistance = Math.floor(totalDistance / stepCount); const remainder = totalDistance - baseDistance * stepCount; diff --git a/packages/selectors/package.json b/packages/selectors/package.json index edd45f37de..7bbe050d85 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -4,7 +4,7 @@ "private": true, "sideEffects": false, "type": "module", - "description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19); `./parameterized-recorded-fill` parameterizes recorded fill payloads against their selectors; the interaction-resolution subpaths (`./selector-pipeline`, `./interaction-targeting`, `./interaction-touch-point`, `./press-retarget`, `./absence-observation*`, …) host the engine execution surface owned by the pipeline; `./snapshot-geometry-fixtures` is the canonical home for the geometry/touch-point test fixtures this package and root tests both build on (it re-exports `makeSnapshotState` from `@agent-device/capture-kit/snapshot-state-fixtures`, its own canonical home).", + "description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19); `./parameterized-recorded-fill` parameterizes recorded fill payloads against their selectors; the interaction-resolution subpaths (`./selector-pipeline`, `./interaction-targeting`, `./interaction-touch-point`, `./press-retarget`, `./absence-observation*`, \u2026) host the engine execution surface owned by the pipeline; `./snapshot-geometry-fixtures` is the canonical home for the geometry/touch-point test fixtures this package and root tests both build on (it re-exports `makeSnapshotState` from `@agent-device/capture-kit/snapshot-state-fixtures`, its own canonical home).", "dependencies": { "@agent-device/ad-script": "workspace:*", "@agent-device/capture-kit": "workspace:*", @@ -68,6 +68,10 @@ "types": "./src/selector-pipeline.ts", "default": "./src/selector-pipeline.ts" }, + "./scroll-until-match": { + "types": "./src/scroll-until-match.ts", + "default": "./src/scroll-until-match.ts" + }, "./selector-pipeline-policy": { "types": "./src/selector-pipeline-policy.ts", "default": "./src/selector-pipeline-policy.ts" diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts new file mode 100644 index 0000000000..94b50aea05 --- /dev/null +++ b/packages/selectors/src/scroll-until-match.test.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { isSelectorVisibleInNodes } from './scroll-until-match.ts'; + +const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; + +function tree(...rows: { ref: string; label: string; y: number }[]): SnapshotNode[] { + return [ + { index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode, + ...rows.map( + (row, offset) => + ({ + index: offset + 1, + parentIndex: 0, + ref: row.ref, + type: 'Button', + label: row.label, + rect: { x: 0, y: row.y, width: 400, height: 40 }, + }) as SnapshotNode, + ), + ]; +} + +test('a match inside the viewport is visible', async () => { + assert.equal( + await isSelectorVisibleInNodes({ + nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), + selector: 'label=Submit', + platform: 'ios', + }), + true, + ); +}); + +test('a match scrolled below the fold is present but not visible', async () => { + assert.equal( + await isSelectorVisibleInNodes({ + nodes: tree({ ref: 'e2', label: 'Submit', y: 2400 }), + selector: 'label=Submit', + platform: 'ios', + }), + false, + ); +}); + +test('a selector matching nothing is not visible', async () => { + assert.equal( + await isSelectorVisibleInNodes({ + nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), + selector: 'label=Missing', + platform: 'ios', + }), + false, + ); +}); + +test('an empty capture is not a match', async () => { + assert.equal( + await isSelectorVisibleInNodes({ nodes: [], selector: 'label=Submit', platform: 'ios' }), + false, + ); +}); + +/** + * The reason the predicate asks "some match", not "the first match": a list can hold rows that + * share a selector, and the one above the fold must not end a scroll that has not yet reached the + * row the caller can act on. + */ +test('an off-screen twin does not satisfy a selector whose other match is on screen', async () => { + assert.equal( + await isSelectorVisibleInNodes({ + nodes: tree({ ref: 'e2', label: 'Row', y: -900 }, { ref: 'e3', label: 'Row', y: 300 }), + selector: 'label=Row', + platform: 'ios', + }), + true, + ); + assert.equal( + await isSelectorVisibleInNodes({ + nodes: tree({ ref: 'e2', label: 'Row', y: -900 }, { ref: 'e3', label: 'Row', y: 3000 }), + selector: 'label=Row', + platform: 'ios', + }), + false, + ); +}); diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts new file mode 100644 index 0000000000..171ee0a7f9 --- /dev/null +++ b/packages/selectors/src/scroll-until-match.ts @@ -0,0 +1,50 @@ +import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; +import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { resolveSelectorPipeline } from './selector-pipeline.ts'; +import { SELECTOR_PIPELINE_POLICIES } from './selector-pipeline-policy.ts'; + +/** + * The stop condition `scroll --until` asks of every capture: does this selector match a node that + * is on screen right now? + * + * One definition for both callers — the daemon's generic scroll route and the in-process command + * runtime — so the two paths cannot disagree about when a scroll has arrived. It is deliberately + * two questions, not one: the `wait` pipeline row answers presence and ignores off-screen, then + * `isVisibleOnScreen` answers the part `--until` actually cares about. Reusing the presence row + * unchanged is what keeps a target that is present-but-scrolled-out from ending the loop early. + */ +export async function isSelectorVisibleInNodes(params: { + nodes: readonly (RawSnapshotNode | SnapshotNode)[]; + selector: string; + platform: Platform | PublicPlatform; +}): Promise { + const nodes = params.nodes as SnapshotNode[]; + if (nodes.length === 0) return false; + const outcome = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.wait, + nodes, + params.selector, + { platform: params.platform }, + ); + const matched = matchedNodes(outcome); + if (matched.length === 0) return false; + const visibility = createSnapshotVisibility(nodes); + // SOME match, not the first: a list whose rows share a selector can hold an off-screen twin above + // the fold, and stopping on that twin would leave the target the agent asked for still hidden. + return matched.some((node) => visibility.isVisibleOnScreen(node)); +} + +function matchedNodes( + outcome: Awaited>, +): readonly SnapshotNode[] { + switch (outcome.kind) { + case 'target': + case 'ambiguous': + return outcome.matchedNodes; + case 'occluded': + return [outcome.node]; + case 'none': + return []; + } +} diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 611a7c134d..7c1d3f858b 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -160,6 +160,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['holdMs', 'press hold duration'], ['jitterPx', 'press jitter'], ['pixels', 'scroll distance'], + ['until', 'scroll-until-visible stop condition'], ['doubleTap', 'double tap gesture'], ['clickButton', 'desktop mouse button selection', ['button']], ['backMode', 'explicit app/system back behavior', ['mode']], diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index faa8be5417..3f7f744ed6 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -418,6 +418,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/screenshot-diff-pixels', '@agent-device/capture-kit/screenshot-overlay', '@agent-device/capture-kit/scroll-edge-state', + '@agent-device/capture-kit/scroll-until-visible', '@agent-device/capture-kit/snapshot-chrome', '@agent-device/capture-kit/snapshot-desktop-projection', '@agent-device/capture-kit/snapshot-desktop-surface', @@ -652,6 +653,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/selectors/interaction-touch-point', '@agent-device/selectors/parameterized-recorded-fill', '@agent-device/selectors/press-retarget', + '@agent-device/selectors/scroll-until-match', '@agent-device/selectors/selector-pipeline', '@agent-device/selectors/selector-pipeline-policy', '@agent-device/selectors/snapshot-geometry-fixtures', diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index e8aafb0a48..a2447b3ed5 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -15,6 +15,8 @@ That starts the session and returns the initial interactive snapshot with `@refs Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. +Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until ` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something. + Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index c585131e01..09b44a5b24 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -148,10 +148,10 @@ Bootstrap: Snapshots and refs: snapshot reads visible state; snapshot -i gets current interactive refs only -- fast path before interaction. Default text is token-efficient; --raw/--json for full provider tree. - Legend: @e12 [button] label="Add to cart" enabled hittable -> press @e12. [off-screen below] -> scroll down (a hint, not a ref). + Legend: @e12 [button] label="Add to cart" enabled hittable -> press @e12. [off-screen below] -> scroll down --until (a hint, not a ref). Refs stay valid until you press/click/fill/type/scroll/back/wait-for-async-UI, or otherwise change app state; open/--relaunch clears the stored snapshot outright. Prefer --settle and its diff when it shows next target; refresh with snapshot -i only when you did not settle, it reported not settled, or output lacks what you need. A known selector/label after a mutation is often enough, since interaction commands refresh state internally. - Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing list target: scroll down/up then snapshot -i. TV/D-pad focus: help tv. + Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing target: scroll --until . TV/D-pad focus: help tv. Selectors: id="field-email", label="Allow", role=button label="Search" -- not bare role keys (button="Search"); no CSS selectors/--selector/--text/raw x-y when refs/selectors exist. @@ -252,10 +252,12 @@ Shapes: agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong agent-device gesture pan 200 420 0 -80 500 agent-device gesture pan 200 420 80 -40 700 --pointer-count 2 + agent-device scroll down --until 'id=submit' agent-device gesture fling right 200 420 180 agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 + scroll --until repeats scroll-and-check passes until that element is on screen, then stops -- one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out before the target appears, and is not accepted on the top/bottom directions, which already carry their own stop condition. One gesture never travels more than 0.8 of the viewport axis, so crossing several screens is what --until and scroll top/bottom are for. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. @@ -990,7 +992,7 @@ Rules: Findings must come from observed runtime behavior, not source reads. After each mutation, use the --settle diff as evidence when available; otherwise re-snapshot. Wait timeouts are integer milliseconds in the trailing positional: agent-device wait 'role=tab' 10000. Do not write duration suffixes such as 10s. - scroll takes a selector-less direction+amount form: agent-device scroll down 3. Use --settle to wait for the UI to go quiet and get the settled diff. + scroll takes a selector-less direction+amount form: agent-device scroll down 0.8. One gesture cannot travel further than 0.8 of the viewport axis, so a larger amount saturates rather than covering more ground; to cross several screens use agent-device scroll down --until or scroll bottom. Use --settle to wait for the UI to go quiet and get the settled diff. Keep commands in the report reproducible; use selectors or refs from fresh snapshots, not guessed coordinates. Prefer refs for exploration and selectors for deterministic replay. Use logs, network, screenshot --overlay-refs, trace, perf frames, perf memory, native profiles, or react-devtools only when they add evidence to a specific issue. diff --git a/src/commands/cli-grammar/flag-definitions-action.ts b/src/commands/cli-grammar/flag-definitions-action.ts index 15be134206..e11ff6b38e 100644 --- a/src/commands/cli-grammar/flag-definitions-action.ts +++ b/src/commands/cli-grammar/flag-definitions-action.ts @@ -115,6 +115,13 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--pixels ', usageDescription: 'Scroll: explicit gesture distance in pixels', }, + { + key: 'until', + names: ['--until'], + type: 'string', + usageLabel: '--until ', + usageDescription: 'Scroll: repeat passes until the selector is visible on screen', + }, { key: 'doubleTap', names: ['--double-tap'], diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index 1cc8c6c53c..f2c8dc5d91 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -89,6 +89,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { holdMs: options.holdMs, jitterPx: options.jitterPx, pixels: options.pixels, + until: options.until, doubleTap: options.doubleTap, verify: options.verify, settle: options.settle, diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index e0f21e93fb..ec3f622ed6 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -145,9 +145,9 @@ const interactionCliSchemas = { }, scroll: { usageOverride: 'scroll [amount]', - usageFlags: ['pixels', 'durationMs', 'settle'], + usageFlags: ['until', 'pixels', 'durationMs', 'settle'], positionalArgs: ['directionOrEdge', 'amount?'], - allowedFlags: ['pixels', 'durationMs', ...postActionObservationCliFlags('scroll')], + allowedFlags: ['pixels', 'durationMs', 'until', ...postActionObservationCliFlags('scroll')], }, } as const satisfies Record; diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 3144d4633f..9839b5ce87 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -101,6 +101,7 @@ export const interactionCliReaders = { amount: optionalCliNumber(positionals[1]), pixels: flags.pixels, durationMs: flags.durationMs, + until: flags.until, }), // The one observation-only reader in this file: `get` can be excluded from a // repair-armed heal by default, so it also takes the `--record` opt-in diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 5774ac1fab..7592733bba 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to keep scrolling until that element is on screen, which finds an off-screen target in one command instead of a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', @@ -143,6 +143,9 @@ const scrollFields = { direction: requiredField(enumField(SCROLL_INPUT_DIRECTIONS)), amount: numberField('Platform scroll amount.'), pixels: integerField('Pixel scroll amount.', { min: 0 }), + until: stringField( + 'Repeat scroll passes until this selector is visible on screen, then stop. Not valid with the top/bottom edge directions, which carry their own stop condition.', + ), durationMs: integerField('Scroll duration in milliseconds when the backend supports pacing.', { min: 0, max: SCROLL_DURATION_MAX_MS, diff --git a/src/commands/interaction/runtime/resolution.test.ts b/src/commands/interaction/runtime/resolution.test.ts index 4d599252c6..6c08c2e138 100644 --- a/src/commands/interaction/runtime/resolution.test.ts +++ b/src/commands/interaction/runtime/resolution.test.ts @@ -187,10 +187,10 @@ test('runtime press names a direction for a partial clip whose center is off-scr assert.equal(details?.reason, 'offscreen_selector'); assert.equal(details?.scrollDirection, 'down'); assert.match(String(details?.hint), /scroll down/i); - // #1366 recovery must be bounded: a single large (fling) scroll overshoots, - // so the hint steers to small steps / a bounded gesture pan. - assert.match(String(details?.hint), /small steps/i); - assert.match(String(details?.hint), /gesture pan/i); + // #1366 recovery must be bounded. `--until` is what bounds it now: it checks the same + // selector between passes, so the hint names one command rather than a manual step loop. + assert.match(String(details?.hint), /scroll down --until 'label=Cash'/); + assert.match(String(details?.hint), /stops on the target/i); return true; }, ); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index b52a0452d8..1230ad1621 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -881,11 +881,11 @@ async function assertVisibleSelectorTarget( // A selector re-resolves against a fresh snapshot on every attempt, so the // recovery is: move the named direction, then retry THIS selector — no // separate snapshot step, and no @ref (a scroll expires the ref frame, - // #1366). Naming the direction stops the wrong-way / retry-the-same-ref loop; - // bounded steps stop the overshoot loop — a single large scroll (fling - // momentum on iOS) can sail past the target, so a short gesture pan lands it. + // #1366). `--until` is that whole loop as one command: it checks the same + // selector between passes, which is also what keeps a large step from + // overshooting, so the hint no longer has to trade distance for accuracy. hint: (direction) => - `${scrollRevealClause(direction)} in small steps, retrying ${action} with the same selector after each (it re-resolves against a fresh snapshot). A single large scroll can overshoot the target; a short bounded gesture pan lands it more reliably. If it is inside a closed drawer or another tab, open that container first.`, + `${scrollRevealClause(direction, selector)} then retry ${action} with the same selector. --until checks the selector between passes, so it stops on the target rather than sailing past it. If it is inside a closed drawer or another tab, open that container first.`, }); } @@ -902,19 +902,29 @@ async function assertVisibleRefTarget( details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) }, // The scroll that reveals the target expires the ref frame (#1366, ADR // 0014), so retrying this @ref would be rejected next. Steer to a selector, - // which re-resolves against a fresh snapshot and bypasses the ref-frame guard. + // which re-resolves against a fresh snapshot and bypasses the ref-frame guard + // — and which `--until` can then check between passes. hint: (direction) => - `${scrollRevealClause(direction)} in small steps (a single large scroll can overshoot; a short bounded gesture pan lands it more reliably), then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, + `${scrollRevealClause(direction, null)} then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, }); } -// Shared lead-in for both off-screen hints. Names the concrete `scroll ` -// when the geometry gives one, and falls back to the generic phrasing when the -// target is off more than one edge in a way that has no single reveal. Callers -// append the bounded-steps guidance: a single large scroll (fling momentum on -// iOS) can sail past the target, so small bounded moves are what actually land. -function scrollRevealClause(direction: OffscreenScrollDirection | null): string { - return direction ? `Scroll ${direction} toward it` : 'Scroll toward it'; +/** + * Shared lead-in for both off-screen hints: the one command that reveals the target. + * + * When the geometry names a direction AND the caller has a selector to check, this is a complete + * `scroll --until ` — one request that stops on the target instead of the + * scroll-then-look-again loop the hint used to prescribe. Without a selector to check (an @ref + * refusal) or without a single reveal direction (off more than one edge), it degrades to naming + * the move and leaves the stop condition to the caller's own next step. + */ +function scrollRevealClause( + direction: OffscreenScrollDirection | null, + selector: string | null, +): string { + if (!direction) return 'Scroll toward it,'; + if (!selector) return `Scroll ${direction} toward it,`; + return `Run scroll ${direction} --until '${selector}' to bring it on screen,`; } /** diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts index fefd017928..373372ce7f 100644 --- a/src/commands/interaction/runtime/scroll.test.ts +++ b/src/commands/interaction/runtime/scroll.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { selector } from './selector-read-utils.ts'; import { AppError } from '@agent-device/kernel/errors'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { createInteractionDevice, runtimeScrollSnapshot, @@ -236,3 +237,98 @@ test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => ); } }); + +/** A viewport-height tree whose target row sits at `targetY`, used to walk a target into view. */ +function untilSnapshot(targetY: number, hiddenBelow: boolean) { + return makeSnapshotState([ + { + index: 1, + depth: 0, + type: 'ScrollView', + label: 'Form', + hiddenContentBelow: hiddenBelow ? true : undefined, + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 2, + depth: 1, + parentIndex: 1, + type: 'TextField', + label: 'Email', + rect: { x: 0, y: targetY, width: 400, height: 40 }, + hittable: true, + }, + ]); +} + +test('runtime scroll --until stops the pass loop as soon as the selector is on screen', async () => { + const scrolls: unknown[] = []; + const frames = [untilSnapshot(2400, true), untilSnapshot(1200, true), untilSnapshot(300, true)]; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ + snapshot: frames[Math.min(scrolls.length, frames.length - 1)], + }), + scroll: async (_context, target, options) => { + scrolls.push({ target, options }); + return { pixels: 480 }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'down', + until: 'label=Email', + }); + + assert.equal(result.until, 'label=Email'); + assert.equal(result.passes, 2); + assert.equal(scrolls.length, 2); + assert.match(String(result.message), /Scrolled down 2 passes until label=Email was visible/); +}); + +test('runtime scroll --until performs no gesture when the target is already on screen', async () => { + const scrolls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ snapshot: untilSnapshot(300, true) }), + scroll: async () => { + scrolls.push('scrolled'); + return {}; + }, + }); + + const result = await device.interactions.scroll({ direction: 'down', until: 'label=Email' }); + + assert.equal(result.passes, 0); + assert.equal(scrolls.length, 0); + assert.match(String(result.message), /already visible/); +}); + +test('runtime scroll --until fails with the end-of-content reason when the list runs out', async () => { + // Nothing below the fold and nothing hidden: the same signal `scroll bottom` stops on. + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ snapshot: untilSnapshot(300, false) }), + scroll: async () => ({}), + }); + + await assert.rejects( + () => device.interactions.scroll({ direction: 'down', until: 'label=Missing' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_edge_reached'); + return true; + }, + ); +}); + +test('runtime scroll --until is refused on the edge directions, which already carry a stop condition', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ snapshot: untilSnapshot(300, true) }), + scroll: async () => { + throw new Error('scroll should be rejected before any backend call'); + }, + }); + + await assert.rejects( + () => device.interactions.scroll({ direction: 'bottom', until: 'label=Email' }), + /scroll bottom already scrolls to the bottom edge and cannot take --until/, + ); +}); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts index 76241bb664..bcf800ec65 100644 --- a/src/commands/interaction/runtime/scroll.ts +++ b/src/commands/interaction/runtime/scroll.ts @@ -1,5 +1,6 @@ import { assertExclusiveScrollDistanceInputs, + assertScrollUntilCompatible, honoredScrollDurationMs, normalizeScrollDurationMs, resolveScrollExecutionOptions, @@ -13,6 +14,12 @@ import { type ScrollEdgeState, type ScrollEdgeTarget, } from '@agent-device/capture-kit/scroll-edge-state'; +import { + formatScrollUntilMessage, + runScrollUntilVisiblePasses, + scrollUntilNotFoundError, +} from '@agent-device/capture-kit/scroll-until-visible'; +import { isSelectorVisibleInNodes } from '@agent-device/selectors/scroll-until-match'; import { AppError } from '@agent-device/kernel/errors'; import { successText } from '@agent-device/kernel/success-text'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; @@ -48,6 +55,8 @@ export type ScrollCommandOptions = CommandContext & { amount?: number; pixels?: number; durationMs?: number; + /** Repeat passes until this selector is visible on screen, then stop. */ + until?: string; }; export type ScrollCommandResult = @@ -55,6 +64,7 @@ export type ScrollCommandResult = kind: 'viewport'; direction: GestureDirection; edge?: 'top' | 'bottom'; + until?: string; passes?: number; amount?: number; pixels?: number; @@ -64,6 +74,7 @@ export type ScrollCommandResult = ResolvedInteractionTarget & { direction: GestureDirection; edge?: 'top' | 'bottom'; + until?: string; passes?: number; amount?: number; pixels?: number; @@ -81,6 +92,52 @@ export const scrollCommand: RuntimeCommand; +}; + +/** Every distance/timing rejection, in one place, before any target resolution or device work. */ +function normalizeScrollDistance( + options: ScrollCommandOptions, + edge: ScrollEdge | undefined, +): NormalizedScrollDistance { + assertScrollUntilCompatible({ + ...(edge ? { edge } : {}), + ...(options.until === undefined ? {} : { until: options.until }), + }); const amount = normalizeOptionalPositiveNumber(options.amount, 'scroll amount'); const pixels = normalizeOptionalPositiveInteger(options.pixels, 'scroll pixels'); const durationMs = normalizeScrollDurationMs(options.durationMs); @@ -88,58 +145,88 @@ export const scrollCommand: RuntimeCommand, +): () => Promise>>> { + const scrollBackend = runtime.backend.scroll; + if (!scrollBackend) { + throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); + } const backendTarget = resolved.kind === 'viewport' ? { kind: 'viewport' as const } : { kind: 'point' as const, point: requireResolvedPoint(resolved) }; - const scrollBackend = runtime.backend.scroll; - const executionOptions = resolveScrollExecutionOptions( - { - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - ...(durationMs !== undefined ? { durationMs } : {}), - }, - target.edge, - ); - const runScroll = async () => + return async () => await scrollBackend(toBackendContext(runtime, options), backendTarget, { - direction: target.direction, - ...executionOptions, + direction, + ...execution, }); - let backendResult: Awaited>> | undefined; - let completedPasses = 0; - if (target.edge) { - const edge = target.edge; - const edgeTarget = buildScrollEdgeTarget(resolved); - const edgeResult = await runScrollEdgePasses({ - edge, - captureState: async (scope) => - await captureRuntimeScrollEdgeState(runtime, options, edge, edgeTarget, scope), - scroll: runScroll, - }); - backendResult = edgeResult.result; - completedPasses = edgeResult.passes; - } else { - backendResult = await runScroll(); - completedPasses = 1; - } - const formattedBackendResult = toBackendResult(backendResult); - const reportedDurationMs = honoredScrollDurationMs(formattedBackendResult); +} + +/** `scroll ` and `scroll top|bottom`: one pass, or passes until the edge stops moving. */ +async function runDirectionOrEdgeScroll(params: { + runtime: AgentDeviceRuntime; + options: ScrollCommandOptions; + resolved: ResolvedScrollTarget; + target: { direction: GestureDirection; edge?: ScrollEdge }; + distance: NormalizedScrollDistance; + scroll: () => Promise>>>; +}): Promise { + const { runtime, options, resolved, target, distance } = params; + const edge = target.edge; + const pass = edge + ? await runScrollEdgePasses({ + edge, + captureState: async (scope) => + await captureRuntimeScrollEdgeState( + runtime, + options, + edge, + buildScrollEdgeTarget(resolved), + scope, + ), + scroll: params.scroll, + }) + : { passes: 1, result: await params.scroll() }; + const backendResult = toBackendResult(pass.result); + const reportedDurationMs = honoredScrollDurationMs(backendResult); return { ...resolved, direction: target.direction, - ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), + ...(edge ? { edge, passes: pass.passes } : {}), + ...distance.reported, ...(reportedDurationMs !== undefined ? { durationMs: reportedDurationMs } : {}), - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...(backendResult ? { backendResult } : {}), ...successText( - formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), + formatScrollEdgeMessage( + target.direction, + edge, + pass.passes, + distance.reported.amount, + distance.reported.pixels, + honoredScrollPixels(backendResult), + ), ), }; -}; +} async function resolveScrollTarget( runtime: AgentDeviceRuntime, @@ -160,7 +247,6 @@ async function resolveScrollTarget( }, ); } - function resolveScrollDirection(direction: ScrollInputDirection): { direction: GestureDirection; edge?: 'top' | 'bottom'; @@ -169,7 +255,6 @@ function resolveScrollDirection(direction: ScrollInputDirection): { if (direction === 'top') return { direction: 'up', edge: 'top' }; return { direction: requireDirection(direction, 'scroll direction') }; } - function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget { return resolved.kind === 'viewport' ? {} @@ -178,7 +263,6 @@ function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget nodeIndex: 'node' in resolved ? resolved.node?.index : undefined, }; } - async function captureRuntimeScrollEdgeState( runtime: AgentDeviceRuntime, options: ScrollCommandOptions, @@ -206,6 +290,86 @@ async function captureRuntimeScrollEdgeState( }); } +/** + * `scroll --until `: repeat the pass until the selector is on screen. + * + * A sibling of the edge branch rather than a variant of the one-pass branch — it owns a different + * stop condition, a different failure vocabulary, and a result that names the selector it stopped + * on, none of which the ordinary scroll result carries. + */ +async function runUntilScroll(params: { + runtime: AgentDeviceRuntime; + options: ScrollCommandOptions; + resolved: ResolvedScrollTarget; + direction: GestureDirection; + until: string; + distance: { amount?: number; pixels?: number }; + scroll: () => Promise>>>; +}): Promise { + const { runtime, options, resolved, direction, until, distance } = params; + const edge = verticalEdgeFor(direction); + const result = await runScrollUntilVisiblePasses({ + ...(edge === undefined ? {} : { edge }), + captureNodes: async () => await captureRuntimeScrollNodes(runtime, options), + isVisibleMatch: async (nodes) => + await isSelectorVisibleInNodes({ + nodes, + selector: until, + platform: runtime.backend.platform, + }), + scroll: params.scroll, + }); + if (result.outcome !== 'matched') { + throw scrollUntilNotFoundError({ + direction, + selector: until, + outcome: result.outcome, + passes: result.passes, + }); + } + const backendResult = toBackendResult(result.result); + return { + ...resolved, + direction, + until, + passes: result.passes, + ...distance, + ...(backendResult ? { backendResult } : {}), + ...successText(formatScrollUntilMessage(direction, until, result.passes)), + }; +} + +/** The travel the planner produced, which saturates below a large requested amount. */ +function honoredScrollPixels(result: Record | undefined): number | undefined { + return typeof result?.pixels === 'number' ? result.pixels : undefined; +} + +/** + * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by + * its pass budget alone rather than by a signal that would always report "no room". + */ +function verticalEdgeFor(direction: GestureDirection): ScrollEdge | undefined { + if (direction === 'down') return 'bottom'; + if (direction === 'up') return 'top'; + return undefined; +} + +async function captureRuntimeScrollNodes( + runtime: AgentDeviceRuntime, + options: ScrollCommandOptions, +) { + if (!runtime.backend.captureSnapshot) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'scroll --until requires snapshot support to check whether the selector became visible', + ); + } + const result = await runtime.backend.captureSnapshot(toBackendContext(runtime, options), { + includeRects: true, + }); + return result.snapshot?.nodes ?? result.nodes ?? []; +} + function requireDirection( direction: GestureDirection | undefined, field: string, diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 34e6c420e4..0c143e4ccd 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -49,6 +49,7 @@ export const DISPATCH_CONTEXT_FLAG_KEYS = [ 'holdMs', 'jitterPx', 'pixels', + 'until', 'doubleTap', 'backMode', 'pauseMs', diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index 46cdc0fdce..90c72990d1 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -338,3 +338,92 @@ test('the edge plan proves its capture statically and the direction plan cannot expectTypeOf().toEqualTypeOf<'scrollDirection'>(); expectTypeOf>().toEqualTypeOf<'scrollDirection'>(); }); + +/** Same shape the command runtime's `--until` tests use: a row walked into the viewport. */ +function untilNodes(targetY: number, hiddenBelow: boolean) { + return [ + { + index: 1, + depth: 0, + type: 'ScrollView', + label: 'Form', + ...(hiddenBelow ? { hiddenContentBelow: true } : {}), + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 2, + depth: 1, + parentIndex: 1, + type: 'TextField', + label: 'Email', + rect: { x: 0, y: targetY, width: 400, height: 40 }, + }, + ]; +} + +test('bound scroll --until stops on the pass whose capture shows the selector on screen', async () => { + const scrolls: string[] = []; + const frames = [untilNodes(2400, true), untilNodes(1200, true), untilNodes(300, true)]; + const result = await runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ nodes: frames[Math.min(scrolls.length, frames.length - 1)] }), + scroll: async (direction) => { + scrolls.push(direction); + return { pixels: 480 }; + }, + }, + ); + + assert.equal(result.until, 'label=Email'); + assert.equal(result.passes, 2); + assert.equal(scrolls.length, 2); + assert.match(String(result.message), /Scrolled down 2 passes until label=Email was visible/); +}); + +test('bound scroll --until reports the end of the content rather than spending its budget', async () => { + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Missing' }, + { + captureSnapshot: async () => ({ nodes: untilNodes(300, false) }), + scroll: async () => ({}), + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_edge_reached'); + return true; + }, + ); +}); + +test('bound scroll rejects --until on an edge direction before any device work', async () => { + await assert.rejects( + () => + runScroll( + ['bottom'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ nodes: untilNodes(300, true) }), + scroll: async () => { + throw new Error('scroll should be rejected before the backend call'); + }, + }, + ), + /scroll bottom already scrolls to the bottom edge and cannot take --until/, + ); +}); + +test('bound scroll --until is refused when the owner advertises no capture', async () => { + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals: ['down'], + context: { until: 'label=Email' } as DaemonCommandContext, + ...bindings({ scroll: async () => ({}) }), + }); + assert.equal(resolved.ok, false); +}); diff --git a/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts index df107f34a4..c63642246f 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts @@ -261,13 +261,12 @@ test('press @ref fails fast when the target is off-screen', async () => { if (response && !response.ok) { expect(response.error.code).toBe('COMMAND_FAILED'); expect(response.error.message).toMatch(/off-screen/i); - // #1366: the hint names the concrete scroll direction, steers to a - // selector-based retry (a @ref would be rejected as expired after the scroll), - // and prescribes bounded movement (a large fling scroll overshoots). + // #1366: the hint names the concrete scroll direction and steers to a selector-based retry + // (a @ref would be rejected as expired after the scroll). A ref refusal has no selector to + // check between passes, so this hint names the move and leaves the stop condition to the + // selector retry it prescribes. expect(response.error.hint).toMatch(/scroll down/i); expect(response.error.hint).toMatch(/selector/i); - expect(response.error.hint).toMatch(/small steps/i); - expect(response.error.hint).toMatch(/gesture pan/i); expect(response.error.details?.reason).toBe('offscreen_ref'); expect(response.error.details?.scrollDirection).toBe('down'); } diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 39c4d66825..454f583ad9 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -1,5 +1,6 @@ import { assertExclusiveScrollDistanceInputs, + assertScrollUntilCompatible, honoredScrollDurationMs, normalizeScrollDurationMs, resolveScrollExecutionOptions, @@ -22,6 +23,13 @@ import { type ScrollEdge, type ScrollEdgeState, } from '@agent-device/capture-kit/scroll-edge-state'; +import { + formatScrollUntilMessage, + runScrollUntilVisiblePasses, + scrollUntilNotFoundError, +} from '@agent-device/capture-kit/scroll-until-visible'; +import { isSelectorVisibleInNodes } from '@agent-device/selectors/scroll-until-match'; +import { publicPlatformString } from '@agent-device/kernel/device'; import { withSuccessText } from '@agent-device/kernel/success-text'; import type { DaemonCommandContext } from './context.ts'; import { errorResponse } from './response.ts'; @@ -43,6 +51,7 @@ type BoundScrollDirection = BoundDeviceRuntime< Extract['use'] >; type BoundScrollEdge = BoundDeviceRuntime['use']>; +type BoundScrollUntil = BoundDeviceRuntime['use']>; /** `scroll bottom` scrolls down to the edge; `scroll top` scrolls up to it. */ function parseScrollTarget(input: string): ScrollTarget { @@ -82,12 +91,20 @@ export async function resolveBoundScrollRuntime( const amount = params.positionals[1] ? Number(params.positionals[1]) : undefined; const pixels = params.context.pixels; const durationMs = params.context.durationMs; + const until = params.context.until; if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); assertScrollCommandInputs(amount, pixels, durationMs); const target = parseScrollTarget(directionInput); + assertScrollUntilCompatible({ + ...(target.edge ? { edge: target.edge } : {}), + ...(until === undefined ? {} : { until }), + }); const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); - const plan = resolveScrollRuntimePlan(target.edge === undefined ? {} : { edge: target.edge }); + const plan = resolveScrollRuntimePlan({ + ...(target.edge === undefined ? {} : { edge: target.edge }), + ...(until === undefined ? {} : { until }), + }); const admission = { command: 'scroll', device: params.device, @@ -115,6 +132,25 @@ export async function resolveBoundScrollRuntime( await executeEdgeScroll(runtime, edge, target, options, dispatchContext), ); } + case 'until': { + const selector = plan.until; + return await resolveBoundGenericRuntime( + { + ...admission, + unavailableResponse: (unavailable) => scrollUntilUnsupported(unavailable.hint), + use: plan.use, + }, + async (runtime, dispatchContext) => + await executeUntilScroll( + runtime, + params.device, + selector, + target, + options, + dispatchContext, + ), + ); + } } } @@ -127,6 +163,15 @@ function scrollEdgeUnsupported(edge: ScrollEdge, hint: string | undefined) { ); } +function scrollUntilUnsupported(hint: string | undefined) { + return errorResponse( + 'UNSUPPORTED_OPERATION', + 'scroll --until requires snapshot support to check whether the selector became visible', + undefined, + hint === undefined ? undefined : { hint }, + ); +} + /** One pass. This binding carries no capture, so an edge-style read will not type-check here. */ async function executeDirectionScroll( runtime: BoundScrollDirection, @@ -158,6 +203,70 @@ async function executeEdgeScroll( return scrollResult(target, options, edgeResult.passes, edgeResult.result ?? {}); } +/** Repeats the pass until the selector is on screen, the content runs out, or the budget does. */ +async function executeUntilScroll( + runtime: BoundScrollUntil, + device: DeviceInfo, + selector: string, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): Promise> { + const untilResult = await runScrollUntilVisiblePasses({ + ...(verticalEdgeFor(target.direction) === undefined + ? {} + : { edge: verticalEdgeFor(target.direction) as ScrollEdge }), + captureNodes: async () => await captureUntilNodes(runtime, context), + isVisibleMatch: async (nodes) => + await isSelectorVisibleInNodes({ + nodes, + selector, + platform: publicPlatformString(device), + }), + scroll: async () => await scrollOnce(runtime, target, options, context), + }); + if (untilResult.outcome !== 'matched') { + throw scrollUntilNotFoundError({ + direction: target.direction, + selector, + outcome: untilResult.outcome, + passes: untilResult.passes, + }); + } + return withSuccessText( + { + direction: target.direction, + until: selector, + passes: untilResult.passes, + ...(options.amount !== undefined ? { amount: options.amount } : {}), + ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), + ...(untilResult.result ?? {}), + }, + formatScrollUntilMessage(target.direction, selector, untilResult.passes), + ); +} + +/** + * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by + * its pass budget alone rather than by a signal that would always report "no room". + */ +function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined { + if (direction === 'down') return 'bottom'; + if (direction === 'up') return 'top'; + return undefined; +} + +async function captureUntilNodes(runtime: BoundScrollUntil, context: DaemonCommandContext) { + return ( + ( + await runtime.operations.captureSnapshot({ + options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, + execution: runtimeExecutionFromContext(context), + }) + ).nodes ?? [] + ); +} + async function captureEdgeState( runtime: BoundScrollEdge, edge: ScrollEdge, @@ -213,10 +322,16 @@ function scrollResult( completedPasses, options.amount, options.pixels, + honoredScrollPixels(interactionResult), ), ); } +/** The travel the planner produced, which saturates below a large requested amount. */ +function honoredScrollPixels(result: Record): number | undefined { + return typeof result.pixels === 'number' ? result.pixels : undefined; +} + /** The neutral intent one scroll carries, projected from a resolved command context. */ function scrollInput( direction: ScrollDirection, diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 2f1561cace..a32b85bfd0 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -531,7 +531,8 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { { direction: enumSchema(['up', 'down', 'left', 'right']), edge: enumSchema(['top', 'bottom']), - passes: numberSchema('Edge scrolls only: how many scroll-and-check passes ran.'), + until: stringSchema('Until scrolls only: the selector the passes stopped on.'), + passes: numberSchema('Edge and until scrolls only: how many scroll-and-check passes ran.'), amount: numberSchema(), pixels: numberSchema(), durationMs: numberSchema(), diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index de84ae789c..e94dbd6050 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -19,7 +19,7 @@ export const MCP_SERVER_INSTRUCTIONS = `agent-device drives iOS, Android, tvOS, Start: known app -> call open {app, foreground: true} at once; do not probe with devices, apps, appstate, snapshot, or screenshot first. open returns the initial interactive snapshot with @refs. Unknown app id: devices, then apps, then open the discovered id; never invent ids. Existing session: continue from its state, do not reopen. -Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. +Loop: press/click/fill/longpress/hover/scroll/back with settle: true; the response is the settled UI diff, continue from it. To reach an off-screen target, scroll {direction, until: ""} scrolls until it is on screen in one call, and direction "bottom" runs to the end of the content; repeated bare scrolls are the slow way to find something. snapshot {interactiveOnly: true} only when the diff lacks the next target or did not settle. Verify with wait {kind: "text", text}, wait {selector}, wait {absent: selector}, is, get, or find; a bare screenshot is not verification. End with close. Targets: copy refs byte-for-byte (@e12, @e12~s4; keep @ and any ~sN). Refs go stale after mutations. Prefer refs, then id/label/role selectors; coordinates last. On a sparse/AX-unavailable warning its refs and selectors are invalid: screenshot, read the image, press {x, y}, then snapshot the changed screen. From 9bb08d0ac32a29c928e4aa394a4f6a659bfad52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 9 Sep 2026 22:21:31 +0200 Subject: [PATCH 03/13] test(scroll): cover --until through the provider-backed integration path --- .../provider-scenarios/scroll-until.test.ts | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 test/integration/provider-scenarios/scroll-until.test.ts diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts new file mode 100644 index 0000000000..6a1b818649 --- /dev/null +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -0,0 +1,119 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_UNTIL_PASS_LIMIT } from '@agent-device/capture-kit/scroll-until-visible'; +import { createAndroidSettingsWorld } from './android-world.ts'; +import { withProviderScenarioResource } from './harness.ts'; + +/** + * `scroll --until ` through the real daemon, provider admission, and capture path. + * + * The world serves a hierarchy whose target row starts below the viewport and climbs on each + * capture, which is what lets the loop's stop condition be observed rather than asserted: the + * command is expected to stop on the first capture that puts the row on screen, and to have spent + * exactly the gestures that took to reach it. + */ +/** + * The row climbs one screen per capture, so it is off-screen for the first captures and on screen + * from the third. Keyed on captures rather than on injected gestures because the Android gesture + * path runs through the persistent helper, not an adb shell command the world can count. + */ +function climbingRow(): () => number { + let captures = 0; + return () => { + const top = Math.max(200, 1400 - captures * 600); + captures += 1; + return top; + }; +} + +function climbingHierarchy(targetTop: () => number): () => string { + return () => + [ + '', + '', + ' ', + ' ', + ` `, + ' ', + '', + ].join('\n'); +} + +test('Provider-backed integration scroll --until stops on the capture that brings the target on screen', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: climbingHierarchy(climbingRow()) }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + const result = await client.interactions.scroll({ + direction: 'down', + until: 'text=Terms', + ...world.selection, + }); + + const passes = typeof result.passes === 'number' ? result.passes : -1; + assert.equal(result.until, 'text=Terms'); + assert.equal(result.direction, 'down'); + assert.ok( + passes >= 1, + `expected at least one pass to reach the off-screen row, saw ${passes}`, + ); + assert.match(String(result.message), /until text=Terms was visible/); + // Stopped on arrival rather than running the budget out. + assert.ok( + passes < SCROLL_UNTIL_PASS_LIMIT, + `expected the loop to stop on arrival, spent ${passes} passes`, + ); + }, + ); +}); + +test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => { + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + // Nothing below the fold and nothing hidden: the content cannot move further. + snapshotXml: climbingHierarchy(() => 200), + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + await assert.rejects( + () => + client.interactions.scroll({ + direction: 'down', + until: 'text=NeverPresent', + ...world.selection, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.match(String(error.message), /without text=NeverPresent becoming visible/); + return true; + }, + ); + }, + ); +}); + +test('Provider-backed integration scroll rejects --until on the edge directions', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: climbingHierarchy(() => 200) }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + await assert.rejects( + () => + client.interactions.scroll({ + direction: 'bottom', + until: 'text=Terms', + ...world.selection, + }), + /cannot take --until/, + ); + }, + ); +}); From 7eaa4635918978bc7abf8a3b422e6b907983d908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 08:59:31 +0200 Subject: [PATCH 04/13] perf(selectors): keep the scroll-until predicate off the eager import path --- .../src/snapshot/scroll-until-visible.ts | 2 +- packages/selectors/src/scroll-until-match.ts | 15 ++++++++++----- .../provider-scenarios/scroll-until.test.ts | 11 +++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-until-visible.ts b/packages/capture-kit/src/snapshot/scroll-until-visible.ts index b29579182b..d6cf466f6e 100644 --- a/packages/capture-kit/src/snapshot/scroll-until-visible.ts +++ b/packages/capture-kit/src/snapshot/scroll-until-visible.ts @@ -103,7 +103,7 @@ export function scrollUntilNotFoundError(params: { { reason: 'scroll_until_edge_reached', details: { selector, direction, passes }, - hint: `Nothing further lies ${direction} of here. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, + hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, }, ); } diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts index 171ee0a7f9..116e98796f 100644 --- a/packages/selectors/src/scroll-until-match.ts +++ b/packages/selectors/src/scroll-until-match.ts @@ -1,8 +1,7 @@ import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; -import { resolveSelectorPipeline } from './selector-pipeline.ts'; -import { SELECTOR_PIPELINE_POLICIES } from './selector-pipeline-policy.ts'; +import type { SelectorPipelineOutcome } from './selector-pipeline.ts'; /** * The stop condition `scroll --until` asks of every capture: does this selector match a node that @@ -21,6 +20,14 @@ export async function isSelectorVisibleInNodes(params: { }): Promise { const nodes = params.nodes as SnapshotNode[]; if (nodes.length === 0) return false; + // Both edges are lazy on purpose. The policy table re-enters the package barrel and the pipeline + // pulls the match engine, which together would make this small predicate a 66-module entry + // surface for every importer. The loop that calls this awaits anyway, and the module cache makes + // every pass after the first free. + const [{ SELECTOR_PIPELINE_POLICIES }, { resolveSelectorPipeline }] = await Promise.all([ + import('./selector-pipeline-policy.ts'), + import('./selector-pipeline.ts'), + ]); const outcome = await resolveSelectorPipeline( SELECTOR_PIPELINE_POLICIES.wait, nodes, @@ -35,9 +42,7 @@ export async function isSelectorVisibleInNodes(params: { return matched.some((node) => visibility.isVisibleOnScreen(node)); } -function matchedNodes( - outcome: Awaited>, -): readonly SnapshotNode[] { +function matchedNodes(outcome: SelectorPipelineOutcome): readonly SnapshotNode[] { switch (outcome.kind) { case 'target': case 'ambiguous': diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index 6a1b818649..32c32e0570 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -8,15 +8,10 @@ import { withProviderScenarioResource } from './harness.ts'; /** * `scroll --until ` through the real daemon, provider admission, and capture path. * - * The world serves a hierarchy whose target row starts below the viewport and climbs on each - * capture, which is what lets the loop's stop condition be observed rather than asserted: the - * command is expected to stop on the first capture that puts the row on screen, and to have spent - * exactly the gestures that took to reach it. - */ -/** * The row climbs one screen per capture, so it is off-screen for the first captures and on screen - * from the third. Keyed on captures rather than on injected gestures because the Android gesture - * path runs through the persistent helper, not an adb shell command the world can count. + * from the third: the loop's stop condition is observed rather than asserted. Keyed on captures + * rather than on injected gestures because the Android gesture path runs through the persistent + * helper, not an adb shell command the world can count. */ function climbingRow(): () => number { let captures = 0; From 9793bb55e45d263a97ebce919c264897f874295d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 10:26:34 +0200 Subject: [PATCH 05/13] fix(scroll): refuse an unreadable capture instead of reporting end-of-content --- .../src/snapshot/scroll-until-visible.ts | 49 ++++++++++++++- .../selectors/src/scroll-until-match.test.ts | 47 ++++++++++++++- packages/selectors/src/scroll-until-match.ts | 60 ++++++++++++++++++- .../interaction/runtime/scroll.test.ts | 43 +++++++++++++ src/commands/interaction/runtime/scroll.ts | 19 +++++- src/daemon/__tests__/scroll-runtime.test.ts | 43 +++++++++++++ src/daemon/scroll-runtime.ts | 34 +++++++---- 7 files changed, 277 insertions(+), 18 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-until-visible.ts b/packages/capture-kit/src/snapshot/scroll-until-visible.ts index d6cf466f6e..730f92becb 100644 --- a/packages/capture-kit/src/snapshot/scroll-until-visible.ts +++ b/packages/capture-kit/src/snapshot/scroll-until-visible.ts @@ -19,6 +19,20 @@ export const SCROLL_UNTIL_PASS_LIMIT = 12; */ export type ScrollUntilVisibleOutcome = 'matched' | 'edge-reached' | 'pass-limit'; +/** + * Why a capture cannot answer the `--until` question at all. + * + * Distinct from the loop's outcomes on purpose: an unreadable capture is not evidence about the + * content, and collapsing the two is how `?? []` used to turn a failed read into "you reached the + * end of the list". The classifier that produces this lives in `@agent-device/selectors`, which is + * where the same readability question is already answered for absence assertions; the vocabulary + * lives here beside the outcomes it must not be confused with. + */ +export type ScrollUntilCaptureRefusal = { + reason: 'no-capture' | 'sparse-tree'; + detail: string; +}; + export type ScrollUntilVisibleResult = { passes: number; outcome: ScrollUntilVisibleOutcome; @@ -102,7 +116,9 @@ export function scrollUntilNotFoundError(params: { `scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`, { reason: 'scroll_until_edge_reached', - details: { selector, direction, passes }, + selector, + direction, + passes, hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, }, ); @@ -112,8 +128,37 @@ export function scrollUntilNotFoundError(params: { `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, { reason: 'scroll_until_pass_limit', - details: { selector, direction, passes }, + selector, + direction, + passes, hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, }, ); } + +/** + * The capture could not be read, so neither the selector match nor the edge analyzer ran. Reported + * as its own failure rather than as an outcome, because "we could not see the screen" and "the + * content ran out" call for different next steps. + */ +export function scrollUntilCaptureError(params: { + direction: ScrollDirection; + selector: string; + refusal: ScrollUntilCaptureRefusal; +}): AppError { + const { direction, selector, refusal } = params; + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} --until ${selector} could not read the screen: ${refusal.detail}`, + { + reason: 'scroll_until_capture_unreadable', + selector, + direction, + captureRefusal: refusal.reason, + hint: + refusal.reason === 'no-capture' + ? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.' + : 'The accessibility tree came back sparse, so its refs and selectors are not trustworthy. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.', + }, + ); +} diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts index 94b50aea05..0dbe13bf94 100644 --- a/packages/selectors/src/scroll-until-match.test.ts +++ b/packages/selectors/src/scroll-until-match.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import { isSelectorVisibleInNodes } from './scroll-until-match.ts'; +import { isSelectorVisibleInNodes, scrollUntilCaptureRefusal } from './scroll-until-match.ts'; const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; @@ -85,3 +85,48 @@ test('an off-screen twin does not satisfy a selector whose other match is on scr false, ); }); + +test('a capture with no tree at all is refused rather than read as an empty screen', () => { + assert.deepEqual(scrollUntilCaptureRefusal({}), { + reason: 'no-capture', + detail: 'the capture returned no accessibility tree', + }); + assert.deepEqual(scrollUntilCaptureRefusal({ nodes: [] }), { + reason: 'no-capture', + detail: 'the capture returned an empty accessibility tree', + }); +}); + +test('a backend sparse verdict is refused and carries the backend reason', () => { + assert.deepEqual( + scrollUntilCaptureRefusal({ + nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), + snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }), + { reason: 'sparse-tree', detail: 'AX bridge unavailable' }, + ); +}); + +test('the legacy iOS application-root-only shape is refused', () => { + assert.deepEqual( + scrollUntilCaptureRefusal({ + backend: 'xctest', + nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], + }), + { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }, + ); +}); + +/** + * Truncation is a readable tree missing its tail, not a failed read. Refusing it would fail large + * screens where the target is plainly in view. + */ +test('a truncated but populated capture is not refused', () => { + assert.equal( + scrollUntilCaptureRefusal({ + nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), + snapshotQuality: { state: 'ok', backend: 'tree' }, + }), + undefined, + ); +}); diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts index 116e98796f..48a09ff1c2 100644 --- a/packages/selectors/src/scroll-until-match.ts +++ b/packages/selectors/src/scroll-until-match.ts @@ -1,6 +1,13 @@ import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; +import type { ScrollUntilCaptureRefusal } from '@agent-device/capture-kit/scroll-until-visible'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { + RawSnapshotNode, + SnapshotNode, + SnapshotQualityVerdict, + SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { isLegacySparseIosInteractiveSnapshot } from './absence-observation.ts'; import type { SelectorPipelineOutcome } from './selector-pipeline.ts'; /** @@ -53,3 +60,54 @@ function matchedNodes(outcome: SelectorPipelineOutcome): readonly SnapshotNode[] return []; } } + +/** The fields a `--until` pass can read from either route's capture without reshaping it. */ +export type ScrollUntilCapture = { + nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; + /** Widened to `string` because the backend capture result carries it untyped. */ + backend?: string | undefined; + snapshotQuality?: SnapshotQualityVerdict | undefined; +}; + +/** + * Whether this capture can answer the `--until` question, asked before the selector match and + * before the edge analyzer. + * + * Both callers previously coerced a missing tree to `[]`, which the vertical edge analyzer reads as + * "no room below" — so a capture that failed reported end-of-content. Refusing here keeps that + * inference from ever being drawn from a tree nobody could read. + * + * Sparseness reuses the same signals absence assertions already trust, rather than a second + * definition of "readable": the backend's own quality verdict, then the legacy iOS shape that + * predates it. Truncation is deliberately NOT refused — a truncated tree is a real, readable tree + * whose tail is missing, and refusing it would fail large screens where the target is plainly in + * view. + */ +export function scrollUntilCaptureRefusal( + capture: ScrollUntilCapture, +): ScrollUntilCaptureRefusal | undefined { + const nodes = capture.nodes; + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } + const quality = capture.snapshotQuality; + if (quality?.state === 'sparse') { + return { + reason: 'sparse-tree', + detail: quality.reason ?? 'the capture backend reported a sparse tree', + }; + } + if ( + isLegacySparseIosInteractiveSnapshot({ + backend: capture.backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(quality ? { snapshotQuality: quality } : {}), + }) + ) { + return { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }; + } + return undefined; +} diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts index 373372ce7f..b2405c0b2b 100644 --- a/src/commands/interaction/runtime/scroll.test.ts +++ b/src/commands/interaction/runtime/scroll.test.ts @@ -332,3 +332,46 @@ test('runtime scroll --until is refused on the edge directions, which already ca /scroll bottom already scrolls to the bottom edge and cannot take --until/, ); }); + +/** + * The defect this pins: a capture that comes back unreadable used to reach the edge analyzer as an + * empty tree, which reads it as "no room below" and reported end-of-content. A failed read is not + * evidence about the content. + */ +test('runtime scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ nodes: [] }), + scroll: async () => ({}), + }); + + await assert.rejects( + () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'no-capture'); + return true; + }, + ); +}); + +test('runtime scroll --until refuses a sparse capture rather than trusting its selectors', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ + snapshot: { + ...untilSnapshot(2400, true), + snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }, + }), + scroll: async () => ({}), + }); + + await assert.rejects( + () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + return true; + }, + ); +}); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts index bcf800ec65..0cfdba266b 100644 --- a/src/commands/interaction/runtime/scroll.ts +++ b/src/commands/interaction/runtime/scroll.ts @@ -17,9 +17,13 @@ import { import { formatScrollUntilMessage, runScrollUntilVisiblePasses, + scrollUntilCaptureError, scrollUntilNotFoundError, } from '@agent-device/capture-kit/scroll-until-visible'; -import { isSelectorVisibleInNodes } from '@agent-device/selectors/scroll-until-match'; +import { + isSelectorVisibleInNodes, + scrollUntilCaptureRefusal, +} from '@agent-device/selectors/scroll-until-match'; import { AppError } from '@agent-device/kernel/errors'; import { successText } from '@agent-device/kernel/success-text'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; @@ -310,7 +314,7 @@ async function runUntilScroll(params: { const edge = verticalEdgeFor(direction); const result = await runScrollUntilVisiblePasses({ ...(edge === undefined ? {} : { edge }), - captureNodes: async () => await captureRuntimeScrollNodes(runtime, options), + captureNodes: async () => await captureRuntimeScrollNodes(runtime, options, direction, until), isVisibleMatch: async (nodes) => await isSelectorVisibleInNodes({ nodes, @@ -354,9 +358,15 @@ function verticalEdgeFor(direction: GestureDirection): ScrollEdge | undefined { return undefined; } +/** + * The tree one pass reads, or a refusal. Never `?? []`: an unreadable capture that reached the edge + * analyzer as an empty tree is exactly how a failed read used to be reported as end-of-content. + */ async function captureRuntimeScrollNodes( runtime: AgentDeviceRuntime, options: ScrollCommandOptions, + direction: GestureDirection, + selector: string, ) { if (!runtime.backend.captureSnapshot) { throw new AppError( @@ -367,7 +377,10 @@ async function captureRuntimeScrollNodes( const result = await runtime.backend.captureSnapshot(toBackendContext(runtime, options), { includeRects: true, }); - return result.snapshot?.nodes ?? result.nodes ?? []; + const capture = result.snapshot ?? result; + const refusal = scrollUntilCaptureRefusal(capture); + if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); + return capture.nodes ?? []; } function requireDirection( diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index 90c72990d1..3a0cb432d3 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -427,3 +427,46 @@ test('bound scroll --until is refused when the owner advertises no capture', asy }); assert.equal(resolved.ok, false); }); + +/** Same defect as the command runtime's: a failed read is not evidence that the content ran out. */ +test('bound scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({}), + scroll: async () => ({}), + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'no-capture'); + return true; + }, + ); +}); + +test('bound scroll --until refuses a sparse capture rather than trusting its selectors', async () => { + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ + nodes: untilNodes(2400, true), + snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }), + scroll: async () => ({}), + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + return true; + }, + ); +}); diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 454f583ad9..8c71cc21aa 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -26,9 +26,13 @@ import { import { formatScrollUntilMessage, runScrollUntilVisiblePasses, + scrollUntilCaptureError, scrollUntilNotFoundError, } from '@agent-device/capture-kit/scroll-until-visible'; -import { isSelectorVisibleInNodes } from '@agent-device/selectors/scroll-until-match'; +import { + isSelectorVisibleInNodes, + scrollUntilCaptureRefusal, +} from '@agent-device/selectors/scroll-until-match'; import { publicPlatformString } from '@agent-device/kernel/device'; import { withSuccessText } from '@agent-device/kernel/success-text'; import type { DaemonCommandContext } from './context.ts'; @@ -216,7 +220,7 @@ async function executeUntilScroll( ...(verticalEdgeFor(target.direction) === undefined ? {} : { edge: verticalEdgeFor(target.direction) as ScrollEdge }), - captureNodes: async () => await captureUntilNodes(runtime, context), + captureNodes: async () => await captureUntilNodes(runtime, context, target.direction, selector), isVisibleMatch: async (nodes) => await isSelectorVisibleInNodes({ nodes, @@ -256,15 +260,23 @@ function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined { return undefined; } -async function captureUntilNodes(runtime: BoundScrollUntil, context: DaemonCommandContext) { - return ( - ( - await runtime.operations.captureSnapshot({ - options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, - execution: runtimeExecutionFromContext(context), - }) - ).nodes ?? [] - ); +/** + * The tree one pass reads, or a refusal. Never `?? []`: an unreadable capture that reached the edge + * analyzer as an empty tree is exactly how a failed read used to be reported as end-of-content. + */ +async function captureUntilNodes( + runtime: BoundScrollUntil, + context: DaemonCommandContext, + direction: ScrollDirection, + selector: string, +) { + const capture = await runtime.operations.captureSnapshot({ + options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, + execution: runtimeExecutionFromContext(context), + }); + const refusal = scrollUntilCaptureRefusal(capture); + if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); + return capture.nodes ?? []; } async function captureEdgeState( From bb520ccfbc92fc9fd14a43c1ff392e23026c58bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 10:30:36 +0200 Subject: [PATCH 06/13] fix(selectors): keep the capture-readability check off the eager import path --- .../selectors/src/scroll-until-match.test.ts | 18 +++++++++--------- packages/selectors/src/scroll-until-match.ts | 9 ++++++--- src/commands/interaction/runtime/scroll.ts | 2 +- src/daemon/scroll-runtime.ts | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts index 0dbe13bf94..fc10e11d94 100644 --- a/packages/selectors/src/scroll-until-match.test.ts +++ b/packages/selectors/src/scroll-until-match.test.ts @@ -86,20 +86,20 @@ test('an off-screen twin does not satisfy a selector whose other match is on scr ); }); -test('a capture with no tree at all is refused rather than read as an empty screen', () => { - assert.deepEqual(scrollUntilCaptureRefusal({}), { +test('a capture with no tree at all is refused rather than read as an empty screen', async () => { + assert.deepEqual(await scrollUntilCaptureRefusal({}), { reason: 'no-capture', detail: 'the capture returned no accessibility tree', }); - assert.deepEqual(scrollUntilCaptureRefusal({ nodes: [] }), { + assert.deepEqual(await scrollUntilCaptureRefusal({ nodes: [] }), { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree', }); }); -test('a backend sparse verdict is refused and carries the backend reason', () => { +test('a backend sparse verdict is refused and carries the backend reason', async () => { assert.deepEqual( - scrollUntilCaptureRefusal({ + await scrollUntilCaptureRefusal({ nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, }), @@ -107,9 +107,9 @@ test('a backend sparse verdict is refused and carries the backend reason', () => ); }); -test('the legacy iOS application-root-only shape is refused', () => { +test('the legacy iOS application-root-only shape is refused', async () => { assert.deepEqual( - scrollUntilCaptureRefusal({ + await scrollUntilCaptureRefusal({ backend: 'xctest', nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], }), @@ -121,9 +121,9 @@ test('the legacy iOS application-root-only shape is refused', () => { * Truncation is a readable tree missing its tail, not a failed read. Refusing it would fail large * screens where the target is plainly in view. */ -test('a truncated but populated capture is not refused', () => { +test('a truncated but populated capture is not refused', async () => { assert.equal( - scrollUntilCaptureRefusal({ + await scrollUntilCaptureRefusal({ nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), snapshotQuality: { state: 'ok', backend: 'tree' }, }), diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts index 48a09ff1c2..c8db831fce 100644 --- a/packages/selectors/src/scroll-until-match.ts +++ b/packages/selectors/src/scroll-until-match.ts @@ -7,7 +7,6 @@ import type { SnapshotQualityVerdict, SnapshotState, } from '@agent-device/kernel/snapshot'; -import { isLegacySparseIosInteractiveSnapshot } from './absence-observation.ts'; import type { SelectorPipelineOutcome } from './selector-pipeline.ts'; /** @@ -83,9 +82,9 @@ export type ScrollUntilCapture = { * whose tail is missing, and refusing it would fail large screens where the target is plainly in * view. */ -export function scrollUntilCaptureRefusal( +export async function scrollUntilCaptureRefusal( capture: ScrollUntilCapture, -): ScrollUntilCaptureRefusal | undefined { +): Promise { const nodes = capture.nodes; if (nodes === undefined) { return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; @@ -100,6 +99,10 @@ export function scrollUntilCaptureRefusal( detail: quality.reason ?? 'the capture backend reported a sparse tree', }; } + // Lazy for the same reason the pipeline edges are: `absence-observation` reaches `ad-script` for + // work unrelated to this two-line shape check, and paying that closure eagerly would put this + // module over the entry-surface ceiling. + const { isLegacySparseIosInteractiveSnapshot } = await import('./absence-observation.ts'); if ( isLegacySparseIosInteractiveSnapshot({ backend: capture.backend as SnapshotState['backend'], diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts index 0cfdba266b..f3f11c9f8b 100644 --- a/src/commands/interaction/runtime/scroll.ts +++ b/src/commands/interaction/runtime/scroll.ts @@ -378,7 +378,7 @@ async function captureRuntimeScrollNodes( includeRects: true, }); const capture = result.snapshot ?? result; - const refusal = scrollUntilCaptureRefusal(capture); + const refusal = await scrollUntilCaptureRefusal(capture); if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); return capture.nodes ?? []; } diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 8c71cc21aa..54aca880f5 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -274,7 +274,7 @@ async function captureUntilNodes( options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, execution: runtimeExecutionFromContext(context), }); - const refusal = scrollUntilCaptureRefusal(capture); + const refusal = await scrollUntilCaptureRefusal(capture); if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); return capture.nodes ?? []; } From f21808ee88a0befe5783923e8dce896c1e13a8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 10:37:26 +0200 Subject: [PATCH 07/13] test(selectors): use a declared snapshot quality state in the capture fixtures --- .../selectors/src/scroll-until-match.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts index fc10e11d94..e78dc68110 100644 --- a/packages/selectors/src/scroll-until-match.test.ts +++ b/packages/selectors/src/scroll-until-match.test.ts @@ -118,15 +118,25 @@ test('the legacy iOS application-root-only shape is refused', async () => { }); /** - * Truncation is a readable tree missing its tail, not a failed read. Refusing it would fail large + * A tree the backend vouches for is readable, and so is one whose tail was truncated: truncation + * drops content, it does not make the capture untrustworthy. Refusing either would fail large * screens where the target is plainly in view. */ -test('a truncated but populated capture is not refused', async () => { +test('a populated capture is not refused, healthy or recovered', async () => { + const nodes = tree({ ref: 'e2', label: 'Submit', y: 200 }); assert.equal( await scrollUntilCaptureRefusal({ - nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), - snapshotQuality: { state: 'ok', backend: 'tree' }, + nodes, + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }), + undefined, + ); + assert.equal( + await scrollUntilCaptureRefusal({ + nodes, + snapshotQuality: { state: 'recovered', backend: 'tree' }, }), undefined, ); + assert.equal(await scrollUntilCaptureRefusal({ nodes }), undefined); }); From 9bf045e0493bcd041e0e5fd83a564b70c335bb75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 11:25:24 +0200 Subject: [PATCH 08/13] fix(scroll): read the capture quality verdict under the spelling the backend uses --- .../selectors/src/scroll-until-match.test.ts | 44 +++++++++- packages/selectors/src/scroll-until-match.ts | 42 ++++++++- .../interaction/runtime/scroll.test.ts | 47 +++++++++- src/commands/interaction/runtime/scroll.ts | 7 +- src/daemon/__tests__/scroll-runtime.test.ts | 88 +++++++++++++++++++ 5 files changed, 215 insertions(+), 13 deletions(-) diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts index e78dc68110..e4646afdcf 100644 --- a/packages/selectors/src/scroll-until-match.test.ts +++ b/packages/selectors/src/scroll-until-match.test.ts @@ -97,16 +97,54 @@ test('a capture with no tree at all is refused rather than read as an empty scre }); }); -test('a backend sparse verdict is refused and carries the backend reason', async () => { +/** + * The verdict arrives under two spellings: `SnapshotState` says `snapshotQuality`, a + * `BackendSnapshotResult` says `quality`. Reading only one is how a real backend sparse verdict + * slipped through the first version of this check. + */ +test('a sparse verdict is refused under either spelling the capture can carry it in', async () => { + const nodes = tree({ ref: 'e2', label: 'Submit', y: 200 }); + const sparse = { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' } as const; + const expected = { reason: 'sparse-tree', detail: 'AX bridge unavailable' }; + + assert.deepEqual(await scrollUntilCaptureRefusal({ nodes, snapshotQuality: sparse }), expected); + // The backend result's own spelling. + assert.deepEqual(await scrollUntilCaptureRefusal({ nodes, quality: sparse }), expected); + // A backend result whose verdict sits above the nested state it also carries. + assert.deepEqual( + await scrollUntilCaptureRefusal({ quality: sparse, snapshot: { nodes } }), + expected, + ); + // A nested state carrying its own verdict. assert.deepEqual( + await scrollUntilCaptureRefusal({ snapshot: { nodes, snapshotQuality: sparse } }), + expected, + ); +}); + +test('a malformed quality payload is not mistaken for a verdict', async () => { + assert.equal( await scrollUntilCaptureRefusal({ nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), - snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + quality: { state: 'not-a-state' }, }), - { reason: 'sparse-tree', detail: 'AX bridge unavailable' }, + undefined, ); }); +test('a nested snapshot supplies the nodes when the top level has none', async () => { + assert.equal( + await scrollUntilCaptureRefusal({ + snapshot: { nodes: tree({ ref: 'e2', label: 'X', y: 10 }) }, + }), + undefined, + ); + assert.deepEqual(await scrollUntilCaptureRefusal({ snapshot: { nodes: [] } }), { + reason: 'no-capture', + detail: 'the capture returned an empty accessibility tree', + }); +}); + test('the legacy iOS application-root-only shape is refused', async () => { assert.deepEqual( await scrollUntilCaptureRefusal({ diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts index c8db831fce..c2b5d37216 100644 --- a/packages/selectors/src/scroll-until-match.ts +++ b/packages/selectors/src/scroll-until-match.ts @@ -1,5 +1,6 @@ import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import type { ScrollUntilCaptureRefusal } from '@agent-device/capture-kit/scroll-until-visible'; +import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { RawSnapshotNode, @@ -60,14 +61,48 @@ function matchedNodes(outcome: SelectorPipelineOutcome): readonly SnapshotNode[] } } -/** The fields a `--until` pass can read from either route's capture without reshaping it. */ +/** + * The fields a `--until` pass can read from either route's capture without reshaping it. + * + * The verdict is accepted under BOTH spellings on purpose. A `SnapshotState` calls it + * `snapshotQuality`; a `BackendSnapshotResult` calls it `quality` and may also carry a nested + * `snapshot`. Asking each caller to normalize is what let a real backend sparse verdict slip past + * the first version of this check, so the one place that asks the question reads every spelling + * the capture can arrive in. + */ export type ScrollUntilCapture = { nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; /** Widened to `string` because the backend capture result carries it untyped. */ backend?: string | undefined; snapshotQuality?: SnapshotQualityVerdict | undefined; + quality?: unknown; + snapshot?: { + nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; + backend?: string | undefined; + snapshotQuality?: SnapshotQualityVerdict | undefined; + }; }; +/** + * The nested `SnapshotState` wins on nodes and backend, and the verdict is taken from whichever + * level carries one — selecting `result.snapshot` alone used to drop a top-level `quality`. + */ +function canonicalCapture(capture: ScrollUntilCapture): { + nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; + backend?: string | undefined; + quality?: SnapshotQualityVerdict | undefined; +} { + const nested = capture.snapshot; + return { + nodes: nested?.nodes ?? capture.nodes, + backend: nested?.backend ?? capture.backend, + quality: + nested?.snapshotQuality ?? + capture.snapshotQuality ?? + readSnapshotQualityVerdict(capture.quality), + }; +} + /** * Whether this capture can answer the `--until` question, asked before the selector match and * before the edge analyzer. @@ -85,14 +120,13 @@ export type ScrollUntilCapture = { export async function scrollUntilCaptureRefusal( capture: ScrollUntilCapture, ): Promise { - const nodes = capture.nodes; + const { nodes, backend, quality } = canonicalCapture(capture); if (nodes === undefined) { return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; } if (nodes.length === 0) { return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; } - const quality = capture.snapshotQuality; if (quality?.state === 'sparse') { return { reason: 'sparse-tree', @@ -105,7 +139,7 @@ export async function scrollUntilCaptureRefusal( const { isLegacySparseIosInteractiveSnapshot } = await import('./absence-observation.ts'); if ( isLegacySparseIosInteractiveSnapshot({ - backend: capture.backend as SnapshotState['backend'], + backend: backend as SnapshotState['backend'], nodes: nodes as SnapshotNode[], ...(quality ? { snapshotQuality: quality } : {}), }) diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts index b2405c0b2b..ee1e92f821 100644 --- a/src/commands/interaction/runtime/scroll.test.ts +++ b/src/commands/interaction/runtime/scroll.test.ts @@ -337,11 +337,20 @@ test('runtime scroll --until is refused on the edge directions, which already ca * The defect this pins: a capture that comes back unreadable used to reach the edge analyzer as an * empty tree, which reads it as "no room below" and reported end-of-content. A failed read is not * evidence about the content. + * + * The sparse cases use the backend's own spelling of the verdict. `BackendSnapshotResult` calls it + * `quality` while the nested `SnapshotState` calls it `snapshotQuality`, and selecting one level + * used to drop the other's. Each case counts gestures, so the refusal is proven to land before + * matching, edge analysis or scrolling. */ test('runtime scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { + let scrolls = 0; const device = createInteractionDevice(selectorSnapshot(), { captureSnapshot: async () => ({ nodes: [] }), - scroll: async () => ({}), + scroll: async () => { + scrolls += 1; + return {}; + }, }); await assert.rejects( @@ -353,9 +362,37 @@ test('runtime scroll --until reports an unreadable capture as a capture failure, return true; }, ); + assert.equal(scrolls, 0); }); -test('runtime scroll --until refuses a sparse capture rather than trusting its selectors', async () => { +test('runtime scroll --until refuses a top-level backend sparse verdict beside a nested snapshot', async () => { + let scrolls = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ + snapshot: untilSnapshot(2400, true), + quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }), + scroll: async () => { + scrolls += 1; + return {}; + }, + }); + + await assert.rejects( + () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(String(error.message), /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('runtime scroll --until refuses a sparse verdict carried on the nested snapshot itself', async () => { + let scrolls = 0; const device = createInteractionDevice(selectorSnapshot(), { captureSnapshot: async () => ({ snapshot: { @@ -363,7 +400,10 @@ test('runtime scroll --until refuses a sparse capture rather than trusting its s snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, }, }), - scroll: async () => ({}), + scroll: async () => { + scrolls += 1; + return {}; + }, }); await assert.rejects( @@ -374,4 +414,5 @@ test('runtime scroll --until refuses a sparse capture rather than trusting its s return true; }, ); + assert.equal(scrolls, 0); }); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts index f3f11c9f8b..163fb5e009 100644 --- a/src/commands/interaction/runtime/scroll.ts +++ b/src/commands/interaction/runtime/scroll.ts @@ -377,10 +377,11 @@ async function captureRuntimeScrollNodes( const result = await runtime.backend.captureSnapshot(toBackendContext(runtime, options), { includeRects: true, }); - const capture = result.snapshot ?? result; - const refusal = await scrollUntilCaptureRefusal(capture); + // The whole result, not `result.snapshot`: the nested state and the top-level backend annotation + // spell the quality verdict differently, and picking one level drops the other's. + const refusal = await scrollUntilCaptureRefusal(result); if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); - return capture.nodes ?? []; + return result.snapshot?.nodes ?? result.nodes ?? []; } function requireDirection( diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index 3a0cb432d3..c3df04ec59 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -428,6 +428,94 @@ test('bound scroll --until is refused when the owner advertises no capture', asy assert.equal(resolved.ok, false); }); +/** Same defect as the command runtime's: a failed read is not evidence that the content ran out. */ +/** + * The owner's capture result spells the verdict `quality`, which is the shape this route actually + * receives — an earlier version of this test asserted through `snapshotQuality` and passed while + * the real field went unread. The scroll spy proves each refusal lands before any gesture, and the + * sparse tree deliberately has content below the fold, so an edge verdict would be wrong there too. + */ +test('bound scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({}), + scroll: async () => { + scrolls += 1; + return {}; + }, + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'no-capture'); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('bound scroll --until refuses a sparse capture before matching, edge analysis or scrolling', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScroll( + ['down'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ + nodes: untilNodes(2400, true), + quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, + }), + scroll: async () => { + scrolls += 1; + return {}; + }, + }, + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(String(error.message), /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('bound scroll rejects --until on an edge direction before any device work', async () => { + await assert.rejects( + () => + runScroll( + ['bottom'], + { until: 'label=Email' }, + { + captureSnapshot: async () => ({ nodes: untilNodes(300, true) }), + scroll: async () => { + throw new Error('scroll should be rejected before the backend call'); + }, + }, + ), + /scroll bottom already scrolls to the bottom edge and cannot take --until/, + ); +}); + +test('bound scroll --until is refused when the owner advertises no capture', async () => { + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals: ['down'], + context: { until: 'label=Email' } as DaemonCommandContext, + ...bindings({ scroll: async () => ({}) }), + }); + assert.equal(resolved.ok, false); +}); + /** Same defect as the command runtime's: a failed read is not evidence that the content ran out. */ test('bound scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { await assert.rejects( From 2f23ba6041e46818bd3e5df596aa417e5e53be4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 11:44:12 +0200 Subject: [PATCH 09/13] refactor(scroll): collapse --until onto the one route that runs it --- packages/capture-kit/package.json | 4 - .../__tests__/scroll-until-visible.test.ts | 152 ----------- .../src/snapshot/scroll-edge-state.ts | 12 + .../src/snapshot/scroll-until-visible.ts | 164 ----------- packages/selectors/package.json | 4 - .../selectors/src/scroll-until-match.test.ts | 180 ------------ packages/selectors/src/scroll-until-match.ts | 150 ---------- scripts/layering/package-boundaries.test.ts | 2 - .../interaction/runtime/scroll.test.ts | 180 ------------ src/commands/interaction/runtime/scroll.ts | 115 -------- src/daemon/scroll-runtime.ts | 68 +---- src/daemon/scroll-until.test.ts | 240 ++++++++++++++++ src/daemon/scroll-until.ts | 258 ++++++++++++++++++ .../provider-scenarios/scroll-until.test.ts | 2 +- 14 files changed, 521 insertions(+), 1010 deletions(-) delete mode 100644 packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts delete mode 100644 packages/capture-kit/src/snapshot/scroll-until-visible.ts delete mode 100644 packages/selectors/src/scroll-until-match.test.ts delete mode 100644 packages/selectors/src/scroll-until-match.ts create mode 100644 src/daemon/scroll-until.test.ts create mode 100644 src/daemon/scroll-until.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index c5a61f9401..4a997111cf 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -118,10 +118,6 @@ "types": "./src/snapshot/scroll-edge-state.ts", "default": "./src/snapshot/scroll-edge-state.ts" }, - "./scroll-until-visible": { - "types": "./src/snapshot/scroll-until-visible.ts", - "default": "./src/snapshot/scroll-until-visible.ts" - }, "./snapshot-chrome": { "types": "./src/snapshot-chrome.ts", "default": "./src/snapshot-chrome.ts" diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts deleted file mode 100644 index 1529e81f61..0000000000 --- a/packages/capture-kit/src/snapshot/__tests__/scroll-until-visible.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import { - SCROLL_UNTIL_PASS_LIMIT, - formatScrollUntilMessage, - runScrollUntilVisiblePasses, - scrollUntilNotFoundError, -} from '../scroll-until-visible.ts'; - -/** A tree the edge analyzer reads as "more content below": a scrollable with a clipped child. */ -function scrollableTree(childY: number): SnapshotNode[] { - return [ - { - index: 0, - ref: 'e1', - type: 'Application', - rect: { x: 0, y: 0, width: 400, height: 800 }, - } as SnapshotNode, - { - index: 1, - parentIndex: 0, - ref: 'e2', - type: 'ScrollView', - rect: { x: 0, y: 0, width: 400, height: 800 }, - } as SnapshotNode, - { - index: 2, - parentIndex: 1, - ref: 'e3', - type: 'TextField', - rect: { x: 0, y: childY, width: 400, height: 40 }, - } as SnapshotNode, - ]; -} - -test('a target that is already visible costs one capture and zero scrolls', async () => { - let scrolls = 0; - const outcome = await runScrollUntilVisiblePasses({ - edge: 'bottom', - captureNodes: async () => scrollableTree(100), - isVisibleMatch: () => true, - scroll: async () => { - scrolls += 1; - return { scrolled: true }; - }, - }); - assert.equal(outcome.outcome, 'matched'); - assert.equal(outcome.passes, 0); - assert.equal(scrolls, 0); -}); - -test('passes repeat until the injected predicate reports the target on screen', async () => { - let scrolls = 0; - const outcome = await runScrollUntilVisiblePasses({ - edge: 'bottom', - captureNodes: async () => scrollableTree(2000), - isVisibleMatch: () => scrolls >= 3, - scroll: async () => { - scrolls += 1; - return { pixels: 250 }; - }, - }); - assert.equal(outcome.outcome, 'matched'); - assert.equal(outcome.passes, 3); - assert.deepEqual(outcome.result, { pixels: 250 }); -}); - -test('running out of content stops the loop before the pass budget does', async () => { - let scrolls = 0; - const outcome = await runScrollUntilVisiblePasses({ - edge: 'bottom', - // No child below the fold: the edge analyzer reports nothing hidden underneath. - captureNodes: async () => scrollableTree(100), - isVisibleMatch: () => false, - scroll: async () => { - scrolls += 1; - return {}; - }, - }); - assert.equal(outcome.outcome, 'edge-reached'); - assert.equal(scrolls, 0); -}); - -test('a horizontal scroll has no edge signal and is bounded by the pass budget alone', async () => { - let scrolls = 0; - const outcome = await runScrollUntilVisiblePasses({ - passLimit: 4, - captureNodes: async () => scrollableTree(100), - isVisibleMatch: () => false, - scroll: async () => { - scrolls += 1; - return {}; - }, - }); - assert.equal(outcome.outcome, 'pass-limit'); - assert.equal(outcome.passes, 4); - assert.equal(scrolls, 4); -}); - -test('the default pass budget is the shared constant', async () => { - const outcome = await runScrollUntilVisiblePasses({ - captureNodes: async () => scrollableTree(100), - isVisibleMatch: () => false, - scroll: async () => ({}), - }); - assert.equal(outcome.passes, SCROLL_UNTIL_PASS_LIMIT); -}); - -test('an empty capture never counts as a match', async () => { - const outcome = await runScrollUntilVisiblePasses({ - passLimit: 1, - captureNodes: async () => [], - isVisibleMatch: (nodes) => nodes.length > 0, - scroll: async () => ({}), - }); - assert.equal(outcome.outcome, 'pass-limit'); -}); - -test('the success message distinguishes an already-visible target from a scrolled one', () => { - assert.equal( - formatScrollUntilMessage('down', 'id=email', 0), - 'id=email was already visible; no down scroll needed', - ); - assert.equal( - formatScrollUntilMessage('down', 'id=email', 1), - 'Scrolled down 1 pass until id=email was visible', - ); - assert.equal( - formatScrollUntilMessage('down', 'id=email', 3), - 'Scrolled down 3 passes until id=email was visible', - ); -}); - -test('the two failures carry distinct typed reasons and distinct corrective hints', () => { - const edge = scrollUntilNotFoundError({ - direction: 'down', - selector: 'id=email', - outcome: 'edge-reached', - passes: 2, - }); - const budget = scrollUntilNotFoundError({ - direction: 'down', - selector: 'id=email', - outcome: 'pass-limit', - passes: 12, - }); - assert.equal(edge.details?.reason, 'scroll_until_edge_reached'); - assert.equal(budget.details?.reason, 'scroll_until_pass_limit'); - assert.match(String(edge.details?.hint), /scroll the opposite direction/); - assert.match(String(budget.details?.hint), /Raise the step with an amount/); -}); diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index 0c2a57104f..9029ff9378 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -37,6 +37,18 @@ export async function captureScrollEdgeState(params: { } } +/** + * Is there hidden content left at this edge? The same question `runScrollEdgePasses` loops on, + * exposed for callers with their own stop condition (`scroll --until`) so both read one signal. + */ +export async function canScrollFurtherAtEdge( + nodes: readonly (RawSnapshotNode | SnapshotNode)[], + edge: ScrollEdge, +): Promise { + const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts'); + return analyzeScrollEdgeState(nodes, edge).canScroll; +} + export async function runScrollEdgePasses(params: { edge: ScrollEdge; captureState: (scope?: string) => Promise; diff --git a/packages/capture-kit/src/snapshot/scroll-until-visible.ts b/packages/capture-kit/src/snapshot/scroll-until-visible.ts deleted file mode 100644 index 730f92becb..0000000000 --- a/packages/capture-kit/src/snapshot/scroll-until-visible.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { AppError } from '@agent-device/kernel/errors'; -import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; -import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { ScrollEdge } from './scroll-edge-state.ts'; - -/** - * How many gestures one `scroll --until` may spend before it gives up. A pass costs a capture plus - * a gesture, so this is the request's whole cost ceiling, not a retry budget: 12 passes at the - * honored 0.8-viewport maximum cover roughly ten screens of content, which is past the point where - * a list is better reached by `scroll bottom` or a search field. - */ -export const SCROLL_UNTIL_PASS_LIMIT = 12; - -/** - * Why the loop stopped. `matched` is the only success; the other two are the two distinguishable - * ways a target never came into view, and callers report them differently because the corrective - * action differs — an exhausted list needs a different direction, an exhausted budget needs a - * bigger step or a narrower selector. - */ -export type ScrollUntilVisibleOutcome = 'matched' | 'edge-reached' | 'pass-limit'; - -/** - * Why a capture cannot answer the `--until` question at all. - * - * Distinct from the loop's outcomes on purpose: an unreadable capture is not evidence about the - * content, and collapsing the two is how `?? []` used to turn a failed read into "you reached the - * end of the list". The classifier that produces this lives in `@agent-device/selectors`, which is - * where the same readability question is already answered for absence assertions; the vocabulary - * lives here beside the outcomes it must not be confused with. - */ -export type ScrollUntilCaptureRefusal = { - reason: 'no-capture' | 'sparse-tree'; - detail: string; -}; - -export type ScrollUntilVisibleResult = { - passes: number; - outcome: ScrollUntilVisibleOutcome; - result?: TResult; -}; - -type CapturedNodes = readonly (RawSnapshotNode | SnapshotNode)[]; - -/** - * Scrolls until an injected predicate says the target is on screen. - * - * The predicate is injected rather than resolved here because the two callers (the daemon's generic - * scroll route and the in-process command runtime) reach selector matching through different - * layers; keeping the loop predicate-shaped is what lets both share one definition of when to stop. - * - * `edge` is the end-of-content signal, and it is the SAME signal `scroll top`/`scroll bottom` - * already trust (`analyzeScrollEdgeState`), so a list that reports no room below stops this loop - * exactly where an edge scroll would stop. Horizontal scrolls have no such analyzer and are bounded - * by `passLimit` alone. - * - * The first capture happens before the first gesture: a target that is already visible costs one - * capture and zero scrolls. - */ -export async function runScrollUntilVisiblePasses(params: { - edge?: ScrollEdge; - passLimit?: number; - captureNodes: () => Promise; - isVisibleMatch: (nodes: CapturedNodes) => Promise | boolean; - scroll: () => Promise; -}): Promise> { - const { edge, captureNodes, isVisibleMatch, scroll } = params; - const passLimit = params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT; - let passes = 0; - let result: TResult | undefined; - const stop = (outcome: ScrollUntilVisibleOutcome): ScrollUntilVisibleResult => ({ - passes, - outcome, - ...(result === undefined ? {} : { result }), - }); - - while (true) { - const nodes = await captureNodes(); - if (await isVisibleMatch(nodes)) return stop('matched'); - if (edge && !(await canScrollFurther(nodes, edge))) return stop('edge-reached'); - if (passes >= passLimit) return stop('pass-limit'); - result = await scroll(); - passes += 1; - } -} - -async function canScrollFurther(nodes: CapturedNodes, edge: ScrollEdge): Promise { - const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts'); - return analyzeScrollEdgeState(nodes, edge).canScroll; -} - -export function formatScrollUntilMessage( - direction: ScrollDirection, - selector: string, - passes: number, -): string { - if (passes === 0) return `${selector} was already visible; no ${direction} scroll needed`; - return `Scrolled ${direction} ${passes} ${passes === 1 ? 'pass' : 'passes'} until ${selector} was visible`; -} - -/** - * The two ways the loop can end without the target on screen. They are separate messages because - * the corrective action differs: content that ran out needs a different direction or a target that - * is not on this screen at all, while an exhausted budget needs a bigger step or a selector that - * matches something nearer. - */ -export function scrollUntilNotFoundError(params: { - direction: ScrollDirection; - selector: string; - outcome: Exclude; - passes: number; -}): AppError { - const { direction, selector, outcome, passes } = params; - if (outcome === 'edge-reached') { - return new AppError( - 'COMMAND_FAILED', - `scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`, - { - reason: 'scroll_until_edge_reached', - selector, - direction, - passes, - hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, - }, - ); - } - return new AppError( - 'COMMAND_FAILED', - `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, - { - reason: 'scroll_until_pass_limit', - selector, - direction, - passes, - hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, - }, - ); -} - -/** - * The capture could not be read, so neither the selector match nor the edge analyzer ran. Reported - * as its own failure rather than as an outcome, because "we could not see the screen" and "the - * content ran out" call for different next steps. - */ -export function scrollUntilCaptureError(params: { - direction: ScrollDirection; - selector: string; - refusal: ScrollUntilCaptureRefusal; -}): AppError { - const { direction, selector, refusal } = params; - return new AppError( - 'COMMAND_FAILED', - `scroll ${direction} --until ${selector} could not read the screen: ${refusal.detail}`, - { - reason: 'scroll_until_capture_unreadable', - selector, - direction, - captureRefusal: refusal.reason, - hint: - refusal.reason === 'no-capture' - ? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.' - : 'The accessibility tree came back sparse, so its refs and selectors are not trustworthy. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.', - }, - ); -} diff --git a/packages/selectors/package.json b/packages/selectors/package.json index 7bbe050d85..079b6eef24 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -68,10 +68,6 @@ "types": "./src/selector-pipeline.ts", "default": "./src/selector-pipeline.ts" }, - "./scroll-until-match": { - "types": "./src/scroll-until-match.ts", - "default": "./src/scroll-until-match.ts" - }, "./selector-pipeline-policy": { "types": "./src/selector-pipeline-policy.ts", "default": "./src/selector-pipeline-policy.ts" diff --git a/packages/selectors/src/scroll-until-match.test.ts b/packages/selectors/src/scroll-until-match.test.ts deleted file mode 100644 index e4646afdcf..0000000000 --- a/packages/selectors/src/scroll-until-match.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import { isSelectorVisibleInNodes, scrollUntilCaptureRefusal } from './scroll-until-match.ts'; - -const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; - -function tree(...rows: { ref: string; label: string; y: number }[]): SnapshotNode[] { - return [ - { index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode, - ...rows.map( - (row, offset) => - ({ - index: offset + 1, - parentIndex: 0, - ref: row.ref, - type: 'Button', - label: row.label, - rect: { x: 0, y: row.y, width: 400, height: 40 }, - }) as SnapshotNode, - ), - ]; -} - -test('a match inside the viewport is visible', async () => { - assert.equal( - await isSelectorVisibleInNodes({ - nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), - selector: 'label=Submit', - platform: 'ios', - }), - true, - ); -}); - -test('a match scrolled below the fold is present but not visible', async () => { - assert.equal( - await isSelectorVisibleInNodes({ - nodes: tree({ ref: 'e2', label: 'Submit', y: 2400 }), - selector: 'label=Submit', - platform: 'ios', - }), - false, - ); -}); - -test('a selector matching nothing is not visible', async () => { - assert.equal( - await isSelectorVisibleInNodes({ - nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), - selector: 'label=Missing', - platform: 'ios', - }), - false, - ); -}); - -test('an empty capture is not a match', async () => { - assert.equal( - await isSelectorVisibleInNodes({ nodes: [], selector: 'label=Submit', platform: 'ios' }), - false, - ); -}); - -/** - * The reason the predicate asks "some match", not "the first match": a list can hold rows that - * share a selector, and the one above the fold must not end a scroll that has not yet reached the - * row the caller can act on. - */ -test('an off-screen twin does not satisfy a selector whose other match is on screen', async () => { - assert.equal( - await isSelectorVisibleInNodes({ - nodes: tree({ ref: 'e2', label: 'Row', y: -900 }, { ref: 'e3', label: 'Row', y: 300 }), - selector: 'label=Row', - platform: 'ios', - }), - true, - ); - assert.equal( - await isSelectorVisibleInNodes({ - nodes: tree({ ref: 'e2', label: 'Row', y: -900 }, { ref: 'e3', label: 'Row', y: 3000 }), - selector: 'label=Row', - platform: 'ios', - }), - false, - ); -}); - -test('a capture with no tree at all is refused rather than read as an empty screen', async () => { - assert.deepEqual(await scrollUntilCaptureRefusal({}), { - reason: 'no-capture', - detail: 'the capture returned no accessibility tree', - }); - assert.deepEqual(await scrollUntilCaptureRefusal({ nodes: [] }), { - reason: 'no-capture', - detail: 'the capture returned an empty accessibility tree', - }); -}); - -/** - * The verdict arrives under two spellings: `SnapshotState` says `snapshotQuality`, a - * `BackendSnapshotResult` says `quality`. Reading only one is how a real backend sparse verdict - * slipped through the first version of this check. - */ -test('a sparse verdict is refused under either spelling the capture can carry it in', async () => { - const nodes = tree({ ref: 'e2', label: 'Submit', y: 200 }); - const sparse = { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' } as const; - const expected = { reason: 'sparse-tree', detail: 'AX bridge unavailable' }; - - assert.deepEqual(await scrollUntilCaptureRefusal({ nodes, snapshotQuality: sparse }), expected); - // The backend result's own spelling. - assert.deepEqual(await scrollUntilCaptureRefusal({ nodes, quality: sparse }), expected); - // A backend result whose verdict sits above the nested state it also carries. - assert.deepEqual( - await scrollUntilCaptureRefusal({ quality: sparse, snapshot: { nodes } }), - expected, - ); - // A nested state carrying its own verdict. - assert.deepEqual( - await scrollUntilCaptureRefusal({ snapshot: { nodes, snapshotQuality: sparse } }), - expected, - ); -}); - -test('a malformed quality payload is not mistaken for a verdict', async () => { - assert.equal( - await scrollUntilCaptureRefusal({ - nodes: tree({ ref: 'e2', label: 'Submit', y: 200 }), - quality: { state: 'not-a-state' }, - }), - undefined, - ); -}); - -test('a nested snapshot supplies the nodes when the top level has none', async () => { - assert.equal( - await scrollUntilCaptureRefusal({ - snapshot: { nodes: tree({ ref: 'e2', label: 'X', y: 10 }) }, - }), - undefined, - ); - assert.deepEqual(await scrollUntilCaptureRefusal({ snapshot: { nodes: [] } }), { - reason: 'no-capture', - detail: 'the capture returned an empty accessibility tree', - }); -}); - -test('the legacy iOS application-root-only shape is refused', async () => { - assert.deepEqual( - await scrollUntilCaptureRefusal({ - backend: 'xctest', - nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], - }), - { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }, - ); -}); - -/** - * A tree the backend vouches for is readable, and so is one whose tail was truncated: truncation - * drops content, it does not make the capture untrustworthy. Refusing either would fail large - * screens where the target is plainly in view. - */ -test('a populated capture is not refused, healthy or recovered', async () => { - const nodes = tree({ ref: 'e2', label: 'Submit', y: 200 }); - assert.equal( - await scrollUntilCaptureRefusal({ - nodes, - snapshotQuality: { state: 'healthy', backend: 'tree' }, - }), - undefined, - ); - assert.equal( - await scrollUntilCaptureRefusal({ - nodes, - snapshotQuality: { state: 'recovered', backend: 'tree' }, - }), - undefined, - ); - assert.equal(await scrollUntilCaptureRefusal({ nodes }), undefined); -}); diff --git a/packages/selectors/src/scroll-until-match.ts b/packages/selectors/src/scroll-until-match.ts deleted file mode 100644 index c2b5d37216..0000000000 --- a/packages/selectors/src/scroll-until-match.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; -import type { ScrollUntilCaptureRefusal } from '@agent-device/capture-kit/scroll-until-visible'; -import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { - RawSnapshotNode, - SnapshotNode, - SnapshotQualityVerdict, - SnapshotState, -} from '@agent-device/kernel/snapshot'; -import type { SelectorPipelineOutcome } from './selector-pipeline.ts'; - -/** - * The stop condition `scroll --until` asks of every capture: does this selector match a node that - * is on screen right now? - * - * One definition for both callers — the daemon's generic scroll route and the in-process command - * runtime — so the two paths cannot disagree about when a scroll has arrived. It is deliberately - * two questions, not one: the `wait` pipeline row answers presence and ignores off-screen, then - * `isVisibleOnScreen` answers the part `--until` actually cares about. Reusing the presence row - * unchanged is what keeps a target that is present-but-scrolled-out from ending the loop early. - */ -export async function isSelectorVisibleInNodes(params: { - nodes: readonly (RawSnapshotNode | SnapshotNode)[]; - selector: string; - platform: Platform | PublicPlatform; -}): Promise { - const nodes = params.nodes as SnapshotNode[]; - if (nodes.length === 0) return false; - // Both edges are lazy on purpose. The policy table re-enters the package barrel and the pipeline - // pulls the match engine, which together would make this small predicate a 66-module entry - // surface for every importer. The loop that calls this awaits anyway, and the module cache makes - // every pass after the first free. - const [{ SELECTOR_PIPELINE_POLICIES }, { resolveSelectorPipeline }] = await Promise.all([ - import('./selector-pipeline-policy.ts'), - import('./selector-pipeline.ts'), - ]); - const outcome = await resolveSelectorPipeline( - SELECTOR_PIPELINE_POLICIES.wait, - nodes, - params.selector, - { platform: params.platform }, - ); - const matched = matchedNodes(outcome); - if (matched.length === 0) return false; - const visibility = createSnapshotVisibility(nodes); - // SOME match, not the first: a list whose rows share a selector can hold an off-screen twin above - // the fold, and stopping on that twin would leave the target the agent asked for still hidden. - return matched.some((node) => visibility.isVisibleOnScreen(node)); -} - -function matchedNodes(outcome: SelectorPipelineOutcome): readonly SnapshotNode[] { - switch (outcome.kind) { - case 'target': - case 'ambiguous': - return outcome.matchedNodes; - case 'occluded': - return [outcome.node]; - case 'none': - return []; - } -} - -/** - * The fields a `--until` pass can read from either route's capture without reshaping it. - * - * The verdict is accepted under BOTH spellings on purpose. A `SnapshotState` calls it - * `snapshotQuality`; a `BackendSnapshotResult` calls it `quality` and may also carry a nested - * `snapshot`. Asking each caller to normalize is what let a real backend sparse verdict slip past - * the first version of this check, so the one place that asks the question reads every spelling - * the capture can arrive in. - */ -export type ScrollUntilCapture = { - nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; - /** Widened to `string` because the backend capture result carries it untyped. */ - backend?: string | undefined; - snapshotQuality?: SnapshotQualityVerdict | undefined; - quality?: unknown; - snapshot?: { - nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; - backend?: string | undefined; - snapshotQuality?: SnapshotQualityVerdict | undefined; - }; -}; - -/** - * The nested `SnapshotState` wins on nodes and backend, and the verdict is taken from whichever - * level carries one — selecting `result.snapshot` alone used to drop a top-level `quality`. - */ -function canonicalCapture(capture: ScrollUntilCapture): { - nodes?: readonly (RawSnapshotNode | SnapshotNode)[] | undefined; - backend?: string | undefined; - quality?: SnapshotQualityVerdict | undefined; -} { - const nested = capture.snapshot; - return { - nodes: nested?.nodes ?? capture.nodes, - backend: nested?.backend ?? capture.backend, - quality: - nested?.snapshotQuality ?? - capture.snapshotQuality ?? - readSnapshotQualityVerdict(capture.quality), - }; -} - -/** - * Whether this capture can answer the `--until` question, asked before the selector match and - * before the edge analyzer. - * - * Both callers previously coerced a missing tree to `[]`, which the vertical edge analyzer reads as - * "no room below" — so a capture that failed reported end-of-content. Refusing here keeps that - * inference from ever being drawn from a tree nobody could read. - * - * Sparseness reuses the same signals absence assertions already trust, rather than a second - * definition of "readable": the backend's own quality verdict, then the legacy iOS shape that - * predates it. Truncation is deliberately NOT refused — a truncated tree is a real, readable tree - * whose tail is missing, and refusing it would fail large screens where the target is plainly in - * view. - */ -export async function scrollUntilCaptureRefusal( - capture: ScrollUntilCapture, -): Promise { - const { nodes, backend, quality } = canonicalCapture(capture); - if (nodes === undefined) { - return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; - } - if (nodes.length === 0) { - return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; - } - if (quality?.state === 'sparse') { - return { - reason: 'sparse-tree', - detail: quality.reason ?? 'the capture backend reported a sparse tree', - }; - } - // Lazy for the same reason the pipeline edges are: `absence-observation` reaches `ad-script` for - // work unrelated to this two-line shape check, and paying that closure eagerly would put this - // module over the entry-surface ceiling. - const { isLegacySparseIosInteractiveSnapshot } = await import('./absence-observation.ts'); - if ( - isLegacySparseIosInteractiveSnapshot({ - backend: backend as SnapshotState['backend'], - nodes: nodes as SnapshotNode[], - ...(quality ? { snapshotQuality: quality } : {}), - }) - ) { - return { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }; - } - return undefined; -} diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 3f7f744ed6..faa8be5417 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -418,7 +418,6 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/screenshot-diff-pixels', '@agent-device/capture-kit/screenshot-overlay', '@agent-device/capture-kit/scroll-edge-state', - '@agent-device/capture-kit/scroll-until-visible', '@agent-device/capture-kit/snapshot-chrome', '@agent-device/capture-kit/snapshot-desktop-projection', '@agent-device/capture-kit/snapshot-desktop-surface', @@ -653,7 +652,6 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/selectors/interaction-touch-point', '@agent-device/selectors/parameterized-recorded-fill', '@agent-device/selectors/press-retarget', - '@agent-device/selectors/scroll-until-match', '@agent-device/selectors/selector-pipeline', '@agent-device/selectors/selector-pipeline-policy', '@agent-device/selectors/snapshot-geometry-fixtures', diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts index ee1e92f821..fefd017928 100644 --- a/src/commands/interaction/runtime/scroll.test.ts +++ b/src/commands/interaction/runtime/scroll.test.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { selector } from './selector-read-utils.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { createInteractionDevice, runtimeScrollSnapshot, @@ -237,182 +236,3 @@ test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => ); } }); - -/** A viewport-height tree whose target row sits at `targetY`, used to walk a target into view. */ -function untilSnapshot(targetY: number, hiddenBelow: boolean) { - return makeSnapshotState([ - { - index: 1, - depth: 0, - type: 'ScrollView', - label: 'Form', - hiddenContentBelow: hiddenBelow ? true : undefined, - rect: { x: 0, y: 0, width: 400, height: 800 }, - }, - { - index: 2, - depth: 1, - parentIndex: 1, - type: 'TextField', - label: 'Email', - rect: { x: 0, y: targetY, width: 400, height: 40 }, - hittable: true, - }, - ]); -} - -test('runtime scroll --until stops the pass loop as soon as the selector is on screen', async () => { - const scrolls: unknown[] = []; - const frames = [untilSnapshot(2400, true), untilSnapshot(1200, true), untilSnapshot(300, true)]; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: frames[Math.min(scrolls.length, frames.length - 1)], - }), - scroll: async (_context, target, options) => { - scrolls.push({ target, options }); - return { pixels: 480 }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'down', - until: 'label=Email', - }); - - assert.equal(result.until, 'label=Email'); - assert.equal(result.passes, 2); - assert.equal(scrolls.length, 2); - assert.match(String(result.message), /Scrolled down 2 passes until label=Email was visible/); -}); - -test('runtime scroll --until performs no gesture when the target is already on screen', async () => { - const scrolls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ snapshot: untilSnapshot(300, true) }), - scroll: async () => { - scrolls.push('scrolled'); - return {}; - }, - }); - - const result = await device.interactions.scroll({ direction: 'down', until: 'label=Email' }); - - assert.equal(result.passes, 0); - assert.equal(scrolls.length, 0); - assert.match(String(result.message), /already visible/); -}); - -test('runtime scroll --until fails with the end-of-content reason when the list runs out', async () => { - // Nothing below the fold and nothing hidden: the same signal `scroll bottom` stops on. - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ snapshot: untilSnapshot(300, false) }), - scroll: async () => ({}), - }); - - await assert.rejects( - () => device.interactions.scroll({ direction: 'down', until: 'label=Missing' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_edge_reached'); - return true; - }, - ); -}); - -test('runtime scroll --until is refused on the edge directions, which already carry a stop condition', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ snapshot: untilSnapshot(300, true) }), - scroll: async () => { - throw new Error('scroll should be rejected before any backend call'); - }, - }); - - await assert.rejects( - () => device.interactions.scroll({ direction: 'bottom', until: 'label=Email' }), - /scroll bottom already scrolls to the bottom edge and cannot take --until/, - ); -}); - -/** - * The defect this pins: a capture that comes back unreadable used to reach the edge analyzer as an - * empty tree, which reads it as "no room below" and reported end-of-content. A failed read is not - * evidence about the content. - * - * The sparse cases use the backend's own spelling of the verdict. `BackendSnapshotResult` calls it - * `quality` while the nested `SnapshotState` calls it `snapshotQuality`, and selecting one level - * used to drop the other's. Each case counts gestures, so the refusal is proven to land before - * matching, edge analysis or scrolling. - */ -test('runtime scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { - let scrolls = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ nodes: [] }), - scroll: async () => { - scrolls += 1; - return {}; - }, - }); - - await assert.rejects( - () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); - assert.equal(error.details?.captureRefusal, 'no-capture'); - return true; - }, - ); - assert.equal(scrolls, 0); -}); - -test('runtime scroll --until refuses a top-level backend sparse verdict beside a nested snapshot', async () => { - let scrolls = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: untilSnapshot(2400, true), - quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, - }), - scroll: async () => { - scrolls += 1; - return {}; - }, - }); - - await assert.rejects( - () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); - assert.equal(error.details?.captureRefusal, 'sparse-tree'); - assert.match(String(error.message), /AX bridge unavailable/); - return true; - }, - ); - assert.equal(scrolls, 0); -}); - -test('runtime scroll --until refuses a sparse verdict carried on the nested snapshot itself', async () => { - let scrolls = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: { - ...untilSnapshot(2400, true), - snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, - }, - }), - scroll: async () => { - scrolls += 1; - return {}; - }, - }); - - await assert.rejects( - () => device.interactions.scroll({ direction: 'down', until: 'label=Email' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.captureRefusal, 'sparse-tree'); - return true; - }, - ); - assert.equal(scrolls, 0); -}); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts index 163fb5e009..c1cf2a5bb2 100644 --- a/src/commands/interaction/runtime/scroll.ts +++ b/src/commands/interaction/runtime/scroll.ts @@ -1,6 +1,5 @@ import { assertExclusiveScrollDistanceInputs, - assertScrollUntilCompatible, honoredScrollDurationMs, normalizeScrollDurationMs, resolveScrollExecutionOptions, @@ -14,16 +13,6 @@ import { type ScrollEdgeState, type ScrollEdgeTarget, } from '@agent-device/capture-kit/scroll-edge-state'; -import { - formatScrollUntilMessage, - runScrollUntilVisiblePasses, - scrollUntilCaptureError, - scrollUntilNotFoundError, -} from '@agent-device/capture-kit/scroll-until-visible'; -import { - isSelectorVisibleInNodes, - scrollUntilCaptureRefusal, -} from '@agent-device/selectors/scroll-until-match'; import { AppError } from '@agent-device/kernel/errors'; import { successText } from '@agent-device/kernel/success-text'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; @@ -59,8 +48,6 @@ export type ScrollCommandOptions = CommandContext & { amount?: number; pixels?: number; durationMs?: number; - /** Repeat passes until this selector is visible on screen, then stop. */ - until?: string; }; export type ScrollCommandResult = @@ -68,7 +55,6 @@ export type ScrollCommandResult = kind: 'viewport'; direction: GestureDirection; edge?: 'top' | 'bottom'; - until?: string; passes?: number; amount?: number; pixels?: number; @@ -78,7 +64,6 @@ export type ScrollCommandResult = ResolvedInteractionTarget & { direction: GestureDirection; edge?: 'top' | 'bottom'; - until?: string; passes?: number; amount?: number; pixels?: number; @@ -106,17 +91,6 @@ export const scrollCommand: RuntimeCommand`: repeat the pass until the selector is on screen. - * - * A sibling of the edge branch rather than a variant of the one-pass branch — it owns a different - * stop condition, a different failure vocabulary, and a result that names the selector it stopped - * on, none of which the ordinary scroll result carries. - */ -async function runUntilScroll(params: { - runtime: AgentDeviceRuntime; - options: ScrollCommandOptions; - resolved: ResolvedScrollTarget; - direction: GestureDirection; - until: string; - distance: { amount?: number; pixels?: number }; - scroll: () => Promise>>>; -}): Promise { - const { runtime, options, resolved, direction, until, distance } = params; - const edge = verticalEdgeFor(direction); - const result = await runScrollUntilVisiblePasses({ - ...(edge === undefined ? {} : { edge }), - captureNodes: async () => await captureRuntimeScrollNodes(runtime, options, direction, until), - isVisibleMatch: async (nodes) => - await isSelectorVisibleInNodes({ - nodes, - selector: until, - platform: runtime.backend.platform, - }), - scroll: params.scroll, - }); - if (result.outcome !== 'matched') { - throw scrollUntilNotFoundError({ - direction, - selector: until, - outcome: result.outcome, - passes: result.passes, - }); - } - const backendResult = toBackendResult(result.result); - return { - ...resolved, - direction, - until, - passes: result.passes, - ...distance, - ...(backendResult ? { backendResult } : {}), - ...successText(formatScrollUntilMessage(direction, until, result.passes)), - }; -} - /** The travel the planner produced, which saturates below a large requested amount. */ function honoredScrollPixels(result: Record | undefined): number | undefined { return typeof result?.pixels === 'number' ? result.pixels : undefined; } -/** - * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by - * its pass budget alone rather than by a signal that would always report "no room". - */ -function verticalEdgeFor(direction: GestureDirection): ScrollEdge | undefined { - if (direction === 'down') return 'bottom'; - if (direction === 'up') return 'top'; - return undefined; -} - -/** - * The tree one pass reads, or a refusal. Never `?? []`: an unreadable capture that reached the edge - * analyzer as an empty tree is exactly how a failed read used to be reported as end-of-content. - */ -async function captureRuntimeScrollNodes( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, - direction: GestureDirection, - selector: string, -) { - if (!runtime.backend.captureSnapshot) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'scroll --until requires snapshot support to check whether the selector became visible', - ); - } - const result = await runtime.backend.captureSnapshot(toBackendContext(runtime, options), { - includeRects: true, - }); - // The whole result, not `result.snapshot`: the nested state and the top-level backend annotation - // spell the quality verdict differently, and picking one level drops the other's. - const refusal = await scrollUntilCaptureRefusal(result); - if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); - return result.snapshot?.nodes ?? result.nodes ?? []; -} - function requireDirection( direction: GestureDirection | undefined, field: string, diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 54aca880f5..587550003d 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -23,16 +23,7 @@ import { type ScrollEdge, type ScrollEdgeState, } from '@agent-device/capture-kit/scroll-edge-state'; -import { - formatScrollUntilMessage, - runScrollUntilVisiblePasses, - scrollUntilCaptureError, - scrollUntilNotFoundError, -} from '@agent-device/capture-kit/scroll-until-visible'; -import { - isSelectorVisibleInNodes, - scrollUntilCaptureRefusal, -} from '@agent-device/selectors/scroll-until-match'; +import { formatScrollUntilMessage, runScrollUntilVisible } from './scroll-until.ts'; import { publicPlatformString } from '@agent-device/kernel/device'; import { withSuccessText } from '@agent-device/kernel/success-text'; import type { DaemonCommandContext } from './context.ts'; @@ -207,7 +198,7 @@ async function executeEdgeScroll( return scrollResult(target, options, edgeResult.passes, edgeResult.result ?? {}); } -/** Repeats the pass until the selector is on screen, the content runs out, or the budget does. */ +/** Repeats the pass until the selector is on screen; every failure shape is owned by the loop. */ async function executeUntilScroll( runtime: BoundScrollUntil, device: DeviceInfo, @@ -216,27 +207,17 @@ async function executeUntilScroll( options: ResolvedScrollExecutionOptions, context: DaemonCommandContext, ): Promise> { - const untilResult = await runScrollUntilVisiblePasses({ - ...(verticalEdgeFor(target.direction) === undefined - ? {} - : { edge: verticalEdgeFor(target.direction) as ScrollEdge }), - captureNodes: async () => await captureUntilNodes(runtime, context, target.direction, selector), - isVisibleMatch: async (nodes) => - await isSelectorVisibleInNodes({ - nodes, - selector, - platform: publicPlatformString(device), + const untilResult = await runScrollUntilVisible({ + selector, + direction: target.direction, + platform: publicPlatformString(device), + capture: async () => + await runtime.operations.captureSnapshot({ + options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, + execution: runtimeExecutionFromContext(context), }), scroll: async () => await scrollOnce(runtime, target, options, context), }); - if (untilResult.outcome !== 'matched') { - throw scrollUntilNotFoundError({ - direction: target.direction, - selector, - outcome: untilResult.outcome, - passes: untilResult.passes, - }); - } return withSuccessText( { direction: target.direction, @@ -250,35 +231,6 @@ async function executeUntilScroll( ); } -/** - * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by - * its pass budget alone rather than by a signal that would always report "no room". - */ -function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined { - if (direction === 'down') return 'bottom'; - if (direction === 'up') return 'top'; - return undefined; -} - -/** - * The tree one pass reads, or a refusal. Never `?? []`: an unreadable capture that reached the edge - * analyzer as an empty tree is exactly how a failed read used to be reported as end-of-content. - */ -async function captureUntilNodes( - runtime: BoundScrollUntil, - context: DaemonCommandContext, - direction: ScrollDirection, - selector: string, -) { - const capture = await runtime.operations.captureSnapshot({ - options: context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }, - execution: runtimeExecutionFromContext(context), - }); - const refusal = await scrollUntilCaptureRefusal(capture); - if (refusal) throw scrollUntilCaptureError({ direction, selector, refusal }); - return capture.nodes ?? []; -} - async function captureEdgeState( runtime: BoundScrollEdge, edge: ScrollEdge, diff --git a/src/daemon/scroll-until.test.ts b/src/daemon/scroll-until.test.ts new file mode 100644 index 0000000000..3a113819c0 --- /dev/null +++ b/src/daemon/scroll-until.test.ts @@ -0,0 +1,240 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + SCROLL_UNTIL_PASS_LIMIT, + formatScrollUntilMessage, + runScrollUntilVisible, + type ScrollUntilCapture, +} from './scroll-until.ts'; + +const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; +const SPARSE = { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' } as const; + +/** A scrollable whose single row sits at `rowY`; below 800 is off-screen with content beneath. */ +function tree(rowY: number, label = 'Email'): SnapshotNode[] { + return [ + { index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode, + { index: 1, parentIndex: 0, ref: 'e2', type: 'ScrollView', rect: VIEWPORT } as SnapshotNode, + { + index: 2, + parentIndex: 1, + ref: 'e3', + type: 'TextField', + label, + rect: { x: 0, y: rowY, width: 400, height: 40 }, + } as SnapshotNode, + ]; +} + +async function run(params: { + captures: ScrollUntilCapture[]; + selector?: string; + passLimit?: number; + onScroll?: () => void; +}) { + let index = 0; + return await runScrollUntilVisible({ + selector: params.selector ?? 'label=Email', + direction: 'down', + platform: 'ios', + ...(params.passLimit === undefined ? {} : { passLimit: params.passLimit }), + capture: async () => params.captures[Math.min(index++, params.captures.length - 1)]!, + scroll: async () => { + params.onScroll?.(); + return { pixels: 480 }; + }, + }); +} + +test('an already visible target costs one capture and no gesture', async () => { + let scrolls = 0; + const result = await run({ captures: [{ nodes: tree(200) }], onScroll: () => (scrolls += 1) }); + assert.equal(result.passes, 0); + assert.equal(scrolls, 0); + assert.equal(result.result, undefined); +}); + +test('passes repeat until the selector is on screen, and the last gesture is reported', async () => { + let scrolls = 0; + const result = await run({ + captures: [{ nodes: tree(2400) }, { nodes: tree(1600) }, { nodes: tree(200) }], + onScroll: () => (scrolls += 1), + }); + assert.equal(result.passes, 2); + assert.equal(scrolls, 2); + assert.deepEqual(result.result, { pixels: 480 }); +}); + +/** + * A target below the fold is present but not visible. Stopping on presence would leave the caller + * with a row it cannot act on, which is the whole reason the check asks about the viewport. + */ +test('a present but scrolled-out target does not end the loop', async () => { + await assert.rejects( + () => run({ captures: [{ nodes: tree(2400) }], passLimit: 1 }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_pass_limit'); + return true; + }, + ); +}); + +test('running out of content stops before the pass budget does', async () => { + let scrolls = 0; + await assert.rejects( + () => + run({ + // The row is on screen, so nothing is hidden below and the selector matches nothing. + captures: [{ nodes: tree(200, 'Other') }], + onScroll: () => (scrolls += 1), + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_edge_reached'); + assert.match(String(error.details?.hint), /scroll the opposite direction/); + return true; + }, + ); + assert.equal(scrolls, 0); +}); + +test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { + let scrolls = 0; + await assert.rejects( + () => + runScrollUntilVisible({ + selector: 'label=Missing', + direction: 'right', + platform: 'ios', + passLimit: 3, + capture: async () => ({ nodes: tree(200) }), + scroll: async () => { + scrolls += 1; + return {}; + }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_pass_limit'); + assert.equal(error.details?.passes, 3); + return true; + }, + ); + assert.equal(scrolls, 3); +}); + +test('the default budget is the shared constant', async () => { + await assert.rejects( + () => + runScrollUntilVisible({ + selector: 'label=Missing', + direction: 'right', + platform: 'ios', + capture: async () => ({ nodes: tree(200) }), + scroll: async () => ({}), + }), + (error: unknown) => + error instanceof AppError && error.details?.passes === SCROLL_UNTIL_PASS_LIMIT, + ); +}); + +/** + * The defect this pins: coercing an unreadable capture to an empty tree makes the edge analyzer + * report "no room below", so a failed read used to be reported as end-of-content. Each case counts + * gestures, so the refusal is proven to land before matching, edge analysis or scrolling. + */ +test('an unreadable capture is refused rather than read as end-of-content', async () => { + for (const capture of [{}, { nodes: [] }] satisfies ScrollUntilCapture[]) { + let scrolls = 0; + await assert.rejects( + () => run({ captures: [capture], onScroll: () => (scrolls += 1) }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'no-capture'); + return true; + }, + ); + assert.equal(scrolls, 0); + } +}); + +/** + * The verdict arrives under two spellings: `SnapshotState` says `snapshotQuality`, a backend result + * says `quality` and can nest a state as well. Reading only one is how a real backend sparse verdict + * went unread once. Every arrangement carries content below the fold, so an edge verdict would be + * wrong here too. + */ +test('a sparse verdict is refused under every spelling a capture can carry it in', async () => { + const arrangements: ScrollUntilCapture[] = [ + { nodes: tree(2400), snapshotQuality: SPARSE }, + { nodes: tree(2400), quality: SPARSE }, + { quality: SPARSE, snapshot: { nodes: tree(2400) } }, + { snapshot: { nodes: tree(2400), snapshotQuality: SPARSE } }, + ]; + for (const capture of arrangements) { + let scrolls = 0; + await assert.rejects( + () => run({ captures: [capture], onScroll: () => (scrolls += 1) }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(String(error.message), /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); + } +}); + +test('the legacy iOS application-root-only shape is refused', async () => { + await assert.rejects( + () => + run({ + captures: [ + { + backend: 'xctest', + nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], + }, + ], + }), + (error: unknown) => + error instanceof AppError && error.details?.captureRefusal === 'sparse-tree', + ); +}); + +test('a malformed quality payload is not mistaken for a verdict', async () => { + const result = await run({ captures: [{ nodes: tree(200), quality: { state: 'not-a-state' } }] }); + assert.equal(result.passes, 0); +}); + +/** + * A tree the backend vouches for is readable, and so is one whose tail was truncated: truncation + * drops content, it does not make the capture untrustworthy. + */ +test('a populated capture is not refused, healthy or recovered', async () => { + for (const state of ['healthy', 'recovered'] as const) { + const result = await run({ + captures: [{ nodes: tree(200), snapshotQuality: { state, backend: 'tree' } }], + }); + assert.equal(result.passes, 0); + } +}); + +test('the success message distinguishes an already visible target from a scrolled one', () => { + assert.equal( + formatScrollUntilMessage('down', 'id=email', 0), + 'id=email was already visible; no down scroll needed', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 1), + 'Scrolled down 1 pass until id=email was visible', + ); + assert.equal( + formatScrollUntilMessage('down', 'id=email', 3), + 'Scrolled down 3 passes until id=email was visible', + ); +}); diff --git a/src/daemon/scroll-until.ts b/src/daemon/scroll-until.ts new file mode 100644 index 0000000000..84ef79c1a6 --- /dev/null +++ b/src/daemon/scroll-until.ts @@ -0,0 +1,258 @@ +import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; +import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; +import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; +import type { + RawSnapshotNode, + SnapshotNode, + SnapshotQualityVerdict, + SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { isLegacySparseIosInteractiveSnapshot } from '@agent-device/selectors/absence-observation'; +import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; +import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; +import { + canScrollFurtherAtEdge, + type ScrollEdge, +} from '@agent-device/capture-kit/scroll-edge-state'; + +/** + * Everything `scroll --until ` needs beyond the ordinary scroll: when a pass has arrived, + * when the capture cannot answer that at all, and how the two failures read. + * + * One module beside the route that runs it. `scroll` reaches a device in exactly one place (ADR + * 0019, `scroll-runtime.ts`), so there is no second caller to keep in agreement and no reason for + * this to be a package surface. + */ + +/** + * How many gestures one `scroll --until` may spend before it gives up. A pass costs a capture plus + * a gesture, so this is the request's whole cost ceiling, not a retry budget: 12 passes at the + * honored 0.8-viewport maximum cover roughly ten screens, which is past the point where a list is + * better reached by `scroll bottom` or a search field. + */ +export const SCROLL_UNTIL_PASS_LIMIT = 12; + +type CapturedNodes = readonly (RawSnapshotNode | SnapshotNode)[]; + +/** + * The capture shape this route receives. The verdict is read under both spellings a capture can + * carry it in — `SnapshotState` says `snapshotQuality`, a backend result says `quality` and may + * nest a state as well — because normalizing at the call site is what let a real sparse verdict go + * unread once already. + */ +export type ScrollUntilCapture = { + nodes?: CapturedNodes | undefined; + backend?: string | undefined; + snapshotQuality?: SnapshotQualityVerdict | undefined; + quality?: unknown; + snapshot?: { + nodes?: CapturedNodes | undefined; + backend?: string | undefined; + snapshotQuality?: SnapshotQualityVerdict | undefined; + }; +}; + +/** Why a capture cannot answer the `--until` question at all. Never an outcome about the content. */ +export type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; + +export type ScrollUntilOutcome = 'matched' | 'edge-reached' | 'pass-limit'; + +/** + * Scrolls until the selector matches a node that is on screen. + * + * Each pass reads the tree once and that read answers three questions in order: is the capture + * usable, has the target arrived, and is there anywhere left to go. Refusing an unusable capture + * first is what keeps a failed read from being reported as end-of-content — coercing it to an empty + * tree makes the edge analyzer say "no room below". + * + * The end-of-content signal is the one `scroll top`/`scroll bottom` already trust, so both stop in + * the same place. Horizontal scrolls have no such analyzer and are bounded by the pass budget alone. + * The first capture happens before the first gesture, so an already-visible target costs no scroll. + */ +export async function runScrollUntilVisible(params: { + selector: string; + direction: ScrollDirection; + platform: Platform | PublicPlatform; + passLimit?: number; + capture: () => Promise; + scroll: () => Promise; +}): Promise<{ passes: number; result?: TResult }> { + const { selector, direction, platform, capture, scroll } = params; + const passLimit = params.passLimit ?? SCROLL_UNTIL_PASS_LIMIT; + const edge = verticalEdgeFor(direction); + let passes = 0; + let result: TResult | undefined; + + while (true) { + const canonical = canonicalCapture(await capture()); + const refusal = captureRefusal(canonical.nodes, canonical.quality, canonical.backend); + if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); + const nodes = canonical.nodes ?? []; + if (await isSelectorVisible(nodes, selector, platform)) { + return { passes, ...(result === undefined ? {} : { result }) }; + } + if (edge && !(await canScrollFurtherAtEdge(nodes, edge))) { + throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); + } + if (passes >= passLimit) { + throw scrollUntilNotFoundError(direction, selector, 'pass-limit', passes); + } + result = await scroll(); + passes += 1; + } +} + +export function formatScrollUntilMessage( + direction: ScrollDirection, + selector: string, + passes: number, +): string { + if (passes === 0) return `${selector} was already visible; no ${direction} scroll needed`; + return `Scrolled ${direction} ${passes} ${passes === 1 ? 'pass' : 'passes'} until ${selector} was visible`; +} + +/** + * Does this selector match a node that is on screen right now? + * + * Two questions, not one: the `wait` pipeline row answers presence and ignores off-screen, then + * `isVisibleOnScreen` answers the part `--until` cares about. Reusing the presence row unchanged is + * what keeps a present-but-scrolled-out target from ending the loop early. SOME match, not the + * first: a list whose rows share a selector can hold an off-screen twin above the fold. + */ +async function isSelectorVisible( + nodes: CapturedNodes, + selector: string, + platform: Platform | PublicPlatform, +): Promise { + const tree = nodes as SnapshotNode[]; + const outcome = await resolveSelectorPipeline(SELECTOR_PIPELINE_POLICIES.wait, tree, selector, { + platform, + }); + const matched = + outcome.kind === 'target' || outcome.kind === 'ambiguous' + ? outcome.matchedNodes + : outcome.kind === 'occluded' + ? [outcome.node] + : []; + if (matched.length === 0) return false; + const visibility = createSnapshotVisibility(tree); + return matched.some((node) => visibility.isVisibleOnScreen(node)); +} + +/** + * Sparseness reuses the signals absence assertions already trust rather than a second definition of + * readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its + * tail missing, and refusing it would fail large screens where the target is plainly in view. + */ +function captureRefusal( + nodes: CapturedNodes | undefined, + quality: SnapshotQualityVerdict | undefined, + backend: string | undefined, +): ScrollUntilCaptureRefusal | undefined { + if (nodes === undefined) { + return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; + } + if (nodes.length === 0) { + return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; + } + if (quality?.state === 'sparse') { + return { + reason: 'sparse-tree', + detail: quality.reason ?? 'the capture backend reported a sparse tree', + }; + } + if ( + isLegacySparseIosInteractiveSnapshot({ + backend: backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(quality ? { snapshotQuality: quality } : {}), + }) + ) { + return { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }; + } + return undefined; +} + +/** The nested state wins on nodes and backend; the verdict comes from whichever level carries one. */ +function canonicalCapture(capture: ScrollUntilCapture): { + nodes: CapturedNodes | undefined; + backend: string | undefined; + quality: SnapshotQualityVerdict | undefined; +} { + const nested = capture.snapshot; + return { + nodes: nested?.nodes ?? capture.nodes, + backend: nested?.backend ?? capture.backend, + quality: + nested?.snapshotQuality ?? + capture.snapshotQuality ?? + readSnapshotQualityVerdict(capture.quality), + }; +} + +/** The content ran out, or the budget did. Separate messages: the corrective action differs. */ +function scrollUntilNotFoundError( + direction: ScrollDirection, + selector: string, + outcome: Exclude, + passes: number, +): AppError { + const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`; + if (outcome === 'edge-reached') { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} reached the end of the scrollable content after ${spent} without ${selector} becoming visible`, + { + reason: 'scroll_until_edge_reached', + selector, + direction, + passes, + hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, + }, + ); + } + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, + { + reason: 'scroll_until_pass_limit', + selector, + direction, + passes, + hint: `Raise the step with an amount (scroll ${direction} 0.8 --until ), or run snapshot -i to confirm the selector matches something on this screen.`, + }, + ); +} + +function scrollUntilCaptureError( + direction: ScrollDirection, + selector: string, + refusal: ScrollUntilCaptureRefusal, +): AppError { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} --until ${selector} could not read the screen: ${refusal.detail}`, + { + reason: 'scroll_until_capture_unreadable', + selector, + direction, + captureRefusal: refusal.reason, + hint: + refusal.reason === 'no-capture' + ? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.' + : 'The accessibility tree came back sparse, so its refs and selectors are not trustworthy. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.', + }, + ); +} + +/** + * The end-of-content analyzer only reads vertical edges, so a horizontal `--until` is bounded by its + * pass budget alone rather than by a signal that would always report "no room". + */ +function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined { + if (direction === 'down') return 'bottom'; + if (direction === 'up') return 'top'; + return undefined; +} diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index 32c32e0570..c3fb3edfb0 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { SCROLL_UNTIL_PASS_LIMIT } from '@agent-device/capture-kit/scroll-until-visible'; +import { SCROLL_UNTIL_PASS_LIMIT } from '../../../src/daemon/scroll-until.ts'; import { createAndroidSettingsWorld } from './android-world.ts'; import { withProviderScenarioResource } from './harness.ts'; From b4ed2dc76c3e8afeab3c3eeaa97b90e688bb0b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 12:00:43 +0200 Subject: [PATCH 10/13] refactor(scroll): drop unexported until types and duplicated guidance prose --- src/cli-schema/cli-help.ts | 2 +- src/commands/interaction/metadata.ts | 2 +- src/daemon/scroll-until.ts | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index 09b44a5b24..1ff39fe8fe 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -257,7 +257,7 @@ Shapes: agent-device gesture pinch 0.5 200 400 agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops -- one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out before the target appears, and is not accepted on the top/bottom directions, which already carry their own stop condition. One gesture never travels more than 0.8 of the viewport axis, so crossing several screens is what --until and scroll top/bottom are for. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 7592733bba..2e78dd567b 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to keep scrolling until that element is on screen, which finds an off-screen target in one command instead of a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', diff --git a/src/daemon/scroll-until.ts b/src/daemon/scroll-until.ts index 84ef79c1a6..dd0a3e14a0 100644 --- a/src/daemon/scroll-until.ts +++ b/src/daemon/scroll-until.ts @@ -55,9 +55,7 @@ export type ScrollUntilCapture = { }; /** Why a capture cannot answer the `--until` question at all. Never an outcome about the content. */ -export type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; - -export type ScrollUntilOutcome = 'matched' | 'edge-reached' | 'pass-limit'; +type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; /** * Scrolls until the selector matches a node that is on screen. @@ -196,7 +194,7 @@ function canonicalCapture(capture: ScrollUntilCapture): { function scrollUntilNotFoundError( direction: ScrollDirection, selector: string, - outcome: Exclude, + outcome: 'edge-reached' | 'pass-limit', passes: number, ): AppError { const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`; From dc8ecaed05b1ac0591dd368fd80fa42716e37fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 14:22:29 +0200 Subject: [PATCH 11/13] test(scroll): fix the climbing fixture and drop duplicated route-level cases --- src/daemon/__tests__/scroll-runtime.test.ts | 164 ++---------------- .../provider-scenarios/scroll-until.test.ts | 30 ++-- 2 files changed, 27 insertions(+), 167 deletions(-) diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index c3df04ec59..575b8a877e 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -361,7 +361,12 @@ function untilNodes(targetY: number, hiddenBelow: boolean) { ]; } -test('bound scroll --until stops on the pass whose capture shows the selector on screen', async () => { +/** + * Route-level only: the executor's result envelope, the parse rejection, and admission. The loop's + * own behavior — arrival, end-of-content, the pass budget, and every capture refusal — is covered + * against the module in `scroll-until.test.ts` rather than duplicated through this harness. + */ +test('bound scroll --until reports the passes it spent and the selector it stopped on', async () => { const scrolls: string[] = []; const frames = [untilNodes(2400, true), untilNodes(1200, true), untilNodes(300, true)]; const result = await runScroll( @@ -377,118 +382,12 @@ test('bound scroll --until stops on the pass whose capture shows the selector on ); assert.equal(result.until, 'label=Email'); + assert.equal(result.direction, 'down'); assert.equal(result.passes, 2); - assert.equal(scrolls.length, 2); + assert.deepEqual(scrolls, ['down', 'down']); assert.match(String(result.message), /Scrolled down 2 passes until label=Email was visible/); }); -test('bound scroll --until reports the end of the content rather than spending its budget', async () => { - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Missing' }, - { - captureSnapshot: async () => ({ nodes: untilNodes(300, false) }), - scroll: async () => ({}), - }, - ), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_edge_reached'); - return true; - }, - ); -}); - -test('bound scroll rejects --until on an edge direction before any device work', async () => { - await assert.rejects( - () => - runScroll( - ['bottom'], - { until: 'label=Email' }, - { - captureSnapshot: async () => ({ nodes: untilNodes(300, true) }), - scroll: async () => { - throw new Error('scroll should be rejected before the backend call'); - }, - }, - ), - /scroll bottom already scrolls to the bottom edge and cannot take --until/, - ); -}); - -test('bound scroll --until is refused when the owner advertises no capture', async () => { - const resolved = await resolveBoundScrollRuntime({ - device: IOS_SIMULATOR, - positionals: ['down'], - context: { until: 'label=Email' } as DaemonCommandContext, - ...bindings({ scroll: async () => ({}) }), - }); - assert.equal(resolved.ok, false); -}); - -/** Same defect as the command runtime's: a failed read is not evidence that the content ran out. */ -/** - * The owner's capture result spells the verdict `quality`, which is the shape this route actually - * receives — an earlier version of this test asserted through `snapshotQuality` and passed while - * the real field went unread. The scroll spy proves each refusal lands before any gesture, and the - * sparse tree deliberately has content below the fold, so an edge verdict would be wrong there too. - */ -test('bound scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { - let scrolls = 0; - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Email' }, - { - captureSnapshot: async () => ({}), - scroll: async () => { - scrolls += 1; - return {}; - }, - }, - ), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); - assert.equal(error.details?.captureRefusal, 'no-capture'); - return true; - }, - ); - assert.equal(scrolls, 0); -}); - -test('bound scroll --until refuses a sparse capture before matching, edge analysis or scrolling', async () => { - let scrolls = 0; - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Email' }, - { - captureSnapshot: async () => ({ - nodes: untilNodes(2400, true), - quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, - }), - scroll: async () => { - scrolls += 1; - return {}; - }, - }, - ), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); - assert.equal(error.details?.captureRefusal, 'sparse-tree'); - assert.match(String(error.message), /AX bridge unavailable/); - return true; - }, - ); - assert.equal(scrolls, 0); -}); - test('bound scroll rejects --until on an edge direction before any device work', async () => { await assert.rejects( () => @@ -502,11 +401,11 @@ test('bound scroll rejects --until on an edge direction before any device work', }, }, ), - /scroll bottom already scrolls to the bottom edge and cannot take --until/, + /cannot take --until/, ); }); -test('bound scroll --until is refused when the owner advertises no capture', async () => { +test('bound scroll --until is refused at admission when the owner declares no capture', async () => { const resolved = await resolveBoundScrollRuntime({ device: IOS_SIMULATOR, positionals: ['down'], @@ -515,46 +414,3 @@ test('bound scroll --until is refused when the owner advertises no capture', asy }); assert.equal(resolved.ok, false); }); - -/** Same defect as the command runtime's: a failed read is not evidence that the content ran out. */ -test('bound scroll --until reports an unreadable capture as a capture failure, not end-of-content', async () => { - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Email' }, - { - captureSnapshot: async () => ({}), - scroll: async () => ({}), - }, - ), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); - assert.equal(error.details?.captureRefusal, 'no-capture'); - return true; - }, - ); -}); - -test('bound scroll --until refuses a sparse capture rather than trusting its selectors', async () => { - await assert.rejects( - () => - runScroll( - ['down'], - { until: 'label=Email' }, - { - captureSnapshot: async () => ({ - nodes: untilNodes(2400, true), - snapshotQuality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' }, - }), - scroll: async () => ({}), - }, - ), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.captureRefusal, 'sparse-tree'); - return true; - }, - ); -}); diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index c3fb3edfb0..b951518c6a 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -13,6 +13,9 @@ import { withProviderScenarioResource } from './harness.ts'; * rather than on injected gestures because the Android gesture path runs through the persistent * helper, not an adb shell command the world can count. */ +/** Two screens below the fold, climbing one screen per capture: visible on the third capture. */ +const ARRIVAL_PASSES = 2; + function climbingRow(): () => number { let captures = 0; return () => { @@ -23,16 +26,20 @@ function climbingRow(): () => number { } function climbingHierarchy(targetTop: () => number): () => string { - return () => - [ + return () => { + // Read the stateful position ONCE: calling it per bound advanced the row twice per capture and + // produced an inverted rectangle on the first one. + const top = targetTop(); + return [ '', '', ' ', ' ', - ` `, + ` `, ' ', '', ].join('\n'); + }; } test('Provider-backed integration scroll --until stops on the capture that brings the target on screen', async () => { @@ -48,18 +55,15 @@ test('Provider-backed integration scroll --until stops on the capture that bring ...world.selection, }); - const passes = typeof result.passes === 'number' ? result.passes : -1; assert.equal(result.until, 'text=Terms'); assert.equal(result.direction, 'down'); - assert.ok( - passes >= 1, - `expected at least one pass to reach the off-screen row, saw ${passes}`, - ); - assert.match(String(result.message), /until text=Terms was visible/); - // Stopped on arrival rather than running the budget out. - assert.ok( - passes < SCROLL_UNTIL_PASS_LIMIT, - `expected the loop to stop on arrival, spent ${passes} passes`, + // An exact count is what proves repeated scrolling on valid geometry, rather than a lucky + // first capture or a budget burned to the limit. + assert.equal(result.passes, ARRIVAL_PASSES); + assert.ok(ARRIVAL_PASSES < SCROLL_UNTIL_PASS_LIMIT); + assert.match( + String(result.message), + new RegExp(`Scrolled down ${ARRIVAL_PASSES} passes until text=Terms was visible`), ); }, ); From 449bc59430fadf13e19580bd1a046d20e40d3794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 16:42:06 +0200 Subject: [PATCH 12/13] refactor(scroll): delete the dead command-runtime executor and reuse canonical predicates --- ...roll-edge-state-pass-orchestration.test.ts | 31 +- .../src/snapshot/scroll-edge-state.ts | 19 +- packages/contracts/src/scroll-command.ts | 11 +- packages/platform-linux/src/input-actions.ts | 8 +- packages/selectors/package.json | 2 +- packages/selectors/src/absence-observation.ts | 9 +- src/__tests__/runtime-public.test.ts | 1 - src/commands/interaction/interactions.ts | 2 +- src/commands/interaction/runtime/index.ts | 7 - .../interaction/runtime/interactions.ts | 2 - .../interaction/runtime/scroll.test.ts | 238 -------------- src/commands/interaction/runtime/scroll.ts | 310 ------------------ src/daemon/__tests__/scroll-runtime.test.ts | 2 +- src/daemon/scroll-runtime.ts | 63 ++-- src/daemon/scroll-until.test.ts | 82 +++-- src/daemon/scroll-until.ts | 111 ++----- 16 files changed, 155 insertions(+), 743 deletions(-) delete mode 100644 src/commands/interaction/runtime/scroll.test.ts delete mode 100644 src/commands/interaction/runtime/scroll.ts diff --git a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts index f005969301..53cd7d612d 100644 --- a/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts @@ -14,45 +14,48 @@ import { captureThrows, scrollSnapshot, windowRoot } from './scroll-edge-state-f test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (bottom)', () => { assert.equal( - formatScrollEdgeMessage('down', 'bottom', 0, undefined, undefined), + formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 0 }), 'Already at bottom; no hidden content below detected', ); }); test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (top)', () => { assert.equal( - formatScrollEdgeMessage('up', 'top', 0, undefined, undefined), + formatScrollEdgeMessage({ direction: 'up', edge: 'top', passes: 0 }), 'Already at top; no hidden content above detected', ); }); test('formatScrollEdgeMessage: edge reached after N passes', () => { assert.equal( - formatScrollEdgeMessage('down', 'bottom', 4, undefined, undefined), + formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 4 }), 'Scrolled to bottom with 4 down passes', ); }); test('formatScrollEdgeMessage: no edge, pixel amount given', () => { assert.equal( - formatScrollEdgeMessage('down', undefined, 0, undefined, 250), + formatScrollEdgeMessage({ direction: 'down', passes: 0, pixels: 250 }), 'Scrolled down by 250px', ); }); test('formatScrollEdgeMessage: no edge, no pixels, symbolic amount given', () => { - assert.equal(formatScrollEdgeMessage('up', undefined, 0, 3, undefined), 'Scrolled up by 3'); + assert.equal( + formatScrollEdgeMessage({ direction: 'up', passes: 0, amount: 3 }), + 'Scrolled up by 3', + ); }); test('formatScrollEdgeMessage: no edge, no pixels, no amount falls back to bare direction', () => { - assert.equal( - formatScrollEdgeMessage('left', undefined, 0, undefined, undefined), - 'Scrolled left', - ); + assert.equal(formatScrollEdgeMessage({ direction: 'left', passes: 0 }), 'Scrolled left'); }); test('formatScrollEdgeMessage: pixels takes priority over amount when both are set', () => { - assert.equal(formatScrollEdgeMessage('down', undefined, 0, 3, 250), 'Scrolled down by 250px'); + assert.equal( + formatScrollEdgeMessage({ direction: 'down', passes: 0, amount: 3, pixels: 250 }), + 'Scrolled down by 250px', + ); }); /** @@ -61,15 +64,15 @@ test('formatScrollEdgeMessage: pixels takes priority over amount when both are s */ test('an amount-based message names the honored travel when the planner reports it', () => { assert.equal( - formatScrollEdgeMessage('down', undefined, 1, 3, undefined, 640), + formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 3, honoredPixels: 640 }), 'Scrolled down by 3 of the viewport (640px)', ); assert.equal( - formatScrollEdgeMessage('down', undefined, 1, 0.65, undefined, undefined), + formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 0.65 }), 'Scrolled down by 0.65', ); assert.equal( - formatScrollEdgeMessage('down', undefined, 1, undefined, 5000, 640), + formatScrollEdgeMessage({ direction: 'down', passes: 1, pixels: 5000, honoredPixels: 640 }), 'Scrolled down by 640px', ); }); @@ -298,7 +301,7 @@ test('runScrollEdgePasses: throws a COMMAND_FAILED AppError once the pass limit 'scroll bottom reached the safety limit before the snapshot showed the edge', ); assert.deepEqual(error.details, { - hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.', + hint: 'The scoped scroll container still reports hidden content. Run scroll --until to stop on the element you are after, or snapshot -i to inspect the current state.', }); return true; }, diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index 9029ff9378..98c8c6e971 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -68,7 +68,7 @@ export async function runScrollEdgePasses(params: { 'COMMAND_FAILED', `scroll ${edge} reached the safety limit before the snapshot showed the edge`, { - hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.', + hint: 'The scoped scroll container still reports hidden content. Run scroll --until to stop on the element you are after, or snapshot -i to inspect the current state.', }, ); } @@ -87,14 +87,15 @@ export async function runScrollEdgePasses(params: { * padding, so a large `amount` saturates. Naming the honored distance is what keeps * `scroll down 3` from reporting a three-viewport scroll it never performed. */ -export function formatScrollEdgeMessage( - direction: ScrollDirection, - edge: ScrollEdge | undefined, - passes: number, - amount: number | undefined, - pixels: number | undefined, - honoredPixels?: number, -): string { +export function formatScrollEdgeMessage(params: { + direction: ScrollDirection; + edge?: ScrollEdge | undefined; + passes: number; + amount?: number | undefined; + pixels?: number | undefined; + honoredPixels?: number | undefined; +}): string { + const { direction, edge, passes, amount, pixels, honoredPixels } = params; if (edge && passes === 0) { return `Already at ${edge}; no hidden content ${edge === 'bottom' ? 'below' : 'above'} detected`; } diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index 50db871878..73b27385bb 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -82,6 +82,13 @@ export function normalizeScrollDurationMs( return durationMs; } +/** The travel the planner produced, which saturates below a large requested amount. */ +export function honoredScrollPixels( + result: Record | undefined, +): number | undefined { + return typeof result?.pixels === 'number' ? result.pixels : undefined; +} + export function honoredScrollDurationMs( result: Record | undefined, ): number | undefined { @@ -102,7 +109,9 @@ export type ScrollCommandResult = { direction: ScrollDirection; /** Set for `top`/`bottom` requests: the extreme being scrolled to. */ edge?: 'top' | 'bottom'; - /** Edge scrolls only: how many scroll-and-check passes ran. */ + /** Set for `--until` requests: the selector the passes stopped on. */ + until?: string; + /** Edge and until scrolls only: how many scroll-and-check passes ran. */ passes?: number; amount?: number; pixels?: number; diff --git a/packages/platform-linux/src/input-actions.ts b/packages/platform-linux/src/input-actions.ts index 1fb14402d4..8fe33c4b35 100644 --- a/packages/platform-linux/src/input-actions.ts +++ b/packages/platform-linux/src/input-actions.ts @@ -2,6 +2,7 @@ import { ensureInputTool } from './linux-env.ts'; import { resolveLinuxToolProvider, type LinuxPointerButton } from './tool-provider.ts'; import { sleep } from '@agent-device/host-kit/retry'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { DEFAULT_SCROLL_AMOUNT } from '@agent-device/contracts/scroll-gesture'; // ── Low-level wrappers ───────────────────────────────────────────────── @@ -227,8 +228,11 @@ export async function scrollLinux( ? Math.max(1, Math.round(options.pixels / 15)) : Math.max(1, Math.round(options.pixels / 40)); } else if (options?.amount != null) { - // amount is a fraction (0–1+) of the viewport; scale relative to default - scrollCount = Math.max(1, Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / 0.6))); + // amount is a fraction (0–1+) of the viewport; scale relative to the shared default + scrollCount = Math.max( + 1, + Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / DEFAULT_SCROLL_AMOUNT)), + ); } // xdotool: button 4=up, 5=down, 6=left, 7=right diff --git a/packages/selectors/package.json b/packages/selectors/package.json index 079b6eef24..edd45f37de 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -4,7 +4,7 @@ "private": true, "sideEffects": false, "type": "module", - "description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19); `./parameterized-recorded-fill` parameterizes recorded fill payloads against their selectors; the interaction-resolution subpaths (`./selector-pipeline`, `./interaction-targeting`, `./interaction-touch-point`, `./press-retarget`, `./absence-observation*`, \u2026) host the engine execution surface owned by the pipeline; `./snapshot-geometry-fixtures` is the canonical home for the geometry/touch-point test fixtures this package and root tests both build on (it re-exports `makeSnapshotState` from `@agent-device/capture-kit/snapshot-state-fixtures`, its own canonical home).", + "description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19); `./parameterized-recorded-fill` parameterizes recorded fill payloads against their selectors; the interaction-resolution subpaths (`./selector-pipeline`, `./interaction-targeting`, `./interaction-touch-point`, `./press-retarget`, `./absence-observation*`, …) host the engine execution surface owned by the pipeline; `./snapshot-geometry-fixtures` is the canonical home for the geometry/touch-point test fixtures this package and root tests both build on (it re-exports `makeSnapshotState` from `@agent-device/capture-kit/snapshot-state-fixtures`, its own canonical home).", "dependencies": { "@agent-device/ad-script": "workspace:*", "@agent-device/capture-kit": "workspace:*", diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 5f5839f582..ac2f811966 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -68,7 +68,7 @@ export function classifyAbsenceObservation( ...(firstMatch ? { firstMatch } : {}), }; } - const sparseQuality = sparseQualityForSnapshot(snapshot); + const sparseQuality = sparseCaptureQuality(snapshot); if (sparseQuality) { return { kind: 'sparse', @@ -86,7 +86,12 @@ export function classifyAbsenceObservation( return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } -function sparseQualityForSnapshot( +/** + * The one definition of "this capture is too sparse to trust": the backend's own verdict, then the + * legacy iOS shape that predates verdicts. Shared with `scroll --until`, which must not stop on a + * tree whose selectors are unreliable. + */ +export function sparseCaptureQuality( snapshot: Pick, ): SparseQuality | undefined { const quality = snapshot.snapshotQuality; diff --git a/src/__tests__/runtime-public.test.ts b/src/__tests__/runtime-public.test.ts index ae7942b5bc..a5fbc18ba6 100644 --- a/src/__tests__/runtime-public.test.ts +++ b/src/__tests__/runtime-public.test.ts @@ -251,7 +251,6 @@ test('internal backend, commands, and io modules are usable', () => { assert.equal(typeof commands.interactions.fill, 'function'); assert.equal(typeof commands.interactions.focus, 'function'); assert.equal(typeof commands.interactions.longPress, 'function'); - assert.equal(typeof commands.interactions.scroll, 'function'); assert.equal(typeof commands.interactions.gesture, 'function'); assert.equal(typeof commands.system.back, 'function'); assert.equal(typeof commands.system.home, 'function'); diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 9839b5ce87..8f15ff32f4 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -33,7 +33,7 @@ import { targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; -import type { ScrollInputDirection } from './runtime/scroll.ts'; +import type { ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; export const interactionCliReaders = { click: (positionals, flags) => ({ diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index dd50012a28..297d8b51d6 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -7,7 +7,6 @@ import { hoverCommand, longPressCommand, pressCommand, - scrollCommand, type ClickCommandOptions, type FillCommandOptions, type FillCommandResult, @@ -20,8 +19,6 @@ import { type LongPressCommandResult, type PressCommandOptions, type PressCommandResult, - type ScrollCommandOptions, - type ScrollCommandResult, } from './interactions.ts'; import { findCommand, @@ -79,7 +76,6 @@ export type InteractionCommands = { focus: RuntimeCommand; longPress: RuntimeCommand; hover: RuntimeCommand; - scroll: RuntimeCommand; gesture: RuntimeCommand; /** * #1638: the observation half of `--settle` for mutations that resolve no @@ -143,7 +139,6 @@ export type BoundInteractionCommands = { target: InteractionTarget, options?: Omit, ) => Promise; - scroll: BoundRuntimeCommand; gesture: BoundRuntimeCommand; settleObservation: BoundRuntimeCommand; }; @@ -167,7 +162,6 @@ export const interactionCommands: InteractionCommands = { focus: focusCommand, longPress: longPressCommand, hover: hoverCommand, - scroll: scrollCommand, gesture: gestureCommand, settleObservation: settleObservationCommand, }; @@ -198,7 +192,6 @@ export function bindInteractionCommands(runtime: AgentDeviceRuntime): BoundInter longPress: (target, options = {}) => interactionCommands.longPress(runtime, { ...options, target }), hover: (target, options = {}) => interactionCommands.hover(runtime, { ...options, target }), - scroll: (options) => interactionCommands.scroll(runtime, options), gesture: (options) => interactionCommands.gesture(runtime, options), settleObservation: (options) => interactionCommands.settleObservation(runtime, options), }; diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index c3829d6ed0..5f5c18afe9 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -35,8 +35,6 @@ export type { LongPressCommandOptions, LongPressCommandResult, } from './gestures.ts'; -export { scrollCommand } from './scroll.ts'; -export type { ScrollCommandOptions, ScrollCommandResult } from './scroll.ts'; export type { InteractionTarget } from './resolution.ts'; export type PressCommandOptions = CommandContext & diff --git a/src/commands/interaction/runtime/scroll.test.ts b/src/commands/interaction/runtime/scroll.test.ts deleted file mode 100644 index fefd017928..0000000000 --- a/src/commands/interaction/runtime/scroll.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { selector } from './selector-read-utils.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import { - createInteractionDevice, - runtimeScrollSnapshot, - selectorSnapshot, -} from './__tests__/test-utils/index.ts'; - -test('runtime scroll resolves selector targets before calling the backend primitive', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { scrolled: true }; - }, - }); - - const selectorResult = await device.interactions.scroll({ - session: 'default', - target: selector('label=Continue'), - direction: 'down', - pixels: 120, - durationMs: 50, - }); - const viewportResult = await device.interactions.scroll({ - direction: 'up', - amount: 0.5, - }); - - assert.equal(selectorResult.kind, 'selector'); - assert.equal(selectorResult.durationMs, undefined); - assert.equal(viewportResult.kind, 'viewport'); - assert.deepEqual(calls, [ - { - target: { kind: 'point', point: { x: 60, y: 40 } }, - options: { - direction: 'down', - pixels: 120, - durationMs: 50, - releaseBehavior: 'controlled', - }, - }, - { - target: { kind: 'viewport' }, - options: { direction: 'up', amount: 0.5, releaseBehavior: 'controlled' }, - }, - ]); -}); - -test('runtime scroll reports duration only when the backend honored it', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), - }); - - const result = await device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 50, - }); - - assert.equal(result.durationMs, 50); - assert.deepEqual(result.backendResult, { durationMs: 50 }); -}); - -test('runtime scroll rejects duration above the shared cap', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 10_001, - }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => { - throw new Error('snapshot unavailable'); - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - /Failed to verify scroll bottom state/, - ); - - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom does not scroll when no hidden content is below', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: unknown[] = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotScopes.push(options?.scope); - return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 1); - assert.equal(result.backendResult?.pass, 1); - assert.deepEqual(calls, [ - { - target: { kind: 'viewport' }, - options: { direction: 'down', releaseBehavior: 'inertial' }, - }, - ]); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - }), - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.passes, 2); - assert.equal(calls.length, 2); -}); - -test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotCount += 1; - if (options?.scope) throw new Error('scoped snapshot failed'); - return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; - }, - scroll: async () => ({}), - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -test('runtime viewport scroll rejects inspect-only macOS surfaces', async () => { - for (const surface of ['desktop', 'menubar'] as const) { - const device = createInteractionDevice(selectorSnapshot(), { - platform: 'macos', - sessionMetadata: { surface }, - scroll: async () => { - throw new Error(`${surface} scroll should be rejected before backend call`); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - target: { kind: 'viewport' }, - session: 'default', - }), - new RegExp(`scroll is not supported on macOS ${surface}`), - ); - } -}); diff --git a/src/commands/interaction/runtime/scroll.ts b/src/commands/interaction/runtime/scroll.ts deleted file mode 100644 index c1cf2a5bb2..0000000000 --- a/src/commands/interaction/runtime/scroll.ts +++ /dev/null @@ -1,310 +0,0 @@ -import { - assertExclusiveScrollDistanceInputs, - honoredScrollDurationMs, - normalizeScrollDurationMs, - resolveScrollExecutionOptions, -} from '@agent-device/contracts/scroll-command'; -import type { ScrollDirection, ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; -import { - captureScrollEdgeState, - formatScrollEdgeMessage, - runScrollEdgePasses, - type ScrollEdge, - type ScrollEdgeState, - type ScrollEdgeTarget, -} from '@agent-device/capture-kit/scroll-edge-state'; -import { AppError } from '@agent-device/kernel/errors'; -import { successText } from '@agent-device/kernel/success-text'; -import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; -import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import { toBackendContext } from '../../runtime-common.ts'; -import { - toBackendResult, - type BackendResultVariant, - type RuntimeCommand, -} from '../../runtime-types.ts'; -import { requireResolvedPoint } from './gestures.ts'; -import { - assertSupportedInteractionSurface, - resolveInteractionTarget, - type InteractionTarget, - type ResolvedInteractionTarget, -} from './resolution.ts'; - -export type GestureDirection = ScrollDirection; -// The input vocabulary lives in contracts/scroll-gesture.ts beside the other scroll vocabularies, -// so the public API can declare `ScrollOptions` without depending on this command runtime. -export { type ScrollInputDirection } from '@agent-device/contracts/scroll-gesture'; - -export type ScrollTarget = - | InteractionTarget - | { - kind: 'viewport'; - }; - -export type ScrollCommandOptions = CommandContext & { - target?: ScrollTarget; - direction: ScrollInputDirection; - amount?: number; - pixels?: number; - durationMs?: number; -}; - -export type ScrollCommandResult = - | BackendResultVariant<{ - kind: 'viewport'; - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - }> - | BackendResultVariant< - ResolvedInteractionTarget & { - direction: GestureDirection; - edge?: 'top' | 'bottom'; - passes?: number; - amount?: number; - pixels?: number; - durationMs?: number; - } - >; - -type ResolvedScrollTarget = { kind: 'viewport' } | ResolvedInteractionTarget; - -export const scrollCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => { - if (!runtime.backend.scroll) { - throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); - } - const target = resolveScrollDirection(options.direction); - const distance = normalizeScrollDistance(options, target.edge); - const resolved = await resolveScrollTarget(runtime, options); - const runScroll = bindScrollPass( - runtime, - options, - resolved, - target.direction, - distance.execution, - ); - - return await runDirectionOrEdgeScroll({ - runtime, - options, - resolved, - target, - distance, - scroll: runScroll, - }); -}; - -type NormalizedScrollDistance = { - /** What the caller asked for, echoed back on the result. */ - reported: { amount?: number; pixels?: number }; - execution: ReturnType; -}; - -/** Every distance/timing rejection, in one place, before any target resolution or device work. */ -function normalizeScrollDistance( - options: ScrollCommandOptions, - edge: ScrollEdge | undefined, -): NormalizedScrollDistance { - const amount = normalizeOptionalPositiveNumber(options.amount, 'scroll amount'); - const pixels = normalizeOptionalPositiveInteger(options.pixels, 'scroll pixels'); - const durationMs = normalizeScrollDurationMs(options.durationMs); - assertExclusiveScrollDistanceInputs( - { amount, pixels }, - 'scroll accepts either amount or pixels, not both', - ); - const reported = { - ...(amount !== undefined ? { amount } : {}), - ...(pixels !== undefined ? { pixels } : {}), - }; - return { - reported, - execution: resolveScrollExecutionOptions( - { ...reported, ...(durationMs !== undefined ? { durationMs } : {}) }, - edge, - ), - }; -} - -/** One pass, with its target and options already resolved: the unit every branch repeats. */ -function bindScrollPass( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, - resolved: ResolvedScrollTarget, - direction: GestureDirection, - execution: ReturnType, -): () => Promise>>> { - const scrollBackend = runtime.backend.scroll; - if (!scrollBackend) { - throw new AppError('UNSUPPORTED_OPERATION', 'scroll is not supported by this backend'); - } - const backendTarget = - resolved.kind === 'viewport' - ? { kind: 'viewport' as const } - : { kind: 'point' as const, point: requireResolvedPoint(resolved) }; - return async () => - await scrollBackend(toBackendContext(runtime, options), backendTarget, { - direction, - ...execution, - }); -} - -/** `scroll ` and `scroll top|bottom`: one pass, or passes until the edge stops moving. */ -async function runDirectionOrEdgeScroll(params: { - runtime: AgentDeviceRuntime; - options: ScrollCommandOptions; - resolved: ResolvedScrollTarget; - target: { direction: GestureDirection; edge?: ScrollEdge }; - distance: NormalizedScrollDistance; - scroll: () => Promise>>>; -}): Promise { - const { runtime, options, resolved, target, distance } = params; - const edge = target.edge; - const pass = edge - ? await runScrollEdgePasses({ - edge, - captureState: async (scope) => - await captureRuntimeScrollEdgeState( - runtime, - options, - edge, - buildScrollEdgeTarget(resolved), - scope, - ), - scroll: params.scroll, - }) - : { passes: 1, result: await params.scroll() }; - const backendResult = toBackendResult(pass.result); - const reportedDurationMs = honoredScrollDurationMs(backendResult); - return { - ...resolved, - direction: target.direction, - ...(edge ? { edge, passes: pass.passes } : {}), - ...distance.reported, - ...(reportedDurationMs !== undefined ? { durationMs: reportedDurationMs } : {}), - ...(backendResult ? { backendResult } : {}), - ...successText( - formatScrollEdgeMessage( - target.direction, - edge, - pass.passes, - distance.reported.amount, - distance.reported.pixels, - honoredScrollPixels(backendResult), - ), - ), - }; -} - -async function resolveScrollTarget( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, -): Promise { - const target = options.target ?? { kind: 'viewport' as const }; - if (target.kind === 'viewport') { - await assertSupportedInteractionSurface(runtime, options, 'scroll'); - return { kind: 'viewport' }; - } - return await resolveInteractionTarget( - runtime, - { ...options, target }, - { - action: 'scroll', - requireInteractive: false, - pipeline: SELECTOR_PIPELINE_POLICIES.resolvedTarget, - }, - ); -} -function resolveScrollDirection(direction: ScrollInputDirection): { - direction: GestureDirection; - edge?: 'top' | 'bottom'; -} { - if (direction === 'bottom') return { direction: 'down', edge: 'bottom' }; - if (direction === 'top') return { direction: 'up', edge: 'top' }; - return { direction: requireDirection(direction, 'scroll direction') }; -} -function buildScrollEdgeTarget(resolved: ResolvedScrollTarget): ScrollEdgeTarget { - return resolved.kind === 'viewport' - ? {} - : { - point: resolved.point, - nodeIndex: 'node' in resolved ? resolved.node?.index : undefined, - }; -} -async function captureRuntimeScrollEdgeState( - runtime: AgentDeviceRuntime, - options: ScrollCommandOptions, - edge: ScrollEdge, - target: ScrollEdgeTarget, - scope?: string, -): Promise { - if (!runtime.backend.captureSnapshot) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - ); - } - const { captureSnapshot } = runtime.backend; - return await captureScrollEdgeState({ - edge, - target, - scope, - captureNodes: async (snapshotScope) => { - const result = await captureSnapshot(toBackendContext(runtime, options), { - scope: snapshotScope, - }); - return result.snapshot?.nodes ?? result.nodes ?? []; - }, - }); -} - -/** The travel the planner produced, which saturates below a large requested amount. */ -function honoredScrollPixels(result: Record | undefined): number | undefined { - return typeof result?.pixels === 'number' ? result.pixels : undefined; -} - -function requireDirection( - direction: GestureDirection | undefined, - field: string, -): GestureDirection { - switch (direction) { - case 'up': - case 'down': - case 'left': - case 'right': - return direction; - default: - throw new AppError('INVALID_ARGS', `${field} must be up, down, left, or right`); - } -} - -function normalizeOptionalPositiveNumber( - value: number | undefined, - field: string, -): number | undefined { - return value === undefined ? undefined : normalizePositiveNumber(value, field); -} - -function normalizePositiveNumber(value: number, field: string): number { - if (!Number.isFinite(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive number`); - } - return value; -} - -function normalizeOptionalPositiveInteger( - value: number | undefined, - field: string, -): number | undefined { - if (value === undefined) return undefined; - if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) { - throw new AppError('INVALID_ARGS', `${field} must be a positive integer`); - } - return value; -} diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts index 575b8a877e..631d04d4d6 100644 --- a/src/daemon/__tests__/scroll-runtime.test.ts +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -339,7 +339,7 @@ test('the edge plan proves its capture statically and the direction plan cannot expectTypeOf>().toEqualTypeOf<'scrollDirection'>(); }); -/** Same shape the command runtime's `--until` tests use: a row walked into the viewport. */ +/** A row walked into the viewport, so the executor's pass count is observable. */ function untilNodes(targetY: number, hiddenBelow: boolean) { return [ { diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts index 587550003d..6e52e556db 100644 --- a/src/daemon/scroll-runtime.ts +++ b/src/daemon/scroll-runtime.ts @@ -2,6 +2,7 @@ import { assertExclusiveScrollDistanceInputs, assertScrollUntilCompatible, honoredScrollDurationMs, + honoredScrollPixels, normalizeScrollDurationMs, resolveScrollExecutionOptions, type ResolvedScrollExecutionOptions, @@ -91,15 +92,13 @@ export async function resolveBoundScrollRuntime( assertScrollCommandInputs(amount, pixels, durationMs); const target = parseScrollTarget(directionInput); - assertScrollUntilCompatible({ - ...(target.edge ? { edge: target.edge } : {}), - ...(until === undefined ? {} : { until }), - }); - const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); - const plan = resolveScrollRuntimePlan({ + const stopCondition = { ...(target.edge === undefined ? {} : { edge: target.edge }), ...(until === undefined ? {} : { until }), - }); + }; + assertScrollUntilCompatible(stopCondition); + const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); + const plan = resolveScrollRuntimePlan(stopCondition); const admission = { command: 'scroll', device: params.device, @@ -120,7 +119,11 @@ export async function resolveBoundScrollRuntime( ...admission, // The retired leaf refused an unsupported edge scroll by naming what the edge needs, so // the capture requirement keeps saying so rather than collapsing into "not supported". - unavailableResponse: (unavailable) => scrollEdgeUnsupported(edge, unavailable.hint), + unavailableResponse: (unavailable) => + scrollCaptureUnsupported( + `scroll ${edge}, which verifies hidden content before scrolling,`, + unavailable.hint, + ), use: plan.use, }, async (runtime, dispatchContext) => @@ -132,7 +135,11 @@ export async function resolveBoundScrollRuntime( return await resolveBoundGenericRuntime( { ...admission, - unavailableResponse: (unavailable) => scrollUntilUnsupported(unavailable.hint), + unavailableResponse: (unavailable) => + scrollCaptureUnsupported( + 'scroll --until, which checks whether the selector became visible,', + unavailable.hint, + ), use: plan.use, }, async (runtime, dispatchContext) => @@ -149,19 +156,14 @@ export async function resolveBoundScrollRuntime( } } -function scrollEdgeUnsupported(edge: ScrollEdge, hint: string | undefined) { - return errorResponse( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - undefined, - hint === undefined ? undefined : { hint }, - ); -} - -function scrollUntilUnsupported(hint: string | undefined) { +/** + * Both verifying tiers refuse the same way and differ only in what they would have checked, so the + * refusal names that rather than collapsing into "not supported" — the shape the retired leaf had. + */ +function scrollCaptureUnsupported(subject: string, hint: string | undefined) { return errorResponse( 'UNSUPPORTED_OPERATION', - 'scroll --until requires snapshot support to check whether the selector became visible', + `${subject} requires snapshot support`, undefined, hint === undefined ? undefined : { hint }, ); @@ -280,22 +282,17 @@ function scrollResult( ...(durationMs !== undefined ? { durationMs } : {}), ...interactionResult, }, - formatScrollEdgeMessage( - target.direction, - target.edge, - completedPasses, - options.amount, - options.pixels, - honoredScrollPixels(interactionResult), - ), + formatScrollEdgeMessage({ + direction: target.direction, + edge: target.edge, + passes: completedPasses, + amount: options.amount, + pixels: options.pixels, + honoredPixels: honoredScrollPixels(interactionResult), + }), ); } -/** The travel the planner produced, which saturates below a large requested amount. */ -function honoredScrollPixels(result: Record): number | undefined { - return typeof result.pixels === 'number' ? result.pixels : undefined; -} - /** The neutral intent one scroll carries, projected from a resolved command context. */ function scrollInput( direction: ScrollDirection, diff --git a/src/daemon/scroll-until.test.ts b/src/daemon/scroll-until.test.ts index 3a113819c0..057fad0afe 100644 --- a/src/daemon/scroll-until.test.ts +++ b/src/daemon/scroll-until.test.ts @@ -2,13 +2,18 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import { SCROLL_UNTIL_PASS_LIMIT, formatScrollUntilMessage, runScrollUntilVisible, - type ScrollUntilCapture, } from './scroll-until.ts'; +/** The provenance every `SnapshotResult` carries; the fields under test are the rest. */ +function capture(fields: Partial): SnapshotResult { + return { backend: 'xctest', producer: 'runner', ...fields } as SnapshotResult; +} + const VIEWPORT = { x: 0, y: 0, width: 400, height: 800 }; const SPARSE = { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' } as const; @@ -29,7 +34,7 @@ function tree(rowY: number, label = 'Email'): SnapshotNode[] { } async function run(params: { - captures: ScrollUntilCapture[]; + captures: SnapshotResult[]; selector?: string; passLimit?: number; onScroll?: () => void; @@ -50,7 +55,10 @@ async function run(params: { test('an already visible target costs one capture and no gesture', async () => { let scrolls = 0; - const result = await run({ captures: [{ nodes: tree(200) }], onScroll: () => (scrolls += 1) }); + const result = await run({ + captures: [capture({ nodes: tree(200) })], + onScroll: () => (scrolls += 1), + }); assert.equal(result.passes, 0); assert.equal(scrolls, 0); assert.equal(result.result, undefined); @@ -59,7 +67,7 @@ test('an already visible target costs one capture and no gesture', async () => { test('passes repeat until the selector is on screen, and the last gesture is reported', async () => { let scrolls = 0; const result = await run({ - captures: [{ nodes: tree(2400) }, { nodes: tree(1600) }, { nodes: tree(200) }], + captures: [tree(2400), tree(1600), tree(200)].map((nodes) => capture({ nodes })), onScroll: () => (scrolls += 1), }); assert.equal(result.passes, 2); @@ -73,7 +81,7 @@ test('passes repeat until the selector is on screen, and the last gesture is rep */ test('a present but scrolled-out target does not end the loop', async () => { await assert.rejects( - () => run({ captures: [{ nodes: tree(2400) }], passLimit: 1 }), + () => run({ captures: [capture({ nodes: tree(2400) })], passLimit: 1 }), (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_pass_limit'); @@ -88,7 +96,7 @@ test('running out of content stops before the pass budget does', async () => { () => run({ // The row is on screen, so nothing is hidden below and the selector matches nothing. - captures: [{ nodes: tree(200, 'Other') }], + captures: [capture({ nodes: tree(200, 'Other') })], onScroll: () => (scrolls += 1), }), (error: unknown) => { @@ -110,7 +118,7 @@ test('a horizontal scroll has no edge signal and is bounded by the budget alone' direction: 'right', platform: 'ios', passLimit: 3, - capture: async () => ({ nodes: tree(200) }), + capture: async () => capture({ nodes: tree(200) }), scroll: async () => { scrolls += 1; return {}; @@ -133,7 +141,7 @@ test('the default budget is the shared constant', async () => { selector: 'label=Missing', direction: 'right', platform: 'ios', - capture: async () => ({ nodes: tree(200) }), + capture: async () => capture({ nodes: tree(200) }), scroll: async () => ({}), }), (error: unknown) => @@ -147,10 +155,10 @@ test('the default budget is the shared constant', async () => { * gestures, so the refusal is proven to land before matching, edge analysis or scrolling. */ test('an unreadable capture is refused rather than read as end-of-content', async () => { - for (const capture of [{}, { nodes: [] }] satisfies ScrollUntilCapture[]) { + for (const frame of [capture({}), capture({ nodes: [] })]) { let scrolls = 0; await assert.rejects( - () => run({ captures: [capture], onScroll: () => (scrolls += 1) }), + () => run({ captures: [frame], onScroll: () => (scrolls += 1) }), (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); @@ -163,31 +171,26 @@ test('an unreadable capture is refused rather than read as end-of-content', asyn }); /** - * The verdict arrives under two spellings: `SnapshotState` says `snapshotQuality`, a backend result - * says `quality` and can nest a state as well. Reading only one is how a real backend sparse verdict - * went unread once. Every arrangement carries content below the fold, so an edge verdict would be - * wrong here too. + * A tree the backend calls sparse is one whose selectors are not trustworthy, so it cannot answer + * the question either way. It carries content below the fold, so an edge verdict would be wrong too. */ -test('a sparse verdict is refused under every spelling a capture can carry it in', async () => { - const arrangements: ScrollUntilCapture[] = [ - { nodes: tree(2400), snapshotQuality: SPARSE }, - { nodes: tree(2400), quality: SPARSE }, - { quality: SPARSE, snapshot: { nodes: tree(2400) } }, - { snapshot: { nodes: tree(2400), snapshotQuality: SPARSE } }, - ]; - for (const capture of arrangements) { - let scrolls = 0; - await assert.rejects( - () => run({ captures: [capture], onScroll: () => (scrolls += 1) }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.details?.captureRefusal, 'sparse-tree'); - assert.match(String(error.message), /AX bridge unavailable/); - return true; - }, - ); - assert.equal(scrolls, 0); - } +test('a sparse capture is refused before matching, edge analysis or scrolling', async () => { + let scrolls = 0; + await assert.rejects( + () => + run({ + captures: [capture({ nodes: tree(2400), quality: SPARSE })], + onScroll: () => (scrolls += 1), + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_capture_unreadable'); + assert.equal(error.details?.captureRefusal, 'sparse-tree'); + assert.match(String(error.message), /AX bridge unavailable/); + return true; + }, + ); + assert.equal(scrolls, 0); }); test('the legacy iOS application-root-only shape is refused', async () => { @@ -195,10 +198,10 @@ test('the legacy iOS application-root-only shape is refused', async () => { () => run({ captures: [ - { + capture({ backend: 'xctest', nodes: [{ index: 0, ref: 'e1', type: 'Application', rect: VIEWPORT } as SnapshotNode], - }, + }), ], }), (error: unknown) => @@ -206,11 +209,6 @@ test('the legacy iOS application-root-only shape is refused', async () => { ); }); -test('a malformed quality payload is not mistaken for a verdict', async () => { - const result = await run({ captures: [{ nodes: tree(200), quality: { state: 'not-a-state' } }] }); - assert.equal(result.passes, 0); -}); - /** * A tree the backend vouches for is readable, and so is one whose tail was truncated: truncation * drops content, it does not make the capture untrustworthy. @@ -218,7 +216,7 @@ test('a malformed quality payload is not mistaken for a verdict', async () => { test('a populated capture is not refused, healthy or recovered', async () => { for (const state of ['healthy', 'recovered'] as const) { const result = await run({ - captures: [{ nodes: tree(200), snapshotQuality: { state, backend: 'tree' } }], + captures: [capture({ nodes: tree(200), quality: { state, backend: 'tree' } })], }); assert.equal(result.passes, 0); } diff --git a/src/daemon/scroll-until.ts b/src/daemon/scroll-until.ts index dd0a3e14a0..0358734b7c 100644 --- a/src/daemon/scroll-until.ts +++ b/src/daemon/scroll-until.ts @@ -1,15 +1,10 @@ -import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; +import type { SnapshotResult } from '@agent-device/contracts/interactor-types'; import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture'; import { AppError } from '@agent-device/kernel/errors'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import type { - RawSnapshotNode, - SnapshotNode, - SnapshotQualityVerdict, - SnapshotState, -} from '@agent-device/kernel/snapshot'; -import { isLegacySparseIosInteractiveSnapshot } from '@agent-device/selectors/absence-observation'; +import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { evaluateIsPredicate } from '@agent-device/selectors'; +import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation'; import { resolveSelectorPipeline } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { @@ -34,26 +29,6 @@ import { */ export const SCROLL_UNTIL_PASS_LIMIT = 12; -type CapturedNodes = readonly (RawSnapshotNode | SnapshotNode)[]; - -/** - * The capture shape this route receives. The verdict is read under both spellings a capture can - * carry it in — `SnapshotState` says `snapshotQuality`, a backend result says `quality` and may - * nest a state as well — because normalizing at the call site is what let a real sparse verdict go - * unread once already. - */ -export type ScrollUntilCapture = { - nodes?: CapturedNodes | undefined; - backend?: string | undefined; - snapshotQuality?: SnapshotQualityVerdict | undefined; - quality?: unknown; - snapshot?: { - nodes?: CapturedNodes | undefined; - backend?: string | undefined; - snapshotQuality?: SnapshotQualityVerdict | undefined; - }; -}; - /** Why a capture cannot answer the `--until` question at all. Never an outcome about the content. */ type ScrollUntilCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string }; @@ -74,7 +49,7 @@ export async function runScrollUntilVisible(params: { direction: ScrollDirection; platform: Platform | PublicPlatform; passLimit?: number; - capture: () => Promise; + capture: () => Promise; scroll: () => Promise; }): Promise<{ passes: number; result?: TResult }> { const { selector, direction, platform, capture, scroll } = params; @@ -84,10 +59,10 @@ export async function runScrollUntilVisible(params: { let result: TResult | undefined; while (true) { - const canonical = canonicalCapture(await capture()); - const refusal = captureRefusal(canonical.nodes, canonical.quality, canonical.backend); + const captured = await capture(); + const refusal = captureRefusal(captured); if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); - const nodes = canonical.nodes ?? []; + const nodes = (captured.nodes ?? []) as SnapshotNode[]; if (await isSelectorVisible(nodes, selector, platform)) { return { passes, ...(result === undefined ? {} : { result }) }; } @@ -112,20 +87,21 @@ export function formatScrollUntilMessage( } /** - * Does this selector match a node that is on screen right now? + * Does this selector match a node that is visible right now? * * Two questions, not one: the `wait` pipeline row answers presence and ignores off-screen, then - * `isVisibleOnScreen` answers the part `--until` cares about. Reusing the presence row unchanged is - * what keeps a present-but-scrolled-out target from ending the loop early. SOME match, not the - * first: a list whose rows share a selector can hold an off-screen twin above the fold. + * `is visible`'s own predicate answers the rest. Borrowing that predicate rather than a narrower + * geometry check is what keeps `scroll --until X` from stopping on a node that `is visible X` would + * then reject — it carries the Android `visibleToUser` rule, non-positive rects, the hittable + * fallback and anchor resolution too. SOME match, not the first: a list whose rows share a selector + * can hold an off-screen twin above the fold. */ async function isSelectorVisible( - nodes: CapturedNodes, + nodes: SnapshotNode[], selector: string, platform: Platform | PublicPlatform, ): Promise { - const tree = nodes as SnapshotNode[]; - const outcome = await resolveSelectorPipeline(SELECTOR_PIPELINE_POLICIES.wait, tree, selector, { + const outcome = await resolveSelectorPipeline(SELECTOR_PIPELINE_POLICIES.wait, nodes, selector, { platform, }); const matched = @@ -134,62 +110,39 @@ async function isSelectorVisible( : outcome.kind === 'occluded' ? [outcome.node] : []; - if (matched.length === 0) return false; - const visibility = createSnapshotVisibility(tree); - return matched.some((node) => visibility.isVisibleOnScreen(node)); + return matched.some( + (node) => evaluateIsPredicate({ predicate: 'visible', node, nodes, platform }).pass, + ); } /** - * Sparseness reuses the signals absence assertions already trust rather than a second definition of - * readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its - * tail missing, and refusing it would fail large screens where the target is plainly in view. + * Sparseness is the same question absence assertions ask, answered by the same helper rather than a + * second definition of readable. Truncation is deliberately NOT refused: a truncated tree is real + * and readable with its tail missing, and refusing it would fail large screens where the target is + * plainly in view. */ -function captureRefusal( - nodes: CapturedNodes | undefined, - quality: SnapshotQualityVerdict | undefined, - backend: string | undefined, -): ScrollUntilCaptureRefusal | undefined { +function captureRefusal(result: SnapshotResult): ScrollUntilCaptureRefusal | undefined { + const nodes = result.nodes; if (nodes === undefined) { return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' }; } if (nodes.length === 0) { return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' }; } - if (quality?.state === 'sparse') { + const sparse = sparseCaptureQuality({ + backend: result.backend as SnapshotState['backend'], + nodes: nodes as SnapshotNode[], + ...(result.quality ? { snapshotQuality: result.quality } : {}), + }); + if (sparse) { return { reason: 'sparse-tree', - detail: quality.reason ?? 'the capture backend reported a sparse tree', + detail: sparse.reason ?? 'the capture backend reported a sparse tree', }; } - if ( - isLegacySparseIosInteractiveSnapshot({ - backend: backend as SnapshotState['backend'], - nodes: nodes as SnapshotNode[], - ...(quality ? { snapshotQuality: quality } : {}), - }) - ) { - return { reason: 'sparse-tree', detail: 'the capture exposed only the application root' }; - } return undefined; } -/** The nested state wins on nodes and backend; the verdict comes from whichever level carries one. */ -function canonicalCapture(capture: ScrollUntilCapture): { - nodes: CapturedNodes | undefined; - backend: string | undefined; - quality: SnapshotQualityVerdict | undefined; -} { - const nested = capture.snapshot; - return { - nodes: nested?.nodes ?? capture.nodes, - backend: nested?.backend ?? capture.backend, - quality: - nested?.snapshotQuality ?? - capture.snapshotQuality ?? - readSnapshotQualityVerdict(capture.quality), - }; -} - /** The content ran out, or the budget did. Separate messages: the corrective action differs. */ function scrollUntilNotFoundError( direction: ScrollDirection, From 52665cfdacea2f5aa08c2d8c47e973ed581bb790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 10 Sep 2026 16:46:16 +0200 Subject: [PATCH 13/13] refactor(interaction): keep requireResolvedPoint local to the gesture runtime --- src/commands/interaction/runtime/gestures.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index 9e17a3f18b..8e8e5fd846 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -347,7 +347,7 @@ function recordedDragTarget(target: ResolvedInteractionTarget): DragRecordingTar }; } -export function requireResolvedPoint(result: { point?: Point }): Point { +function requireResolvedPoint(result: { point?: Point }): Point { if (!result.point) { throw new AppError('COMMAND_FAILED', 'Interaction target resolved without coordinates'); }