feat(scroll): find off-screen targets in one command with --until (#2436)

* refactor(interaction): extract the scroll command runtime out of gestures.ts

* feat(scroll): add --until <selector>, report honored travel, fix web amount units

* test(scroll): cover --until through the provider-backed integration path

* perf(selectors): keep the scroll-until predicate off the eager import path

* fix(scroll): refuse an unreadable capture instead of reporting end-of-content

* fix(selectors): keep the capture-readability check off the eager import path

* test(selectors): use a declared snapshot quality state in the capture fixtures

* fix(scroll): read the capture quality verdict under the spelling the backend uses

* refactor(scroll): collapse --until onto the one route that runs it

* refactor(scroll): drop unexported until types and duplicated guidance prose

* test(scroll): fix the climbing fixture and drop duplicated route-level cases

* refactor(scroll): delete the dead command-runtime executor and reuse canonical predicates

* refactor(interaction): keep requireResolvedPoint local to the gesture runtime

Author: thymikee

packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts

@@ -14,45 +14,67 @@ import { captureThrows, scrollSnapshot, windowRoot } from './scroll-edge-state-f
 
 test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (bottom)', () => {
   assert.equal(
-    formatScrollEdgeMessage('down', 'bottom', 0, undefined, undefined),
+    formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 0 }),
     'Already at bottom; no hidden content below detected',
   );
 });
 
 test('formatScrollEdgeMessage: edge reached with zero passes reports already-at-edge (top)', () => {
   assert.equal(
-    formatScrollEdgeMessage('up', 'top', 0, undefined, undefined),
+    formatScrollEdgeMessage({ direction: 'up', edge: 'top', passes: 0 }),
     'Already at top; no hidden content above detected',
   );
 });
 
 test('formatScrollEdgeMessage: edge reached after N passes', () => {
   assert.equal(
-    formatScrollEdgeMessage('down', 'bottom', 4, undefined, undefined),
+    formatScrollEdgeMessage({ direction: 'down', edge: 'bottom', passes: 4 }),
     'Scrolled to bottom with 4 down passes',
   );
 });
 
 test('formatScrollEdgeMessage: no edge, pixel amount given', () => {
   assert.equal(
-    formatScrollEdgeMessage('down', undefined, 0, undefined, 250),
+    formatScrollEdgeMessage({ direction: 'down', passes: 0, pixels: 250 }),
     'Scrolled down by 250px',
   );
 });
 
 test('formatScrollEdgeMessage: no edge, no pixels, symbolic amount given', () => {
-  assert.equal(formatScrollEdgeMessage('up', undefined, 0, 3, undefined), 'Scrolled up by 3');
+  assert.equal(
+    formatScrollEdgeMessage({ direction: 'up', passes: 0, amount: 3 }),
+    'Scrolled up by 3',
+  );
 });
 
 test('formatScrollEdgeMessage: no edge, no pixels, no amount falls back to bare direction', () => {
+  assert.equal(formatScrollEdgeMessage({ direction: 'left', passes: 0 }), 'Scrolled left');
+});
+
+test('formatScrollEdgeMessage: pixels takes priority over amount when both are set', () => {
   assert.equal(
-    formatScrollEdgeMessage('left', undefined, 0, undefined, undefined),
-    'Scrolled left',
+    formatScrollEdgeMessage({ direction: 'down', passes: 0, amount: 3, pixels: 250 }),
+    'Scrolled down by 250px',
   );
 });
 
-test('formatScrollEdgeMessage: pixels takes priority over amount when both are set', () => {
-  assert.equal(formatScrollEdgeMessage('down', undefined, 0, 3, 250), 'Scrolled down by 250px');
+/**
+ * One gesture saturates at the viewport axis minus its edge padding, so a large amount buys less
+ * travel than it names. The message reports what the planner honored rather than what was asked.
+ */
+test('an amount-based message names the honored travel when the planner reports it', () => {
+  assert.equal(
+    formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 3, honoredPixels: 640 }),
+    'Scrolled down by 3 of the viewport (640px)',
+  );
+  assert.equal(
+    formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 0.65 }),
+    'Scrolled down by 0.65',
+  );
+  assert.equal(
+    formatScrollEdgeMessage({ direction: 'down', passes: 1, pixels: 5000, honoredPixels: 640 }),
+    'Scrolled down by 640px',
+  );
 });
 
 // ---------------------------------------------------------------------------
@@ -279,7 +301,7 @@ test('runScrollEdgePasses: throws a COMMAND_FAILED AppError once the pass limit
         'scroll bottom reached the safety limit before the snapshot showed the edge',
       );
       assert.deepEqual(error.details, {
-        hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.',
+        hint: 'The scoped scroll container still reports hidden content. Run scroll <dir> --until <selector> to stop on the element you are after, or snapshot -i to inspect the current state.',
       });
       return true;
     },

packages/capture-kit/src/snapshot/scroll-edge-state.ts

@@ -37,6 +37,18 @@ export async function captureScrollEdgeState(params: {
   }
 }
 
+/**
+ * Is there hidden content left at this edge? The same question `runScrollEdgePasses` loops on,
+ * exposed for callers with their own stop condition (`scroll --until`) so both read one signal.
+ */
+export async function canScrollFurtherAtEdge(
+  nodes: readonly (RawSnapshotNode | SnapshotNode)[],
+  edge: ScrollEdge,
+): Promise<boolean> {
+  const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts');
+  return analyzeScrollEdgeState(nodes, edge).canScroll;
+}
+
 export async function runScrollEdgePasses<TResult>(params: {
   edge: ScrollEdge;
   captureState: (scope?: string) => Promise<ScrollEdgeState>;
@@ -56,7 +68,7 @@ export async function runScrollEdgePasses<TResult>(params: {
         'COMMAND_FAILED',
         `scroll ${edge} reached the safety limit before the snapshot showed the edge`,
         {
-          hint: 'The scoped scroll container still reports hidden content. Use a smaller manual scroll + snapshot loop to inspect the current state.',
+          hint: 'The scoped scroll container still reports hidden content. Run scroll <dir> --until <selector> to stop on the element you are after, or snapshot -i to inspect the current state.',
         },
       );
     }
@@ -69,19 +81,31 @@ export async function runScrollEdgePasses<TResult>(params: {
   return { passes, result };
 }
 
-export function formatScrollEdgeMessage(
-  direction: ScrollDirection,
-  edge: ScrollEdge | undefined,
-  passes: number,
-  amount: number | undefined,
-  pixels: number | undefined,
-): string {
+/**
+ * `honoredPixels` is the travel the gesture planner actually produced, which is not always the
+ * travel that was asked for: one gesture cannot cross more than the viewport axis minus its edge
+ * padding, so a large `amount` saturates. Naming the honored distance is what keeps
+ * `scroll down 3` from reporting a three-viewport scroll it never performed.
+ */
+export function formatScrollEdgeMessage(params: {
+  direction: ScrollDirection;
+  edge?: ScrollEdge | undefined;
+  passes: number;
+  amount?: number | undefined;
+  pixels?: number | undefined;
+  honoredPixels?: number | undefined;
+}): string {
+  const { direction, edge, passes, amount, pixels, honoredPixels } = params;
   if (edge && passes === 0) {
     return `Already at ${edge}; no hidden content ${edge === 'bottom' ? 'below' : 'above'} detected`;
   }
   if (edge) return `Scrolled to ${edge} with ${passes} ${direction} passes`;
-  if (pixels !== undefined) return `Scrolled ${direction} by ${pixels}px`;
-  if (amount !== undefined) return `Scrolled ${direction} by ${amount}`;
+  if (pixels !== undefined) return `Scrolled ${direction} by ${honoredPixels ?? pixels}px`;
+  if (amount !== undefined) {
+    return honoredPixels === undefined
+      ? `Scrolled ${direction} by ${amount}`
+      : `Scrolled ${direction} by ${amount} of the viewport (${honoredPixels}px)`;
+  }
   return `Scrolled ${direction}`;
 }
 

packages/contracts/src/cli-flags.ts

@@ -109,6 +109,8 @@ export type CliFlags = CloudProviderProfileFields &
     holdMs?: number;
     jitterPx?: number;
     pixels?: number;
+    /** Scroll: repeat passes until this selector is visible on screen. */
+    until?: string;
     doubleTap?: boolean;
     verify?: boolean;
     settle?: boolean;

packages/contracts/src/client-gesture.ts

@@ -146,4 +146,6 @@ export type ScrollOptions = DeviceCommandBaseOptions &
     amount?: number;
     pixels?: number;
     durationMs?: number;
+    /** Repeat scroll passes until this selector is visible on screen, then stop. */
+    until?: string;
   };

packages/contracts/src/client-request.ts

@@ -39,6 +39,8 @@ export type CommandExecutionOptions = Partial<ScreenshotRequestFlags> &
     holdMs?: number;
     jitterPx?: number;
     pixels?: number;
+    /** Scroll: repeat passes until this selector is visible on screen. */
+    until?: string;
     doubleTap?: boolean;
     verify?: boolean;
     settle?: boolean;

packages/contracts/src/platform-runtime-operations.ts

@@ -249,12 +249,14 @@ export const gestureViewportRuntimeUse = defineUse({ required: ['gestureViewport
 /** `scroll <direction>` executes one pass and needs nothing else. */
 const scrollDirectionUse = defineUse({ required: ['scrollDirection'] });
 /**
- * `scroll top` / `scroll bottom` verify hidden content between passes, so the capture is part of
- * the tier's requirement rather than something discovered mid-run — the retired leaf's
+ * Every scroll that verifies between passes: `scroll top`/`scroll bottom` read hidden content at
+ * the edge, and `scroll --until <selector>` re-reads the tree to decide whether the target came
+ * into view. Both check against that capture rather than a stale session snapshot, so the capture
+ * is part of the tier's requirement rather than something discovered mid-run — the retired leaf's
  * "requires snapshot support to verify hidden content before scrolling" refusal, moved to
- * admission.
+ * admission. One declaration, because the two tiers admit on identical facts.
  */
-const scrollEdgeUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] });
+const scrollVerifiedPassUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] });
 
 const gestureUsesByTier = Object.freeze({
   plan: gesturePlanUse,
@@ -275,7 +277,10 @@ export const gestureRuntimePlanUses = Object.freeze([
 export const swipeRuntimePlanUses = Object.freeze([gesturePlanUse] as const);
 
 /** Every use `scroll` can select between. */
-export const scrollRuntimePlanUses = Object.freeze([scrollDirectionUse, scrollEdgeUse] as const);
+export const scrollRuntimePlanUses = Object.freeze([
+  scrollDirectionUse,
+  scrollVerifiedPassUse,
+] as const);
 
 type GesturePlanFor<Tier extends GestureRuntimeTier> = Readonly<{
   tier: Tier;
@@ -337,15 +342,28 @@ function gesturePlan<const Tier extends GestureRuntimeTier>(tier: Tier): Gesture
  */
 export type ScrollRuntimePlan =
   | Readonly<{ kind: 'direction'; use: typeof scrollDirectionUse }>
-  | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollEdgeUse }>;
+  | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollVerifiedPassUse }>
+  | Readonly<{ kind: 'until'; until: string; use: typeof scrollVerifiedPassUse }>;
 
-/** `scroll top`/`scroll bottom` verify between passes; every other scroll executes one pass. */
+/**
+ * `scroll top`/`scroll bottom` verify between passes, `scroll --until` re-reads the tree between
+ * passes to check its selector, and every other scroll executes one pass. The edge directions carry
+ * their own stop condition, so pairing them with `--until` names two, which the caller rejects
+ * before this resolves.
+ */
 export function resolveScrollRuntimePlan(
-  input: Readonly<{ edge?: 'top' | 'bottom' }>,
+  input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>,
 ): ScrollRuntimePlan {
+  if (input.until !== undefined) {
+    return Object.freeze({
+      kind: 'until',
+      until: input.until,
+      use: scrollVerifiedPassUse,
+    } as const);
+  }
   return input.edge === undefined
     ? Object.freeze({ kind: 'direction', use: scrollDirectionUse } as const)
-    : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollEdgeUse } as const);
+    : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollVerifiedPassUse } as const);
 }
 const captureSnapshotWithCustomActionsUse = defineUse({
   required: ['captureSnapshot', 'captureSnapshotWithCustomActions'],

packages/contracts/src/scroll-command.ts

@@ -47,6 +47,24 @@ export function assertExclusiveScrollDistanceInputs(
   }
 }
 
+/**
+ * `top`/`bottom` are scroll-to-extreme requests that already carry a stop condition, so pairing one
+ * with `--until` names two and the request has no single meaning. Rejected at the surface rather
+ * than resolved by precedence, so neither stop condition can silently win.
+ */
+export function assertScrollUntilCompatible(
+  input: Readonly<{ edge?: 'top' | 'bottom'; until?: string }>,
+): void {
+  if (input.until === undefined || input.edge === undefined) return;
+  throw new AppError(
+    'INVALID_ARGS',
+    `scroll ${input.edge} already scrolls to the ${input.edge} edge and cannot take --until`,
+    {
+      hint: `Use scroll ${input.edge === 'bottom' ? 'down' : 'up'} --until <selector> to stop at the target, or scroll ${input.edge} to reach the edge.`,
+    },
+  );
+}
+
 export function normalizeScrollDurationMs(
   durationMs: number | undefined,
   options: { field?: string; invalidMessage?: string; max?: number } = {},
@@ -64,6 +82,13 @@ export function normalizeScrollDurationMs(
   return durationMs;
 }
 
+/** The travel the planner produced, which saturates below a large requested amount. */
+export function honoredScrollPixels(
+  result: Record<string, unknown> | undefined,
+): number | undefined {
+  return typeof result?.pixels === 'number' ? result.pixels : undefined;
+}
+
 export function honoredScrollDurationMs(
   result: Record<string, unknown> | undefined,
 ): number | undefined {
@@ -84,7 +109,9 @@ export type ScrollCommandResult = {
   direction: ScrollDirection;
   /** Set for `top`/`bottom` requests: the extreme being scrolled to. */
   edge?: 'top' | 'bottom';
-  /** Edge scrolls only: how many scroll-and-check passes ran. */
+  /** Set for `--until` requests: the selector the passes stopped on. */
+  until?: string;
+  /** Edge and until scrolls only: how many scroll-and-check passes ran. */
   passes?: number;
   amount?: number;
   pixels?: number;

packages/contracts/src/scroll-gesture.ts

@@ -83,7 +83,13 @@ export type InPageSwipeGesturePlan = {
   referenceHeight: number;
 };
 
-const DEFAULT_SCROLL_AMOUNT = 0.6;
+/**
+ * The finger-path fraction of the viewport axis one scroll covers when the caller names no
+ * distance. Exported because a backend with no viewport to measure against (the browser) scales its
+ * own default step by the ratio to this, and that ratio is only meaningful while both sides read
+ * the same number.
+ */
+export const DEFAULT_SCROLL_AMOUNT = 0.6;
 // Scroll gestures never touch the outer 10% of either axis. Modern app windows are edge-to-edge,
 // so the viewport includes the system bars: a swipe that starts inside the status bar (5.7% of a
 // Pixel 7's height, 6.9% of an iPhone's with a Dynamic Island) pulls the notification shade or

packages/platform-linux/src/input-actions.ts

@@ -2,6 +2,7 @@ import { ensureInputTool } from './linux-env.ts';
 import { resolveLinuxToolProvider, type LinuxPointerButton } from './tool-provider.ts';
 import { sleep } from '@agent-device/host-kit/retry';
 import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture';
+import { DEFAULT_SCROLL_AMOUNT } from '@agent-device/contracts/scroll-gesture';
 
 // ── Low-level wrappers ─────────────────────────────────────────────────
 
@@ -227,8 +228,11 @@ export async function scrollLinux(
         ? Math.max(1, Math.round(options.pixels / 15))
         : Math.max(1, Math.round(options.pixels / 40));
   } else if (options?.amount != null) {
-    // amount is a fraction (0–1+) of the viewport; scale relative to default
-    scrollCount = Math.max(1, Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / 0.6)));
+    // amount is a fraction (0–1+) of the viewport; scale relative to the shared default
+    scrollCount = Math.max(
+      1,
+      Math.round(DEFAULT_SCROLL_CLICKS * (options.amount / DEFAULT_SCROLL_AMOUNT)),
+    );
   }
 
   // xdotool: button 4=up, 5=down, 6=left, 7=right

packages/platform-web/src/agent-browser-provider.test.ts

@@ -851,3 +851,28 @@ function createAudioProbeScriptPage(): AudioProbeScriptPage {
     },
   };
 }
+
+/**
+ * #2432: `amount` is a fraction of the viewport axis everywhere else, and the browser scrolls by
+ * CSS pixels. Passing it through raw made `scroll down 0.5` travel half a pixel.
+ */
+test('a relative scroll amount reaches agent-browser as pixels, not as the fraction itself', async () => {
+  await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
+    const calls: AgentBrowserCall[] = [];
+
+    await withCommandExecutorOverride(recordingExecutor(calls), async () => {
+      await provider.scroll('down', { amount: 0.6 });
+      await provider.scroll('down', { amount: 1.2 });
+      await provider.scroll('down', undefined);
+    });
+
+    assert.deepEqual(
+      calls.map((call) => call.args),
+      [
+        ['scroll', 'down', '300', '--json', '--session', 'web-session'],
+        ['scroll', 'down', '600', '--json', '--session', 'web-session'],
+        ['scroll', 'down', '--json', '--session', 'web-session'],
+      ],
+    );
+  });
+});