From edaef57c4914be56155b27544da4cb5274325011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 21:01:18 +0200 Subject: [PATCH 1/3] fix(contracts): state the scroll keyboard clip once for every platform --- .../fixtures/scroll-keyboard-policy.json | 145 ++++++++++++++++++ packages/contracts/src/scroll-command.ts | 11 ++ packages/contracts/src/scroll-gesture.test.ts | 135 ++++++++++++++++ packages/contracts/src/scroll-gesture.ts | 133 ++++++++++++++++ 4 files changed, 424 insertions(+) create mode 100644 contracts/fixtures/scroll-keyboard-policy.json diff --git a/contracts/fixtures/scroll-keyboard-policy.json b/contracts/fixtures/scroll-keyboard-policy.json new file mode 100644 index 0000000000..39bc64aca2 --- /dev/null +++ b/contracts/fixtures/scroll-keyboard-policy.json @@ -0,0 +1,145 @@ +{ + "constants": { + "minVisibleFraction": 0.15, + "accessoryAllowance": 12, + "occlusionReason": "scroll_keyboard_occludes_surface" + }, + "cases": [ + { + "name": "iPhone portrait keyboard covering the lower 310pt leaves the band above the accessory allowance", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 564, "width": 402, "height": 310 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 552 }, + "keyboardMinY": 564 + } + }, + { + "name": "keyboard flush with the viewport bottom occludes nothing", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 874, "width": 402, "height": 310 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "keyboard entirely above the viewport occludes nothing", + "viewport": { "x": 0, "y": 400, "width": 402, "height": 474 }, + "keyboard": { "x": 0, "y": 100, "width": 402, "height": 300 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "landscape keyboard leaving 28pt of a 402pt axis is refused instead of swiping under the keys", + "viewport": { "x": 0, "y": 0, "width": 874, "height": 402 }, + "keyboard": { "x": 0, "y": 40, "width": 874, "height": 362 }, + "expected": { "kind": "occluded", "keyboardMinY": 40, "visibleHeight": 28 } + }, + { + "name": "keyboard covering the whole viewport is refused with zero visible height", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "expected": { "kind": "occluded", "keyboardMinY": 0, "visibleHeight": 0 } + }, + { + "name": "keyboard whose top sits above the viewport top is refused with the visible height clamped to zero", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": -100, "width": 402, "height": 974 }, + "expected": { "kind": "occluded", "keyboardMinY": -100, "visibleHeight": 0 } + }, + { + "name": "band exactly at the 15% floor is kept: the refusal boundary is inclusive", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 0, "y": 132, "width": 400, "height": 668 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 120 }, + "keyboardMinY": 132 + } + }, + { + "name": "band one point below the 15% floor is refused", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 0, "y": 131, "width": 400, "height": 669 }, + "expected": { "kind": "occluded", "keyboardMinY": 131, "visibleHeight": 119 } + }, + { + "name": "keyboard starting exactly on the swipe centre line owns the surface", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 200, "y": 500, "width": 200, "height": 300 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 488 }, + "keyboardMinY": 500 + } + }, + { + "name": "floating keyboard starting one point past the swipe centre line is not struck", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 201, "y": 500, "width": 199, "height": 300 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "centred floating keyboard under half the viewport width sits in the swipe's path", + "viewport": { "x": 0, "y": 0, "width": 834, "height": 1112 }, + "keyboard": { "x": 257, "y": 780, "width": 320, "height": 332 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 834, "height": 768 }, + "keyboardMinY": 780 + } + }, + { + "name": "keyboard beside a split-screen app leaves that app's viewport swipeable", + "viewport": { "x": 0, "y": 0, "width": 390, "height": 844 }, + "keyboard": { "x": 390, "y": 500, "width": 390, "height": 344 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "Android absolute-pixel window clips against the window origin, not the display origin", + "viewport": { "x": 0, "y": 120, "width": 1080, "height": 2000 }, + "keyboard": { "x": 0, "y": 1600, "width": 1080, "height": 520 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 120, "width": 1080, "height": 1468 }, + "keyboardMinY": 1600 + } + }, + { + "name": "adjustPan window panned up under the IME still clips to its own 188px remainder", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 900 }, + "keyboard": { "x": 0, "y": 1100, "width": 1080, "height": 700 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 188 }, + "keyboardMinY": 1100 + } + }, + { + "name": "adjustPan window with only 88px of 900px left above the IME is refused", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 900 }, + "keyboard": { "x": 0, "y": 1000, "width": 1080, "height": 800 }, + "expected": { "kind": "occluded", "keyboardMinY": 1000, "visibleHeight": 88 } + }, + { + "name": "shallow Android keyboard covering under a quarter of a tall window still clips its own top", + "viewport": { "x": 0, "y": 0, "width": 1080, "height": 2200 }, + "keyboard": { "x": 0, "y": 1700, "width": 1080, "height": 500 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 1080, "height": 1688 }, + "keyboardMinY": 1700 + } + }, + { + "name": "empty keyboard frame fails open: a keyboard the platform cannot measure is not evidence of occlusion", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "empty viewport frame fails open too: resolving the best frame is the caller's job", + "viewport": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "keyboard": { "x": 0, "y": 564, "width": 402, "height": 310 }, + "expected": { "kind": "unobstructed" } + } + ] +} diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index 73b27385bb..59b2146b1a 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -118,4 +118,15 @@ export type ScrollCommandResult = { durationMs?: number; message?: string; settle?: SettleObservation; + /** + * Set only when an on-screen keyboard made the owner clip the swipe into the band above it + * (#2500). Absent means the swipe was not clipped, which is not the same claim as `false`: a + * platform that never runs the clip has nothing to report. The platform leaf's `referenceHeight` + * names the shortened axis the reported `pixels` were planned against, and `keyboardMinY` names + * where the keyboard began. A surface the owner refused to swipe at all fails instead, under the + * `scroll_keyboard_occludes_surface` reason. + */ + keyboardAvoided?: true; + /** The keyboard's edge in the same unit as the gesture coordinates, when the swipe was clipped. */ + keyboardMinY?: number; }; diff --git a/packages/contracts/src/scroll-gesture.test.ts b/packages/contracts/src/scroll-gesture.test.ts index 452ba476f4..41b8ad23d9 100644 --- a/packages/contracts/src/scroll-gesture.test.ts +++ b/packages/contracts/src/scroll-gesture.test.ts @@ -3,11 +3,17 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { assertScrollGestureInput, buildInPageSwipeGesturePlan, buildScrollGesturePlan, clampGestureCoordinate, + SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE, + SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, + clipScrollViewportAboveKeyboard, + scrollKeyboardOccludesSurfaceError, } from './scroll-gesture.ts'; import { DEFAULT_IOS_SCROLL_AMOUNT, @@ -201,3 +207,132 @@ test('clampGestureCoordinate rounds values and clamps them into the safe gesture test('clampGestureCoordinate returns the lower bound for non-finite coordinates', () => { assert.equal(clampGestureCoordinate(Number.POSITIVE_INFINITY, 8, 100), 8); }); + +// Golden parity table: the SAME JSON is asserted against the Swift twin +// (ScrollViewportPolicy in apple/runner/AgentDeviceRunner/ +// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in the same file), so +// a clip that drifts between the iOS runner and the Android/TS owner turns CI red on whichever +// side changed, without a simulator. + +type ExpectedClip = + | { kind: 'unobstructed' } + | { kind: 'avoided'; viewport: Rect; keyboardMinY: number } + | { kind: 'occluded'; keyboardMinY: number; visibleHeight: number }; + +type FixtureCase = { + name: string; + viewport: Rect; + keyboard: Rect; + expected: ExpectedClip; +}; + +type Fixture = { + constants: { + minVisibleFraction: number; + accessoryAllowance: number; + occlusionReason: string; + }; + cases: FixtureCase[]; +}; + +const TABLE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'scroll-keyboard-policy.json', +); + +function loadTable(): Fixture { + return JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as Fixture; +} + +test('the scroll keyboard clip agrees with every golden parity table case', () => { + const table = loadTable(); + assert.ok(table.cases.length > 0, 'parity table must not be empty'); + const names = new Set(table.cases.map((fixture) => fixture.name)); + assert.equal(names.size, table.cases.length, 'parity table case names must be unique'); + for (const fixture of table.cases) { + assert.deepEqual( + clipScrollViewportAboveKeyboard(fixture.viewport, fixture.keyboard), + fixture.expected, + fixture.name, + ); + } +}); + +test('the keyboard clip thresholds and refusal keys belong to the table, not this file', () => { + const { constants } = loadTable(); + assert.equal(constants.minVisibleFraction, SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION); + assert.equal(constants.accessoryAllowance, SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE); + assert.equal(constants.occlusionReason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); +}); + +test('a clipped viewport keeps even a saturated swipe inside the visible band', () => { + // The whole point of clipping before planning: `buildScrollGesturePlan` is untouched and gets the + // clipped axis, so its own edge padding is what holds the gesture above the keyboard. + const viewport: Rect = { x: 0, y: 0, width: 402, height: 874 }; + const clip = clipScrollViewportAboveKeyboard(viewport, { + x: 0, + y: 564, + width: 402, + height: 310, + }); + assert.equal(clip.kind, 'avoided'); + if (clip.kind !== 'avoided') return; + const plan = buildScrollGesturePlan({ + direction: 'down', + amount: 10, + referenceWidth: clip.viewport.width, + referenceHeight: clip.viewport.height, + }); + assert.ok(Math.max(plan.y1, plan.y2) <= clip.viewport.height); + assert.ok(plan.pixels < clip.viewport.height, 'honored travel must name the clipped axis'); +}); + +test('an unusable keyboard frame fails open instead of refusing every scroll', () => { + const viewport: Rect = { x: 0, y: 0, width: 402, height: 874 }; + for (const keyboard of [ + { x: 0, y: Number.NaN, width: 402, height: 310 }, + { x: 0, y: 564, width: Number.POSITIVE_INFINITY, height: 310 }, + { x: 0, y: 564, width: 402, height: -310 }, + { x: 0, y: 564, width: 402, height: 0 }, + ] satisfies Rect[]) { + assert.equal(clipScrollViewportAboveKeyboard(viewport, keyboard).kind, 'unobstructed'); + } +}); + +test('the occlusion refusal is keyed on its reason, not on its message', () => { + const error = scrollKeyboardOccludesSurfaceError('down', { + kind: 'occluded', + keyboardMinY: 564, + visibleHeight: 40, + viewportHeight: 874, + }); + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); + assert.equal(error.details?.keyboardMinY, 564); + assert.equal(error.details?.visibleHeight, 40); + assert.equal(error.details?.viewportHeight, 874); + assert.match(String(error.details?.hint), /keyboard dismiss/); +}); + +test('an unmeasured refusal still names the same reason, so error text never gates recovery', () => { + // The iOS runner refuses in its own coordinate space and reports only its typed runner code, so + // the Apple owner rebuilds this error without numbers. Matching on `reason` has to yield the + // same key with or without a measurement, or the message becomes the discriminator. + const unmeasured = scrollKeyboardOccludesSurfaceError('down'); + const measured = scrollKeyboardOccludesSurfaceError('down', { + kind: 'occluded', + keyboardMinY: 564, + visibleHeight: 40, + viewportHeight: 874, + }); + assert.equal(unmeasured.details?.reason, measured.details?.reason); + assert.equal(unmeasured.details?.keyboardMinY, undefined); + assert.equal(unmeasured.details?.visibleHeight, undefined); + assert.ok(!String(unmeasured.message).includes('px of'), 'no fabricated measurement may appear'); +}); diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index cdc7ddc971..3619f28fdc 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -329,3 +329,136 @@ export function clampGestureCoordinate(value: number, marginPx: number, size: nu return Math.min(max, Math.max(min, Math.round(value))); } + +/** + * What an on-screen keyboard leaves of the scroll viewport, and what happens when it leaves too + * little. + * + * `buildScrollGesturePlan` centres a directional swipe, so a focused field puts its lower endpoint + * under the keyboard: the gesture lands on keys, the surface never moves, and the edge loop reads a + * stuck container (#2499) instead of a refusal. Reducing the reference height BEFORE the planner + * runs keeps the swipe inside what is visible without touching the planner, so reported travel + * stays honest. The runner and the Android helper each read their own live keyboard frame; a frame + * threaded from the daemon would be a snapshot that predates the keyboard. + * + * Pure geometry, so the decision is proven against a golden table both this file and the Swift twin + * (`ScrollViewportPolicy` in apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/ + * RunnerScrollViewportPolicy.swift) assert against, so drift turns CI red without a simulator. The + * table carries only frames representable in both languages: `CGRect` standardizes a negative + * extent into a positive height at a moved origin, so a negative `height` is tested here alone. + */ + +/** Below this fraction of the viewport, the clipped band cannot hold a reliable swipe. */ +export const SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION = 0.15; + +/** + * A fixed allowance kept above the keyboard's top edge, in the frame's own unit (points on iOS, + * pixels on Android). `keyboard.frame` reports the key plane, not the input accessory or composer + * bar riding above it, so a swipe that ends exactly at the reported edge can still land on a bar. + */ +export const SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE = 12; + +/** + * The one reason a directional scroll refuses to swipe at all (#2500). The iOS runner answers with + * its own runner error code for the same condition; that code is the Apple runner's wire + * vocabulary and lives with it, not here, because Android raises this reason locally. + */ +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON = 'scroll_keyboard_occludes_surface'; + +export type ScrollKeyboardClip = + /** No keyboard, or one that does not own this surface: swipe the whole viewport. */ + | { kind: 'unobstructed' } + /** The viewport trimmed above the keyboard. Report the reduced reference height honestly. */ + | { kind: 'avoided'; viewport: Rect; keyboardMinY: number } + /** + * Too little surface left above the keyboard to swipe. Swiping anyway reads as a stuck + * container to the no-progress fingerprint, so the caller refuses with + * `SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON` instead. + */ + | { kind: 'occluded'; keyboardMinY: number; visibleHeight: number }; + +/** The numbers a refusing owner can name about the surface it declined to swipe. */ +export type ScrollKeyboardOcclusion = Extract & { + viewportHeight: number; +}; + +/** + * Clips a scroll viewport to the band above an occluding keyboard. + * + * Fails open on an unusable frame: a keyboard the platform cannot measure is not evidence that the + * surface is blocked, and turning a missing frame into a refusal would refuse every scroll on a + * device with a broken keyboard query. The floor is the caller's refusal, not this rule. + */ +export function clipScrollViewportAboveKeyboard( + viewport: Rect, + keyboard: Rect, +): ScrollKeyboardClip { + if (!isMeasurableRect(viewport) || !isMeasurableRect(keyboard)) return { kind: 'unobstructed' }; + // A vertical swipe runs along the viewport's centre line, which is the only part of the width the + // keyboard has to reach to be struck: a 320pt keyboard centred in an 834pt viewport is 38% of the + // width and sits exactly in the path. A horizontal swipe runs along the OTHER centre line, so + // clipping it too is early rather than wrong — it only lifts the swipe clear of the keys. + const swipeCenterX = viewport.x + viewport.width / 2; + if (swipeCenterX < keyboard.x || swipeCenterX >= keyboard.x + keyboard.width) { + return { kind: 'unobstructed' }; + } + const keyboardMinY = keyboard.y; + if (keyboardMinY >= viewport.y + viewport.height || keyboard.y + keyboard.height <= viewport.y) { + return { kind: 'unobstructed' }; + } + const visibleHeight = Math.max( + 0, + keyboardMinY - SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE - viewport.y, + ); + if (visibleHeight < SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION * viewport.height) { + return { kind: 'occluded', keyboardMinY, visibleHeight }; + } + return { + kind: 'avoided', + viewport: { ...viewport, height: visibleHeight }, + keyboardMinY, + }; +} + +/** + * The refusal a scroll reports when the keyboard owns the surface. `avoidanceNeverDismisses` is the + * point: a dismiss drops focus, which breaks a `type`/`scroll`/`type` loop, is not idempotent + * across platforms (Android's ESC loop can throw `UNSUPPORTED_OPERATION`), and mutates state + * session-action provenance does not record. So the caller names the tradeoff instead of paying it. + * + * `occlusion` is optional because the owner that measured the frame may be the runner rather than + * this process: the iOS XCTest runner refuses in its own coordinate space and reports the typed + * runner code, and re-deriving its numbers here would be a second source of truth. + */ +export function scrollKeyboardOccludesSurfaceError( + direction: ScrollDirection, + occlusion?: ScrollKeyboardOcclusion, +): AppError { + const percent = Math.round(SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION * 100); + const measured = + occlusion === undefined + ? 'the keyboard leaves too little visible surface for a swipe' + : `the keyboard leaves ${occlusion.visibleHeight}px of ${occlusion.viewportHeight}px visible, below the ${percent}% needed for a swipe`; + return new AppError('COMMAND_FAILED', `scroll ${direction} refused: ${measured}`, { + reason: SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, + ...(occlusion === undefined + ? {} + : { + keyboardMinY: occlusion.keyboardMinY, + visibleHeight: occlusion.visibleHeight, + viewportHeight: occlusion.viewportHeight, + }), + minVisibleFraction: SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, + hint: + 'The on-screen keyboard covers the surface this scroll would swipe, so it cannot reach it. ' + + 'Run `keyboard dismiss` and retry, accepting that it drops focus (re-tap the field to keep typing), or scroll before focusing the field.', + }); +} + +function isMeasurableRect(rect: Rect): boolean { + return ( + [rect.x, rect.y, rect.width, rect.height].every((value) => Number.isFinite(value)) && + rect.width > 0 && + rect.height > 0 + ); +} From 1b8548af6dfbeb087bbf0ff521b061ebf4c42d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 21:01:18 +0200 Subject: [PATCH 2/3] fix(apple): surface the scroll keyboard clip as evidence and a typed reason --- .../src/core/__tests__/scroll.test.ts | 109 ++++++++++++++++++ packages/platform-apple/src/core/scroll.ts | 33 ++++++ packages/platform-apple/src/interactions.ts | 28 +++-- .../src/runner/runner-contract.ts | 12 +- src/commands/interaction/metadata.ts | 2 +- src/mcp/command-output-schemas.ts | 6 + website/docs/docs/commands.md | 1 + 7 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 packages/platform-apple/src/core/__tests__/scroll.test.ts diff --git a/packages/platform-apple/src/core/__tests__/scroll.test.ts b/packages/platform-apple/src/core/__tests__/scroll.test.ts new file mode 100644 index 0000000000..b090f1143b --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/scroll.test.ts @@ -0,0 +1,109 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON } from '@agent-device/contracts/scroll-gesture'; +import { + normalizeAppleScrollResultWithResolvedFrame, + withAppleScrollKeyboardOcclusion, +} from '../scroll.ts'; + +const RUNNER_OCCLUSION_CODE = 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'; + +function runnerOcclusionError(): AppError { + return new AppError( + 'COMMAND_FAILED', + 'scroll down refused: the keyboard leaves 28pt of surface', + { + runnerErrorCode: RUNNER_OCCLUSION_CODE, + logPath: '/tmp/runner.log', + }, + ); +} + +test('a clipped runner frame yields travel and evidence for the band actually swiped', () => { + // The runner clips the interaction frame above the keyboard and reports the clipped axis, so the + // TS recomputation of `pixels` must be honest about the shorter travel (#2500). Reading the + // unclipped screen height here would over-report travel the surface never had. + const result = normalizeAppleScrollResultWithResolvedFrame( + { + x: 201, + y: 500, + x2: 201, + y2: 100, + referenceWidth: 402, + referenceHeight: 552, + keyboardAvoided: true, + keyboardMinY: 564, + }, + 'down', + { amount: 0.65, durationMs: 400 }, + ); + assert.equal(result.referenceHeight, 552); + assert.equal(result.pixels, Math.round(552 * 0.65)); + assert.equal(result.keyboardAvoided, true); + assert.equal(result.keyboardMinY, 564); +}); + +test('an unclipped scroll reports no avoidance evidence rather than a fabricated false', () => { + // A plain `false` could not tell "no keyboard was up" from "this platform never runs the clip", + // so absence is the negative case, and the schema has to keep both distinguishable. + const result = normalizeAppleScrollResultWithResolvedFrame( + { x: 201, y: 500, x2: 201, y2: 100, referenceWidth: 402, referenceHeight: 874 }, + 'down', + { amount: 0.65 }, + ); + assert.equal('keyboardAvoided' in result, false); + assert.equal('keyboardMinY' in result, false); +}); + +test('avoidance from a runner that reports no keyboard edge is still avoidance', () => { + // `keyboardMinY` is additive evidence; a runner build that clipped without naming the edge must not + // lose the fact that it clipped at all. + const result = normalizeAppleScrollResultWithResolvedFrame( + { + x: 201, + y: 500, + x2: 201, + y2: 100, + referenceWidth: 402, + referenceHeight: 552, + keyboardAvoided: true, + }, + 'down', + { amount: 0.65 }, + ); + assert.equal(result.keyboardAvoided, true); + assert.equal('keyboardMinY' in result, false); +}); + +test('the runner keyboard refusal becomes the typed reason a caller can branch on', () => { + const mapped = withAppleScrollKeyboardOcclusion(runnerOcclusionError(), 'down'); + assert.ok(mapped instanceof AppError); + assert.equal(mapped.code, 'COMMAND_FAILED'); + assert.equal(mapped.details?.reason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); + assert.match(String(mapped.details?.hint), /keyboard dismiss/); +}); + +test('the mapped refusal keeps the transport diagnostics the original error carried', () => { + const mapped = withAppleScrollKeyboardOcclusion(runnerOcclusionError(), 'down'); + assert.ok(mapped instanceof AppError); + assert.equal(mapped.details?.logPath, '/tmp/runner.log'); + assert.equal(mapped.details?.runnerErrorCode, RUNNER_OCCLUSION_CODE); +}); + +test('the nearest negatives stay untouched, so only the refusal code renames an error', () => { + // Same code, different classification: a generic scroll failure must not read as an occlusion, or + // the caller would be told to dismiss a keyboard that is not in the way. + const generic = new AppError( + 'COMMAND_FAILED', + 'scroll could not resolve a usable interaction frame', + { + logPath: '/tmp/runner.log', + }, + ); + assert.equal(withAppleScrollKeyboardOcclusion(generic, 'down'), generic); + const transport = new Error('socket hang up'); + assert.equal(withAppleScrollKeyboardOcclusion(transport, 'down'), transport); + const alert = new AppError('COMMAND_FAILED', 'no alert', { runnerErrorCode: 'ALERT_NOT_FOUND' }); + assert.equal(withAppleScrollKeyboardOcclusion(alert, 'down'), alert); +}); diff --git a/packages/platform-apple/src/core/scroll.ts b/packages/platform-apple/src/core/scroll.ts index daeb70dd49..3c7587b37d 100644 --- a/packages/platform-apple/src/core/scroll.ts +++ b/packages/platform-apple/src/core/scroll.ts @@ -7,7 +7,10 @@ import { import { type ScrollDirection, buildScrollGesturePlan, + scrollKeyboardOccludesSurfaceError, } from '@agent-device/contracts/scroll-gesture'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE } from '../runner/runner-contract.ts'; export type NormalizedScrollOptions = { amount?: number; @@ -18,6 +21,30 @@ export type NormalizedScrollOptions = { export type AppleScrollOptions = ScrollExecutionOptions; +/** + * Turns the runner's keyboard-occlusion refusal into the reason a caller acts on (#2500). + * + * The runner owns the live keyboard frame and declines to place a swipe it cannot keep above the + * keys; it answers with a typed runner code rather than prose so nothing here has to read an error + * message. The numbers stay on the runner's side of the boundary — re-deriving them from a frame + * this process does not hold would be a second source of truth — so the reason and hint are the + * evidence, joined to whatever the transport already recorded (`logPath`, `runnerErrorCode`). + */ +export function withAppleScrollKeyboardOcclusion( + error: unknown, + direction: ScrollDirection, +): unknown { + if (!(error instanceof AppError)) return error; + if (error.details?.['runnerErrorCode'] !== SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE) { + return error; + } + const refusal = scrollKeyboardOccludesSurfaceError(direction); + return new AppError(refusal.code, refusal.message, { + ...(error.details ?? {}), + ...(refusal.details ?? {}), + }); +} + export function materializeIosScrollOptions( options: AppleScrollOptions | undefined, ): AppleScrollOptions { @@ -91,6 +118,7 @@ export function normalizeAppleScrollResult( const verticalTravel = y1 !== undefined && y2 !== undefined ? Math.round(Math.abs(y2 - y1)) : undefined; const travelPixels = selectScrollTravelPixels(options, horizontalTravel, verticalTravel); + const keyboardMinY = readFiniteNumber(runnerResult.keyboardMinY); return { ...(x1 !== undefined ? { x1 } : {}), @@ -99,6 +127,11 @@ export function normalizeAppleScrollResult( ...(y2 !== undefined ? { y2 } : {}), ...(referenceWidth !== undefined ? { referenceWidth } : {}), ...(referenceHeight !== undefined ? { referenceHeight } : {}), + // Avoidance evidence (#2500) is reported only when it happened: `referenceHeight` above already + // names the clipped axis, and a plain `false` here could not tell "no keyboard" apart from a + // platform that never runs the clip. + ...(runnerResult.keyboardAvoided === true ? { keyboardAvoided: true } : {}), + ...(keyboardMinY !== undefined ? { keyboardMinY } : {}), ...(options?.amount !== undefined ? { amount: options.amount } : {}), ...(travelPixels !== undefined ? { pixels: travelPixels } : {}), ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index 515b4189cb..601a7b44de 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -32,6 +32,7 @@ import { normalizeAppleScrollResult, normalizeAppleScrollResultWithResolvedFrame, scrollRunnerFields, + withAppleScrollKeyboardOcclusion, type AppleScrollOptions, } from './core/scroll.ts'; import { runMacosDesktopScroll } from './os/macos/desktop-scroll.ts'; @@ -429,16 +430,23 @@ async function runAppleScroll( const iosOptions = materializeIosScrollOptions(options); // Single fused lifecycle command: the runner resolves the interaction frame and runs the drag. - const runnerResult = await runRunnerCommand( - device, - { - command: 'scroll', - direction, - ...scrollRunnerFields(iosOptions), - appBundleId: ctx.appBundleId, - }, - runnerOpts, - ); + // The runner is also the only party holding the live keyboard frame, so a surface it refuses to + // swipe under the keys arrives here as its typed runner code and leaves as a typed reason (#2500). + let runnerResult: Record; + try { + runnerResult = await runRunnerCommand( + device, + { + command: 'scroll', + direction, + ...scrollRunnerFields(iosOptions), + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + } catch (error) { + throw withAppleScrollKeyboardOcclusion(error, direction); + } return normalizeAppleScrollResultWithResolvedFrame(runnerResult, direction, iosOptions); } diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 1e954b612e..649ffce8f1 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -323,14 +323,24 @@ export function shouldRestartRunnerBeforeCommandSend(error: unknown): boolean { ); } +/** + * The code the XCTest runner answers with when it declines to place a scroll gesture under the + * on-screen keyboard (#2500). It is the runner's own vocabulary, so it is declared here beside the + * set that keeps it off the wire, and the Apple scroll owner matches it on `details.runnerErrorCode` + * rather than on error text. + */ +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'; + /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: - * `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there yet. + * `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there yet, and + * the scroll keyboard refusal for a surface the runner declined to swipe under the keys. */ const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlySet = new Set([ RUNNER_BUSY_RUNNER_CODE, ALERT_NOT_FOUND_RUNNER_CODE, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, ]); /** Wire code plus the details every path must publish for one runner-reported error code. */ diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 2e78dd567b..93438735a6 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band rather than dismissing it, which lowers the reported pixels; when too little surface is left to swipe, the command refuses with scroll_keyboard_occludes_surface and leaves focus alone.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 47b34efccc..20e83f6b49 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -554,6 +554,12 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { pixels: numberSchema(), durationMs: numberSchema(), message: stringSchema(), + keyboardAvoided: booleanSchema( + 'Present only when an on-screen keyboard forced the swipe into the band above it; the reported pixels were planned against the shorter referenceHeight.', + ), + keyboardMinY: numberSchema( + 'Where the keyboard began, in the same unit as the gesture coordinates. Clipped scrolls only.', + ), }, ['direction'], ), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3e1e6a1219..4c389e95c9 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -492,6 +492,7 @@ Target-authored drag is supported on Android touch devices and iOS/iPadOS. Backe On iOS simulators it uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path, so verify app-level metrics instead of assuming the requested values map exactly to recognizer output. On Android, `gesture transform` injects a geometric two-finger path. App recognizers may report non-exact pan, scale, and rotation values, so verify qualitative state such as `pan changed yes`, `pinch changed yes`, and `rotate changed yes` unless the app explicitly promises exact centroid metrics. If exact app-state values matter, prefer isolated `gesture pan`, `gesture pinch`, or `gesture rotate` commands. `scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. Directional scrolls decelerate through the drag on Android to reduce release momentum within the requested duration; `scroll top` and `scroll bottom` retain inertial release for edge traversal. Reduced momentum does not guarantee an exact content offset, especially for very short gestures: apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS. +A directional scroll places its swipe across the middle of the viewport, so a focused field and its keyboard would put the swipe under the keys: the gesture would land on the keyboard, the surface would not move, and the scroll would read as stuck. On iOS the scroll instead keeps the whole swipe in the band above the keyboard, reporting `keyboardAvoided` and `keyboardMinY` alongside a `referenceHeight` and `pixels` measured against that shorter band. It never dismisses the keyboard, because dismissing drops focus and breaks a `fill`/`scroll`/`fill` loop; run `keyboard dismiss` yourself when you want that. When the keyboard leaves too little room to swipe, the command refuses with the `scroll_keyboard_occludes_surface` reason rather than swiping into the keys, so a scroll that cannot work says so instead of appearing stuck. Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs. When a target only appears in an off-screen summary, use `scroll --settle`: the response waits for the UI to go quiet and returns the diff against the tree you last observed, with fresh refs on the added lines, so no follow-up `snapshot -i` is needed. `back --settle` does the same for navigation. Both are best-effort and never fail the action. For repeated checks without settle, a small shell loop is enough: From 673ea455d6912679c2aaeb657e33c8acf74e53a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 13 Sep 2026 10:52:36 +0200 Subject: [PATCH 3/3] refactor(contracts): state the scroll keyboard refusal details once and keep the runner's message The Apple scroll owner rebuilt the refusal per command, discarding the runner's measured message and carrying an unmeasured variant of the error builder for it. The shared reason and hint are now one frozen object in scroll-gesture; the Apple owner adds it to the runner's own error (matched on the typed runner code, transport details kept), and the error builder takes a plain measured occlusion, which only Android produces in-process. The help text names the behaviour in one clause; the hint carries the recovery at the moment it matters. --- packages/contracts/src/scroll-gesture.test.ts | 18 ----- packages/contracts/src/scroll-gesture.ts | 65 ++++++++++--------- .../src/core/__tests__/scroll.test.ts | 52 +++++---------- packages/platform-apple/src/core/scroll.ts | 25 +++---- packages/platform-apple/src/interactions.ts | 2 +- src/commands/interaction/metadata.ts | 2 +- 6 files changed, 61 insertions(+), 103 deletions(-) diff --git a/packages/contracts/src/scroll-gesture.test.ts b/packages/contracts/src/scroll-gesture.test.ts index 41b8ad23d9..1c5f700560 100644 --- a/packages/contracts/src/scroll-gesture.test.ts +++ b/packages/contracts/src/scroll-gesture.test.ts @@ -306,7 +306,6 @@ test('an unusable keyboard frame fails open instead of refusing every scroll', ( test('the occlusion refusal is keyed on its reason, not on its message', () => { const error = scrollKeyboardOccludesSurfaceError('down', { - kind: 'occluded', keyboardMinY: 564, visibleHeight: 40, viewportHeight: 874, @@ -319,20 +318,3 @@ test('the occlusion refusal is keyed on its reason, not on its message', () => { assert.equal(error.details?.viewportHeight, 874); assert.match(String(error.details?.hint), /keyboard dismiss/); }); - -test('an unmeasured refusal still names the same reason, so error text never gates recovery', () => { - // The iOS runner refuses in its own coordinate space and reports only its typed runner code, so - // the Apple owner rebuilds this error without numbers. Matching on `reason` has to yield the - // same key with or without a measurement, or the message becomes the discriminator. - const unmeasured = scrollKeyboardOccludesSurfaceError('down'); - const measured = scrollKeyboardOccludesSurfaceError('down', { - kind: 'occluded', - keyboardMinY: 564, - visibleHeight: 40, - viewportHeight: 874, - }); - assert.equal(unmeasured.details?.reason, measured.details?.reason); - assert.equal(unmeasured.details?.keyboardMinY, undefined); - assert.equal(unmeasured.details?.visibleHeight, undefined); - assert.ok(!String(unmeasured.message).includes('px of'), 'no fabricated measurement may appear'); -}); diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index 3619f28fdc..5b8be2fe8f 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -359,12 +359,23 @@ export const SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION = 0.15; export const SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE = 12; /** - * The one reason a directional scroll refuses to swipe at all (#2500). The iOS runner answers with - * its own runner error code for the same condition; that code is the Apple runner's wire - * vocabulary and lives with it, not here, because Android raises this reason locally. + * The one reason a directional scroll refuses to swipe at all (#2500), with the hint every owner + * publishes beside it: Android measures the occlusion in this process, and the iOS runner answers + * with its own runner code that the Apple scroll owner joins to these same details, so a + * caller branches on `reason` and reads one hint whichever owner refused. Avoidance never dismisses + * the keyboard: a dismiss drops focus, which breaks a `type`/`scroll`/`type` loop, is not idempotent + * across platforms, and mutates state session-action provenance does not record. So the hint names + * the tradeoff instead of paying it. */ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON = 'scroll_keyboard_occludes_surface'; +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS = Object.freeze({ + reason: SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, + hint: + 'The on-screen keyboard covers the surface this scroll would swipe, so it cannot reach it. ' + + 'Run `keyboard dismiss` and retry, accepting that it drops focus (re-tap the field to keep typing), or scroll before focusing the field.', +}); + export type ScrollKeyboardClip = /** No keyboard, or one that does not own this surface: swipe the whole viewport. */ | { kind: 'unobstructed' } @@ -377,8 +388,10 @@ export type ScrollKeyboardClip = */ | { kind: 'occluded'; keyboardMinY: number; visibleHeight: number }; -/** The numbers a refusing owner can name about the surface it declined to swipe. */ -export type ScrollKeyboardOcclusion = Extract & { +/** The numbers a refusing owner names about the surface it declined to swipe. */ +export type ScrollKeyboardOcclusion = { + keyboardMinY: number; + visibleHeight: number; viewportHeight: number; }; @@ -421,38 +434,26 @@ export function clipScrollViewportAboveKeyboard( } /** - * The refusal a scroll reports when the keyboard owns the surface. `avoidanceNeverDismisses` is the - * point: a dismiss drops focus, which breaks a `type`/`scroll`/`type` loop, is not idempotent - * across platforms (Android's ESC loop can throw `UNSUPPORTED_OPERATION`), and mutates state - * session-action provenance does not record. So the caller names the tradeoff instead of paying it. - * - * `occlusion` is optional because the owner that measured the frame may be the runner rather than - * this process: the iOS XCTest runner refuses in its own coordinate space and reports the typed - * runner code, and re-deriving its numbers here would be a second source of truth. + * The refusal an owner that measured the keyboard in this process reports. The iOS runner measures + * in its own coordinate space and answers with its runner code instead; the Apple scroll owner + * adds the same `SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS` to that error. */ export function scrollKeyboardOccludesSurfaceError( direction: ScrollDirection, - occlusion?: ScrollKeyboardOcclusion, + occlusion: ScrollKeyboardOcclusion, ): AppError { const percent = Math.round(SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION * 100); - const measured = - occlusion === undefined - ? 'the keyboard leaves too little visible surface for a swipe' - : `the keyboard leaves ${occlusion.visibleHeight}px of ${occlusion.viewportHeight}px visible, below the ${percent}% needed for a swipe`; - return new AppError('COMMAND_FAILED', `scroll ${direction} refused: ${measured}`, { - reason: SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, - ...(occlusion === undefined - ? {} - : { - keyboardMinY: occlusion.keyboardMinY, - visibleHeight: occlusion.visibleHeight, - viewportHeight: occlusion.viewportHeight, - }), - minVisibleFraction: SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, - hint: - 'The on-screen keyboard covers the surface this scroll would swipe, so it cannot reach it. ' + - 'Run `keyboard dismiss` and retry, accepting that it drops focus (re-tap the field to keep typing), or scroll before focusing the field.', - }); + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} refused: the keyboard leaves ${occlusion.visibleHeight}px of ${occlusion.viewportHeight}px visible, below the ${percent}% needed for a swipe`, + { + keyboardMinY: occlusion.keyboardMinY, + visibleHeight: occlusion.visibleHeight, + viewportHeight: occlusion.viewportHeight, + minVisibleFraction: SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, + ...SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, + }, + ); } function isMeasurableRect(rect: Rect): boolean { diff --git a/packages/platform-apple/src/core/__tests__/scroll.test.ts b/packages/platform-apple/src/core/__tests__/scroll.test.ts index b090f1143b..599860cf7c 100644 --- a/packages/platform-apple/src/core/__tests__/scroll.test.ts +++ b/packages/platform-apple/src/core/__tests__/scroll.test.ts @@ -7,19 +7,6 @@ import { withAppleScrollKeyboardOcclusion, } from '../scroll.ts'; -const RUNNER_OCCLUSION_CODE = 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'; - -function runnerOcclusionError(): AppError { - return new AppError( - 'COMMAND_FAILED', - 'scroll down refused: the keyboard leaves 28pt of surface', - { - runnerErrorCode: RUNNER_OCCLUSION_CODE, - logPath: '/tmp/runner.log', - }, - ); -} - test('a clipped runner frame yields travel and evidence for the band actually swiped', () => { // The runner clips the interaction frame above the keyboard and reports the clipped axis, so the // TS recomputation of `pixels` must be honest about the shorter travel (#2500). Reading the @@ -76,34 +63,29 @@ test('avoidance from a runner that reports no keyboard edge is still avoidance', assert.equal('keyboardMinY' in result, false); }); -test('the runner keyboard refusal becomes the typed reason a caller can branch on', () => { - const mapped = withAppleScrollKeyboardOcclusion(runnerOcclusionError(), 'down'); +test('the runner keyboard refusal gains the shared reason and hint and keeps its own message', () => { + const runnerError = new AppError( + 'COMMAND_FAILED', + 'scroll down refused: the keyboard leaves 28pt of visible surface above it', + { runnerErrorCode: 'SCROLL_KEYBOARD_OCCLUDES_SURFACE', logPath: '/tmp/runner.log' }, + ); + const mapped = withAppleScrollKeyboardOcclusion(runnerError); assert.ok(mapped instanceof AppError); assert.equal(mapped.code, 'COMMAND_FAILED'); + assert.equal(mapped.message, runnerError.message); assert.equal(mapped.details?.reason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); assert.match(String(mapped.details?.hint), /keyboard dismiss/); -}); - -test('the mapped refusal keeps the transport diagnostics the original error carried', () => { - const mapped = withAppleScrollKeyboardOcclusion(runnerOcclusionError(), 'down'); - assert.ok(mapped instanceof AppError); assert.equal(mapped.details?.logPath, '/tmp/runner.log'); - assert.equal(mapped.details?.runnerErrorCode, RUNNER_OCCLUSION_CODE); + assert.equal(mapped.details?.runnerErrorCode, 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'); }); -test('the nearest negatives stay untouched, so only the refusal code renames an error', () => { - // Same code, different classification: a generic scroll failure must not read as an occlusion, or - // the caller would be told to dismiss a keyboard that is not in the way. - const generic = new AppError( - 'COMMAND_FAILED', - 'scroll could not resolve a usable interaction frame', - { - logPath: '/tmp/runner.log', - }, - ); - assert.equal(withAppleScrollKeyboardOcclusion(generic, 'down'), generic); - const transport = new Error('socket hang up'); - assert.equal(withAppleScrollKeyboardOcclusion(transport, 'down'), transport); +test('only the refusal code is renamed, so a generic scroll failure never reads as an occlusion', () => { + const generic = new AppError('COMMAND_FAILED', 'scroll could not resolve a usable frame', { + logPath: '/tmp/runner.log', + }); + assert.equal(withAppleScrollKeyboardOcclusion(generic), generic); const alert = new AppError('COMMAND_FAILED', 'no alert', { runnerErrorCode: 'ALERT_NOT_FOUND' }); - assert.equal(withAppleScrollKeyboardOcclusion(alert, 'down'), alert); + assert.equal(withAppleScrollKeyboardOcclusion(alert), alert); + const transport = new Error('socket hang up'); + assert.equal(withAppleScrollKeyboardOcclusion(transport), transport); }); diff --git a/packages/platform-apple/src/core/scroll.ts b/packages/platform-apple/src/core/scroll.ts index 3c7587b37d..81a0fdff2f 100644 --- a/packages/platform-apple/src/core/scroll.ts +++ b/packages/platform-apple/src/core/scroll.ts @@ -6,8 +6,8 @@ import { } from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, buildScrollGesturePlan, - scrollKeyboardOccludesSurfaceError, } from '@agent-device/contracts/scroll-gesture'; import { AppError } from '@agent-device/kernel/errors'; import { SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE } from '../runner/runner-contract.ts'; @@ -22,26 +22,19 @@ export type NormalizedScrollOptions = { export type AppleScrollOptions = ScrollExecutionOptions; /** - * Turns the runner's keyboard-occlusion refusal into the reason a caller acts on (#2500). - * - * The runner owns the live keyboard frame and declines to place a swipe it cannot keep above the - * keys; it answers with a typed runner code rather than prose so nothing here has to read an error - * message. The numbers stay on the runner's side of the boundary — re-deriving them from a frame - * this process does not hold would be a second source of truth — so the reason and hint are the - * evidence, joined to whatever the transport already recorded (`logPath`, `runnerErrorCode`). + * Gives the runner's keyboard-occlusion refusal the shared reason and hint (#2500). The runner + * measured the keyboard in its own coordinate space, so its message and transport details + * (`runnerErrorCode`, `logPath`) are kept as they are; only the details every platform publishes + * are added, matched on the typed runner code rather than on error text. */ -export function withAppleScrollKeyboardOcclusion( - error: unknown, - direction: ScrollDirection, -): unknown { +export function withAppleScrollKeyboardOcclusion(error: unknown): unknown { if (!(error instanceof AppError)) return error; if (error.details?.['runnerErrorCode'] !== SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE) { return error; } - const refusal = scrollKeyboardOccludesSurfaceError(direction); - return new AppError(refusal.code, refusal.message, { - ...(error.details ?? {}), - ...(refusal.details ?? {}), + return new AppError(error.code, error.message, { + ...error.details, + ...SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, }); } diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index 601a7b44de..c6e5f07f46 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -445,7 +445,7 @@ async function runAppleScroll( runnerOpts, ); } catch (error) { - throw withAppleScrollKeyboardOcclusion(error, direction); + throw withAppleScrollKeyboardOcclusion(error); } return normalizeAppleScrollResultWithResolvedFrame(runnerResult, direction, iosOptions); diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 93438735a6..d7eb7a8cce 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band rather than dismissing it, which lowers the reported pixels; when too little surface is left to swipe, the command refuses with scroll_keyboard_occludes_surface and leaves focus alone.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band instead of being dismissed; when too little is left, the command refuses with scroll_keyboard_occludes_surface.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action',