diff --git a/contracts/fixtures/scroll-keyboard-policy.json b/contracts/fixtures/scroll-keyboard-policy.json new file mode 100644 index 0000000000..39bc64aca2 --- /dev/null +++ b/contracts/fixtures/scroll-keyboard-policy.json @@ -0,0 +1,145 @@ +{ + "constants": { + "minVisibleFraction": 0.15, + "accessoryAllowance": 12, + "occlusionReason": "scroll_keyboard_occludes_surface" + }, + "cases": [ + { + "name": "iPhone portrait keyboard covering the lower 310pt leaves the band above the accessory allowance", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 564, "width": 402, "height": 310 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 552 }, + "keyboardMinY": 564 + } + }, + { + "name": "keyboard flush with the viewport bottom occludes nothing", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 874, "width": 402, "height": 310 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "keyboard entirely above the viewport occludes nothing", + "viewport": { "x": 0, "y": 400, "width": 402, "height": 474 }, + "keyboard": { "x": 0, "y": 100, "width": 402, "height": 300 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "landscape keyboard leaving 28pt of a 402pt axis is refused instead of swiping under the keys", + "viewport": { "x": 0, "y": 0, "width": 874, "height": 402 }, + "keyboard": { "x": 0, "y": 40, "width": 874, "height": 362 }, + "expected": { "kind": "occluded", "keyboardMinY": 40, "visibleHeight": 28 } + }, + { + "name": "keyboard covering the whole viewport is refused with zero visible height", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "expected": { "kind": "occluded", "keyboardMinY": 0, "visibleHeight": 0 } + }, + { + "name": "keyboard whose top sits above the viewport top is refused with the visible height clamped to zero", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": -100, "width": 402, "height": 974 }, + "expected": { "kind": "occluded", "keyboardMinY": -100, "visibleHeight": 0 } + }, + { + "name": "band exactly at the 15% floor is kept: the refusal boundary is inclusive", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 0, "y": 132, "width": 400, "height": 668 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 120 }, + "keyboardMinY": 132 + } + }, + { + "name": "band one point below the 15% floor is refused", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 0, "y": 131, "width": 400, "height": 669 }, + "expected": { "kind": "occluded", "keyboardMinY": 131, "visibleHeight": 119 } + }, + { + "name": "keyboard starting exactly on the swipe centre line owns the surface", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 200, "y": 500, "width": 200, "height": 300 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 488 }, + "keyboardMinY": 500 + } + }, + { + "name": "floating keyboard starting one point past the swipe centre line is not struck", + "viewport": { "x": 0, "y": 0, "width": 400, "height": 800 }, + "keyboard": { "x": 201, "y": 500, "width": 199, "height": 300 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "centred floating keyboard under half the viewport width sits in the swipe's path", + "viewport": { "x": 0, "y": 0, "width": 834, "height": 1112 }, + "keyboard": { "x": 257, "y": 780, "width": 320, "height": 332 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 834, "height": 768 }, + "keyboardMinY": 780 + } + }, + { + "name": "keyboard beside a split-screen app leaves that app's viewport swipeable", + "viewport": { "x": 0, "y": 0, "width": 390, "height": 844 }, + "keyboard": { "x": 390, "y": 500, "width": 390, "height": 344 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "Android absolute-pixel window clips against the window origin, not the display origin", + "viewport": { "x": 0, "y": 120, "width": 1080, "height": 2000 }, + "keyboard": { "x": 0, "y": 1600, "width": 1080, "height": 520 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 120, "width": 1080, "height": 1468 }, + "keyboardMinY": 1600 + } + }, + { + "name": "adjustPan window panned up under the IME still clips to its own 188px remainder", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 900 }, + "keyboard": { "x": 0, "y": 1100, "width": 1080, "height": 700 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 188 }, + "keyboardMinY": 1100 + } + }, + { + "name": "adjustPan window with only 88px of 900px left above the IME is refused", + "viewport": { "x": 0, "y": 900, "width": 1080, "height": 900 }, + "keyboard": { "x": 0, "y": 1000, "width": 1080, "height": 800 }, + "expected": { "kind": "occluded", "keyboardMinY": 1000, "visibleHeight": 88 } + }, + { + "name": "shallow Android keyboard covering under a quarter of a tall window still clips its own top", + "viewport": { "x": 0, "y": 0, "width": 1080, "height": 2200 }, + "keyboard": { "x": 0, "y": 1700, "width": 1080, "height": 500 }, + "expected": { + "kind": "avoided", + "viewport": { "x": 0, "y": 0, "width": 1080, "height": 1688 }, + "keyboardMinY": 1700 + } + }, + { + "name": "empty keyboard frame fails open: a keyboard the platform cannot measure is not evidence of occlusion", + "viewport": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "keyboard": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "expected": { "kind": "unobstructed" } + }, + { + "name": "empty viewport frame fails open too: resolving the best frame is the caller's job", + "viewport": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "keyboard": { "x": 0, "y": 564, "width": 402, "height": 310 }, + "expected": { "kind": "unobstructed" } + } + ] +} diff --git a/packages/contracts/src/scroll-command.ts b/packages/contracts/src/scroll-command.ts index 73b27385bb..59b2146b1a 100644 --- a/packages/contracts/src/scroll-command.ts +++ b/packages/contracts/src/scroll-command.ts @@ -118,4 +118,15 @@ export type ScrollCommandResult = { durationMs?: number; message?: string; settle?: SettleObservation; + /** + * Set only when an on-screen keyboard made the owner clip the swipe into the band above it + * (#2500). Absent means the swipe was not clipped, which is not the same claim as `false`: a + * platform that never runs the clip has nothing to report. The platform leaf's `referenceHeight` + * names the shortened axis the reported `pixels` were planned against, and `keyboardMinY` names + * where the keyboard began. A surface the owner refused to swipe at all fails instead, under the + * `scroll_keyboard_occludes_surface` reason. + */ + keyboardAvoided?: true; + /** The keyboard's edge in the same unit as the gesture coordinates, when the swipe was clipped. */ + keyboardMinY?: number; }; diff --git a/packages/contracts/src/scroll-gesture.test.ts b/packages/contracts/src/scroll-gesture.test.ts index 452ba476f4..1c5f700560 100644 --- a/packages/contracts/src/scroll-gesture.test.ts +++ b/packages/contracts/src/scroll-gesture.test.ts @@ -3,11 +3,17 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { assertScrollGestureInput, buildInPageSwipeGesturePlan, buildScrollGesturePlan, clampGestureCoordinate, + SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE, + SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, + clipScrollViewportAboveKeyboard, + scrollKeyboardOccludesSurfaceError, } from './scroll-gesture.ts'; import { DEFAULT_IOS_SCROLL_AMOUNT, @@ -201,3 +207,114 @@ test('clampGestureCoordinate rounds values and clamps them into the safe gesture test('clampGestureCoordinate returns the lower bound for non-finite coordinates', () => { assert.equal(clampGestureCoordinate(Number.POSITIVE_INFINITY, 8, 100), 8); }); + +// Golden parity table: the SAME JSON is asserted against the Swift twin +// (ScrollViewportPolicy in apple/runner/AgentDeviceRunner/ +// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in the same file), so +// a clip that drifts between the iOS runner and the Android/TS owner turns CI red on whichever +// side changed, without a simulator. + +type ExpectedClip = + | { kind: 'unobstructed' } + | { kind: 'avoided'; viewport: Rect; keyboardMinY: number } + | { kind: 'occluded'; keyboardMinY: number; visibleHeight: number }; + +type FixtureCase = { + name: string; + viewport: Rect; + keyboard: Rect; + expected: ExpectedClip; +}; + +type Fixture = { + constants: { + minVisibleFraction: number; + accessoryAllowance: number; + occlusionReason: string; + }; + cases: FixtureCase[]; +}; + +const TABLE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'scroll-keyboard-policy.json', +); + +function loadTable(): Fixture { + return JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as Fixture; +} + +test('the scroll keyboard clip agrees with every golden parity table case', () => { + const table = loadTable(); + assert.ok(table.cases.length > 0, 'parity table must not be empty'); + const names = new Set(table.cases.map((fixture) => fixture.name)); + assert.equal(names.size, table.cases.length, 'parity table case names must be unique'); + for (const fixture of table.cases) { + assert.deepEqual( + clipScrollViewportAboveKeyboard(fixture.viewport, fixture.keyboard), + fixture.expected, + fixture.name, + ); + } +}); + +test('the keyboard clip thresholds and refusal keys belong to the table, not this file', () => { + const { constants } = loadTable(); + assert.equal(constants.minVisibleFraction, SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION); + assert.equal(constants.accessoryAllowance, SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE); + assert.equal(constants.occlusionReason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); +}); + +test('a clipped viewport keeps even a saturated swipe inside the visible band', () => { + // The whole point of clipping before planning: `buildScrollGesturePlan` is untouched and gets the + // clipped axis, so its own edge padding is what holds the gesture above the keyboard. + const viewport: Rect = { x: 0, y: 0, width: 402, height: 874 }; + const clip = clipScrollViewportAboveKeyboard(viewport, { + x: 0, + y: 564, + width: 402, + height: 310, + }); + assert.equal(clip.kind, 'avoided'); + if (clip.kind !== 'avoided') return; + const plan = buildScrollGesturePlan({ + direction: 'down', + amount: 10, + referenceWidth: clip.viewport.width, + referenceHeight: clip.viewport.height, + }); + assert.ok(Math.max(plan.y1, plan.y2) <= clip.viewport.height); + assert.ok(plan.pixels < clip.viewport.height, 'honored travel must name the clipped axis'); +}); + +test('an unusable keyboard frame fails open instead of refusing every scroll', () => { + const viewport: Rect = { x: 0, y: 0, width: 402, height: 874 }; + for (const keyboard of [ + { x: 0, y: Number.NaN, width: 402, height: 310 }, + { x: 0, y: 564, width: Number.POSITIVE_INFINITY, height: 310 }, + { x: 0, y: 564, width: 402, height: -310 }, + { x: 0, y: 564, width: 402, height: 0 }, + ] satisfies Rect[]) { + assert.equal(clipScrollViewportAboveKeyboard(viewport, keyboard).kind, 'unobstructed'); + } +}); + +test('the occlusion refusal is keyed on its reason, not on its message', () => { + const error = scrollKeyboardOccludesSurfaceError('down', { + keyboardMinY: 564, + visibleHeight: 40, + viewportHeight: 874, + }); + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); + assert.equal(error.details?.keyboardMinY, 564); + assert.equal(error.details?.visibleHeight, 40); + assert.equal(error.details?.viewportHeight, 874); + assert.match(String(error.details?.hint), /keyboard dismiss/); +}); diff --git a/packages/contracts/src/scroll-gesture.ts b/packages/contracts/src/scroll-gesture.ts index cdc7ddc971..5b8be2fe8f 100644 --- a/packages/contracts/src/scroll-gesture.ts +++ b/packages/contracts/src/scroll-gesture.ts @@ -329,3 +329,137 @@ export function clampGestureCoordinate(value: number, marginPx: number, size: nu return Math.min(max, Math.max(min, Math.round(value))); } + +/** + * What an on-screen keyboard leaves of the scroll viewport, and what happens when it leaves too + * little. + * + * `buildScrollGesturePlan` centres a directional swipe, so a focused field puts its lower endpoint + * under the keyboard: the gesture lands on keys, the surface never moves, and the edge loop reads a + * stuck container (#2499) instead of a refusal. Reducing the reference height BEFORE the planner + * runs keeps the swipe inside what is visible without touching the planner, so reported travel + * stays honest. The runner and the Android helper each read their own live keyboard frame; a frame + * threaded from the daemon would be a snapshot that predates the keyboard. + * + * Pure geometry, so the decision is proven against a golden table both this file and the Swift twin + * (`ScrollViewportPolicy` in apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/ + * RunnerScrollViewportPolicy.swift) assert against, so drift turns CI red without a simulator. The + * table carries only frames representable in both languages: `CGRect` standardizes a negative + * extent into a positive height at a moved origin, so a negative `height` is tested here alone. + */ + +/** Below this fraction of the viewport, the clipped band cannot hold a reliable swipe. */ +export const SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION = 0.15; + +/** + * A fixed allowance kept above the keyboard's top edge, in the frame's own unit (points on iOS, + * pixels on Android). `keyboard.frame` reports the key plane, not the input accessory or composer + * bar riding above it, so a swipe that ends exactly at the reported edge can still land on a bar. + */ +export const SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE = 12; + +/** + * The one reason a directional scroll refuses to swipe at all (#2500), with the hint every owner + * publishes beside it: Android measures the occlusion in this process, and the iOS runner answers + * with its own runner code that the Apple scroll owner joins to these same details, so a + * caller branches on `reason` and reads one hint whichever owner refused. Avoidance never dismisses + * the keyboard: a dismiss drops focus, which breaks a `type`/`scroll`/`type` loop, is not idempotent + * across platforms, and mutates state session-action provenance does not record. So the hint names + * the tradeoff instead of paying it. + */ +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON = 'scroll_keyboard_occludes_surface'; + +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS = Object.freeze({ + reason: SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON, + hint: + 'The on-screen keyboard covers the surface this scroll would swipe, so it cannot reach it. ' + + 'Run `keyboard dismiss` and retry, accepting that it drops focus (re-tap the field to keep typing), or scroll before focusing the field.', +}); + +export type ScrollKeyboardClip = + /** No keyboard, or one that does not own this surface: swipe the whole viewport. */ + | { kind: 'unobstructed' } + /** The viewport trimmed above the keyboard. Report the reduced reference height honestly. */ + | { kind: 'avoided'; viewport: Rect; keyboardMinY: number } + /** + * Too little surface left above the keyboard to swipe. Swiping anyway reads as a stuck + * container to the no-progress fingerprint, so the caller refuses with + * `SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON` instead. + */ + | { kind: 'occluded'; keyboardMinY: number; visibleHeight: number }; + +/** The numbers a refusing owner names about the surface it declined to swipe. */ +export type ScrollKeyboardOcclusion = { + keyboardMinY: number; + visibleHeight: number; + viewportHeight: number; +}; + +/** + * Clips a scroll viewport to the band above an occluding keyboard. + * + * Fails open on an unusable frame: a keyboard the platform cannot measure is not evidence that the + * surface is blocked, and turning a missing frame into a refusal would refuse every scroll on a + * device with a broken keyboard query. The floor is the caller's refusal, not this rule. + */ +export function clipScrollViewportAboveKeyboard( + viewport: Rect, + keyboard: Rect, +): ScrollKeyboardClip { + if (!isMeasurableRect(viewport) || !isMeasurableRect(keyboard)) return { kind: 'unobstructed' }; + // A vertical swipe runs along the viewport's centre line, which is the only part of the width the + // keyboard has to reach to be struck: a 320pt keyboard centred in an 834pt viewport is 38% of the + // width and sits exactly in the path. A horizontal swipe runs along the OTHER centre line, so + // clipping it too is early rather than wrong — it only lifts the swipe clear of the keys. + const swipeCenterX = viewport.x + viewport.width / 2; + if (swipeCenterX < keyboard.x || swipeCenterX >= keyboard.x + keyboard.width) { + return { kind: 'unobstructed' }; + } + const keyboardMinY = keyboard.y; + if (keyboardMinY >= viewport.y + viewport.height || keyboard.y + keyboard.height <= viewport.y) { + return { kind: 'unobstructed' }; + } + const visibleHeight = Math.max( + 0, + keyboardMinY - SCROLL_KEYBOARD_ACCESSORY_ALLOWANCE - viewport.y, + ); + if (visibleHeight < SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION * viewport.height) { + return { kind: 'occluded', keyboardMinY, visibleHeight }; + } + return { + kind: 'avoided', + viewport: { ...viewport, height: visibleHeight }, + keyboardMinY, + }; +} + +/** + * The refusal an owner that measured the keyboard in this process reports. The iOS runner measures + * in its own coordinate space and answers with its runner code instead; the Apple scroll owner + * adds the same `SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS` to that error. + */ +export function scrollKeyboardOccludesSurfaceError( + direction: ScrollDirection, + occlusion: ScrollKeyboardOcclusion, +): AppError { + const percent = Math.round(SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION * 100); + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} refused: the keyboard leaves ${occlusion.visibleHeight}px of ${occlusion.viewportHeight}px visible, below the ${percent}% needed for a swipe`, + { + keyboardMinY: occlusion.keyboardMinY, + visibleHeight: occlusion.visibleHeight, + viewportHeight: occlusion.viewportHeight, + minVisibleFraction: SCROLL_KEYBOARD_MIN_VISIBLE_FRACTION, + ...SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, + }, + ); +} + +function isMeasurableRect(rect: Rect): boolean { + return ( + [rect.x, rect.y, rect.width, rect.height].every((value) => Number.isFinite(value)) && + rect.width > 0 && + rect.height > 0 + ); +} diff --git a/packages/platform-apple/src/core/__tests__/scroll.test.ts b/packages/platform-apple/src/core/__tests__/scroll.test.ts new file mode 100644 index 0000000000..599860cf7c --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/scroll.test.ts @@ -0,0 +1,91 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON } from '@agent-device/contracts/scroll-gesture'; +import { + normalizeAppleScrollResultWithResolvedFrame, + withAppleScrollKeyboardOcclusion, +} from '../scroll.ts'; + +test('a clipped runner frame yields travel and evidence for the band actually swiped', () => { + // The runner clips the interaction frame above the keyboard and reports the clipped axis, so the + // TS recomputation of `pixels` must be honest about the shorter travel (#2500). Reading the + // unclipped screen height here would over-report travel the surface never had. + const result = normalizeAppleScrollResultWithResolvedFrame( + { + x: 201, + y: 500, + x2: 201, + y2: 100, + referenceWidth: 402, + referenceHeight: 552, + keyboardAvoided: true, + keyboardMinY: 564, + }, + 'down', + { amount: 0.65, durationMs: 400 }, + ); + assert.equal(result.referenceHeight, 552); + assert.equal(result.pixels, Math.round(552 * 0.65)); + assert.equal(result.keyboardAvoided, true); + assert.equal(result.keyboardMinY, 564); +}); + +test('an unclipped scroll reports no avoidance evidence rather than a fabricated false', () => { + // A plain `false` could not tell "no keyboard was up" from "this platform never runs the clip", + // so absence is the negative case, and the schema has to keep both distinguishable. + const result = normalizeAppleScrollResultWithResolvedFrame( + { x: 201, y: 500, x2: 201, y2: 100, referenceWidth: 402, referenceHeight: 874 }, + 'down', + { amount: 0.65 }, + ); + assert.equal('keyboardAvoided' in result, false); + assert.equal('keyboardMinY' in result, false); +}); + +test('avoidance from a runner that reports no keyboard edge is still avoidance', () => { + // `keyboardMinY` is additive evidence; a runner build that clipped without naming the edge must not + // lose the fact that it clipped at all. + const result = normalizeAppleScrollResultWithResolvedFrame( + { + x: 201, + y: 500, + x2: 201, + y2: 100, + referenceWidth: 402, + referenceHeight: 552, + keyboardAvoided: true, + }, + 'down', + { amount: 0.65 }, + ); + assert.equal(result.keyboardAvoided, true); + assert.equal('keyboardMinY' in result, false); +}); + +test('the runner keyboard refusal gains the shared reason and hint and keeps its own message', () => { + const runnerError = new AppError( + 'COMMAND_FAILED', + 'scroll down refused: the keyboard leaves 28pt of visible surface above it', + { runnerErrorCode: 'SCROLL_KEYBOARD_OCCLUDES_SURFACE', logPath: '/tmp/runner.log' }, + ); + const mapped = withAppleScrollKeyboardOcclusion(runnerError); + assert.ok(mapped instanceof AppError); + assert.equal(mapped.code, 'COMMAND_FAILED'); + assert.equal(mapped.message, runnerError.message); + assert.equal(mapped.details?.reason, SCROLL_KEYBOARD_OCCLUDES_SURFACE_REASON); + assert.match(String(mapped.details?.hint), /keyboard dismiss/); + assert.equal(mapped.details?.logPath, '/tmp/runner.log'); + assert.equal(mapped.details?.runnerErrorCode, 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'); +}); + +test('only the refusal code is renamed, so a generic scroll failure never reads as an occlusion', () => { + const generic = new AppError('COMMAND_FAILED', 'scroll could not resolve a usable frame', { + logPath: '/tmp/runner.log', + }); + assert.equal(withAppleScrollKeyboardOcclusion(generic), generic); + const alert = new AppError('COMMAND_FAILED', 'no alert', { runnerErrorCode: 'ALERT_NOT_FOUND' }); + assert.equal(withAppleScrollKeyboardOcclusion(alert), alert); + const transport = new Error('socket hang up'); + assert.equal(withAppleScrollKeyboardOcclusion(transport), transport); +}); diff --git a/packages/platform-apple/src/core/scroll.ts b/packages/platform-apple/src/core/scroll.ts index daeb70dd49..81a0fdff2f 100644 --- a/packages/platform-apple/src/core/scroll.ts +++ b/packages/platform-apple/src/core/scroll.ts @@ -6,8 +6,11 @@ import { } from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, buildScrollGesturePlan, } from '@agent-device/contracts/scroll-gesture'; +import { AppError } from '@agent-device/kernel/errors'; +import { SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE } from '../runner/runner-contract.ts'; export type NormalizedScrollOptions = { amount?: number; @@ -18,6 +21,23 @@ export type NormalizedScrollOptions = { export type AppleScrollOptions = ScrollExecutionOptions; +/** + * Gives the runner's keyboard-occlusion refusal the shared reason and hint (#2500). The runner + * measured the keyboard in its own coordinate space, so its message and transport details + * (`runnerErrorCode`, `logPath`) are kept as they are; only the details every platform publishes + * are added, matched on the typed runner code rather than on error text. + */ +export function withAppleScrollKeyboardOcclusion(error: unknown): unknown { + if (!(error instanceof AppError)) return error; + if (error.details?.['runnerErrorCode'] !== SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE) { + return error; + } + return new AppError(error.code, error.message, { + ...error.details, + ...SCROLL_KEYBOARD_OCCLUDES_SURFACE_DETAILS, + }); +} + export function materializeIosScrollOptions( options: AppleScrollOptions | undefined, ): AppleScrollOptions { @@ -91,6 +111,7 @@ export function normalizeAppleScrollResult( const verticalTravel = y1 !== undefined && y2 !== undefined ? Math.round(Math.abs(y2 - y1)) : undefined; const travelPixels = selectScrollTravelPixels(options, horizontalTravel, verticalTravel); + const keyboardMinY = readFiniteNumber(runnerResult.keyboardMinY); return { ...(x1 !== undefined ? { x1 } : {}), @@ -99,6 +120,11 @@ export function normalizeAppleScrollResult( ...(y2 !== undefined ? { y2 } : {}), ...(referenceWidth !== undefined ? { referenceWidth } : {}), ...(referenceHeight !== undefined ? { referenceHeight } : {}), + // Avoidance evidence (#2500) is reported only when it happened: `referenceHeight` above already + // names the clipped axis, and a plain `false` here could not tell "no keyboard" apart from a + // platform that never runs the clip. + ...(runnerResult.keyboardAvoided === true ? { keyboardAvoided: true } : {}), + ...(keyboardMinY !== undefined ? { keyboardMinY } : {}), ...(options?.amount !== undefined ? { amount: options.amount } : {}), ...(travelPixels !== undefined ? { pixels: travelPixels } : {}), ...(options?.durationMs !== undefined ? { durationMs: options.durationMs } : {}), diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index 515b4189cb..c6e5f07f46 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -32,6 +32,7 @@ import { normalizeAppleScrollResult, normalizeAppleScrollResultWithResolvedFrame, scrollRunnerFields, + withAppleScrollKeyboardOcclusion, type AppleScrollOptions, } from './core/scroll.ts'; import { runMacosDesktopScroll } from './os/macos/desktop-scroll.ts'; @@ -429,16 +430,23 @@ async function runAppleScroll( const iosOptions = materializeIosScrollOptions(options); // Single fused lifecycle command: the runner resolves the interaction frame and runs the drag. - const runnerResult = await runRunnerCommand( - device, - { - command: 'scroll', - direction, - ...scrollRunnerFields(iosOptions), - appBundleId: ctx.appBundleId, - }, - runnerOpts, - ); + // The runner is also the only party holding the live keyboard frame, so a surface it refuses to + // swipe under the keys arrives here as its typed runner code and leaves as a typed reason (#2500). + let runnerResult: Record; + try { + runnerResult = await runRunnerCommand( + device, + { + command: 'scroll', + direction, + ...scrollRunnerFields(iosOptions), + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + } catch (error) { + throw withAppleScrollKeyboardOcclusion(error); + } return normalizeAppleScrollResultWithResolvedFrame(runnerResult, direction, iosOptions); } diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 1e954b612e..649ffce8f1 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -323,14 +323,24 @@ export function shouldRestartRunnerBeforeCommandSend(error: unknown): boolean { ); } +/** + * The code the XCTest runner answers with when it declines to place a scroll gesture under the + * on-screen keyboard (#2500). It is the runner's own vocabulary, so it is declared here beside the + * set that keeps it off the wire, and the Apple scroll owner matches it on `details.runnerErrorCode` + * rather than on error text. + */ +export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCCLUDES_SURFACE'; + /** * Runner codes that classify a failure for the host without renaming it on the wire. They stay * `COMMAND_FAILED` and survive as `details.runnerErrorCode`, which is what family policy reads: - * `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there yet. + * `RUNNER_BUSY` for retriable contention, `ALERT_NOT_FOUND` for an alert that is not there yet, and + * the scroll keyboard refusal for a surface the runner declined to swipe under the keys. */ const DIAGNOSTIC_ONLY_RUNNER_ERROR_CODES: ReadonlySet = new Set([ RUNNER_BUSY_RUNNER_CODE, ALERT_NOT_FOUND_RUNNER_CODE, + SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, ]); /** Wire code plus the details every path must publish for one runner-reported error code. */ diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 2e78dd567b..d7eb7a8cce 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -68,7 +68,7 @@ const interactionCommandDescriptions = { 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.', scroll: - 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.', + 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band instead of being dismissed; when too little is left, the command refuses with scroll_keyboard_occludes_surface.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 47b34efccc..20e83f6b49 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -554,6 +554,12 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { pixels: numberSchema(), durationMs: numberSchema(), message: stringSchema(), + keyboardAvoided: booleanSchema( + 'Present only when an on-screen keyboard forced the swipe into the band above it; the reported pixels were planned against the shorter referenceHeight.', + ), + keyboardMinY: numberSchema( + 'Where the keyboard began, in the same unit as the gesture coordinates. Clipped scrolls only.', + ), }, ['direction'], ), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3e1e6a1219..4c389e95c9 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -492,6 +492,7 @@ Target-authored drag is supported on Android touch devices and iOS/iPadOS. Backe On iOS simulators it uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path, so verify app-level metrics instead of assuming the requested values map exactly to recognizer output. On Android, `gesture transform` injects a geometric two-finger path. App recognizers may report non-exact pan, scale, and rotation values, so verify qualitative state such as `pan changed yes`, `pinch changed yes`, and `rotate changed yes` unless the app explicitly promises exact centroid metrics. If exact app-state values matter, prefer isolated `gesture pan`, `gesture pinch`, or `gesture rotate` commands. `scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. Directional scrolls decelerate through the drag on Android to reduce release momentum within the requested duration; `scroll top` and `scroll bottom` retain inertial release for edge traversal. Reduced momentum does not guarantee an exact content offset, especially for very short gestures: apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS. +A directional scroll places its swipe across the middle of the viewport, so a focused field and its keyboard would put the swipe under the keys: the gesture would land on the keyboard, the surface would not move, and the scroll would read as stuck. On iOS the scroll instead keeps the whole swipe in the band above the keyboard, reporting `keyboardAvoided` and `keyboardMinY` alongside a `referenceHeight` and `pixels` measured against that shorter band. It never dismisses the keyboard, because dismissing drops focus and breaks a `fill`/`scroll`/`fill` loop; run `keyboard dismiss` yourself when you want that. When the keyboard leaves too little room to swipe, the command refuses with the `scroll_keyboard_occludes_surface` reason rather than swiping into the keys, so a scroll that cannot work says so instead of appearing stuck. Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs. When a target only appears in an off-screen summary, use `scroll --settle`: the response waits for the UI to go quiet and returns the diff against the tree you last observed, with fresh refs on the added lines, so no follow-up `snapshot -i` is needed. `back --settle` does the same for navigation. Both are best-effort and never fail the action. For repeated checks without settle, a small shell loop is enough: