fix(android): ease controlled scrolls within the requested duration

Author: thymikee

packages/platform-android/src/__tests__/input-actions.test.ts

@@ -95,6 +95,64 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi
   );
 });
 
+test.each([undefined, 'inertial'] as const)(
+  'scrollAndroid preserves path and duration with %s release',
+  async (releaseBehavior) => {
+    const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
+    await withAndroidAdbProvider(
+      {
+        exec: async () => {
+          throw new Error('adb must not run');
+        },
+        gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
+        touch: async (request) => {
+          touchCalls.push(request);
+        },
+      },
+      { serial: ANDROID_EMULATOR.id },
+      async () => {
+        for (const direction of ['up', 'down', 'left', 'right'] as const) {
+          for (const durationMs of [16, 120, 300, 9841, 10000]) {
+            await scrollAndroid(ANDROID_EMULATOR, direction, {
+              pixels: 240,
+              durationMs,
+              releaseBehavior,
+            });
+          }
+        }
+      },
+    );
+    for (const touch of touchCalls) {
+      const samples = touch.pointers[0]!.samples;
+      const start = samples[0]!;
+      const end = samples.at(-1)!;
+      assert.equal(start.offsetMs, 0);
+      assert.equal(end.offsetMs, touch.durationMs);
+      const distance = (a: typeof start, b: typeof start) =>
+        Math.hypot(b.point.x - a.point.x, b.point.y - a.point.y);
+      assert.equal(distance(start, end), 240);
+      const velocities = samples
+        .slice(1)
+        .map(
+          (sample, index) =>
+            distance(samples[index]!, sample) / (sample.offsetMs - samples[index]!.offsetMs),
+        );
+      if (releaseBehavior === 'inertial') {
+        for (const velocity of velocities) assert.ok(Math.abs(velocity - velocities[0]!) < 1e-8);
+        continue;
+      }
+      const firstMove = samples[1]!;
+      assert.ok(
+        distance(start, firstMove) <= ((240 * firstMove.offsetMs) / touch.durationMs) * 1.1,
+      );
+      assert.ok(velocities.at(-1)! < Math.max(...velocities) / 2);
+      for (let i = Math.ceil(velocities.length / 2); i < velocities.length; i += 1) {
+        assert.ok(velocities[i]! <= velocities[i - 1]! + 1e-8);
+      }
+    }
+  },
+);
+
 test('longPressAndroid sends a stationary semantic touch plan', async () => {
   const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
   const result = await withAndroidAdbProvider(

packages/platform-android/src/input-actions.ts

@@ -5,7 +5,10 @@
 import { DEVICE_ROTATION_SURFACE_INDEX, type DeviceRotation } from '@agent-device/contracts/device';
 import { buildGesturePlan } from '@agent-device/contracts/gesture-plan';
 import { GESTURE_DURATION_MIN_MS } from '@agent-device/contracts/gesture-plan-types';
-import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command';
+import {
+  type ScrollReleaseBehavior,
+  DEFAULT_MOBILE_SCROLL_DURATION_MS,
+} from '@agent-device/contracts/scroll-command';
 import {
   type ScrollDirection,
   buildScrollGesturePlan,
@@ -163,7 +166,12 @@ export async function focusAndroid(device: DeviceInfo, x: number, y: number): Pr
 export async function scrollAndroid(
   device: DeviceInfo,
   direction: ScrollDirection,
-  options?: { amount?: number; pixels?: number; durationMs?: number } & AndroidHelperSessionOptions,
+  options?: {
+    amount?: number;
+    pixels?: number;
+    durationMs?: number;
+    releaseBehavior?: ScrollReleaseBehavior;
+  } & AndroidHelperSessionOptions,
 ): Promise<Record<string, unknown>> {
   // The viewport read and the gesture are two helper calls one command apart: giving the read the
   // command's session scope keeps both on the same instrumentation.
@@ -192,9 +200,8 @@ export async function scrollAndroid(
     options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS,
     GESTURE_DURATION_MIN_MS,
   );
-  const backend = await executeAndroidTouchPlan(
-    device,
-    buildGesturePlan(
+  const backend = await executeAndroidTouchPlan(device, {
+    ...buildGesturePlan(
       {
         intent: 'pan',
         origin: { x: scrollPlan.x1, y: scrollPlan.y1 },
@@ -207,7 +214,8 @@ export async function scrollAndroid(
       viewport,
       'android',
     ),
-  );
+    releaseBehavior: options?.releaseBehavior ?? 'controlled',
+  });
 
   return {
     ...scrollPlan,

packages/platform-android/src/touch-plan-lowering.ts

@@ -1,4 +1,6 @@
+import type { ScrollReleaseBehavior } from '@agent-device/contracts/scroll-command';
 import {
+  GESTURE_SAMPLE_INTERVAL_MS,
   interpolateGesturePoint,
   sampleGestureOffsets,
 } from '@agent-device/contracts/gesture-plan';
@@ -18,7 +20,9 @@ export type AndroidLongPressTouchPlan = {
   pointers: readonly [PointerTrajectory];
 };
 
-export type AndroidTouchPlan = GesturePlan | AndroidLongPressTouchPlan;
+export type AndroidTouchPlan =
+  | (GesturePlan & { releaseBehavior?: ScrollReleaseBehavior })
+  | AndroidLongPressTouchPlan;
 
 /**
  * Transport samples are strictly denser than the canonical endpoint pair, so the shared plan a
@@ -57,7 +61,8 @@ export type AndroidProviderTouchPlan =
 export function lowerAndroidTouchPlan(plan: AndroidTouchPlan): AndroidLoweredTouchPlan {
   if (plan.topology === 'two' || plan.intent === 'longPress') return plan;
 
-  const [{ pointerId, samples: canonicalSamples }] = plan.pointers;
+  const { releaseBehavior, ...gesturePlan } = plan;
+  const [{ pointerId, samples: canonicalSamples }] = gesturePlan.pointers;
   const offsets = [
     ...new Set([
       ...sampleGestureOffsets(plan.durationMs, 'android'),
@@ -75,6 +80,7 @@ export function lowerAndroidTouchPlan(plan: AndroidTouchPlan): AndroidLoweredTou
     const start = canonicalSamples[segmentIndex]!;
     const end = canonicalSamples[segmentIndex + 1]!;
     const segmentDurationMs = end.offsetMs - start.offsetMs;
+    const progress = (offsetMs - start.offsetMs) / segmentDurationMs;
     return {
       offsetMs,
       point:
@@ -83,15 +89,24 @@ export function lowerAndroidTouchPlan(plan: AndroidTouchPlan): AndroidLoweredTou
           : interpolateGesturePoint(
               start.point,
               end.point,
-              (offsetMs - start.offsetMs) / segmentDurationMs,
+              releaseBehavior === 'controlled'
+                ? controlledScrollProgress(progress, segmentDurationMs)
+                : progress,
             ),
     };
   });
   // The sampler floors the frame count at three, so `dense` always holds at least four samples.
   const samples: AndroidTransportSamples = [dense[0]!, dense[1]!, dense[2]!, ...dense.slice(3)];
 
   return {
-    ...plan,
+    ...gesturePlan,
     pointers: [{ pointerId, samples }],
   };
 }
+
+function controlledScrollProgress(progress: number, durationMs: number): number {
+  const accelerationFraction = Math.min(GESTURE_SAMPLE_INTERVAL_MS / durationMs, 0.5);
+  return progress < accelerationFraction
+    ? progress ** 2 / accelerationFraction
+    : 1 - (1 - progress) ** 2 / (1 - accelerationFraction);
+}

src/__tests__/test-utils/property-arbitraries.ts

@@ -77,6 +77,13 @@ const viewportRectArb: fc.Arbitrary<Rect> = fc.oneof(
   }),
 );
 
+export const scrollInViewportArb = fc.record({
+  viewport: viewportRectArb.filter(({ width, height }) => width >= 32 && height >= 32),
+  direction: fc.constantFrom(...SCROLL_DIRECTIONS),
+  durationMs: fc.integer({ min: 16, max: 10000 }),
+  pixels: fc.integer({ min: 1, max: 2000 }),
+});
+
 export const scrollingContainerTypeArb = fc.constantFrom(
   'XCUIElementTypeScrollView',
   'XCUIElementTypeTable',

src/commands/interaction/metadata.ts

@@ -68,7 +68,7 @@ const interactionCommandDescriptions = {
     'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.',
   type: 'Append text to the currently focused input. Use fill when the existing field value should be replaced, and focus first when no input is active.',
   scroll:
-    'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; app scroll physics determine the final content offset.',
+    'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; directional scrolls reduce release momentum, while app scroll physics determine the final content offset.',
   get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.',
   is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.',
   find: 'Find by text/label/value/role/id and run action',

src/core/__tests__/gesture-plan-viewport.test.ts

@@ -1,3 +1,5 @@
+import { lowerAndroidTouchPlan } from '@agent-device/platform-android/mechanics';
+import { buildScrollGesturePlan } from '@agent-device/contracts/scroll-gesture';
 import {
   gesturePayloadFromPositionals,
   normalizePublicGesture,
@@ -11,6 +13,7 @@ import { describe, test } from 'vitest';
 import {
   COMPACT_VIEWPORTS,
   gestureInViewportArb,
+  scrollInViewportArb,
   PROPERTY_RUNS_SMALL,
 } from '../../__tests__/test-utils/property-arbitraries.ts';
 import {
@@ -206,3 +209,52 @@ function buildTransformPlan(durationMs: number) {
     ),
   );
 }
+
+test('Android controlled scroll sampling preserves the inertial path and viewport bounds', () => {
+  fc.assert(
+    fc.property(scrollInViewportArb, ({ viewport, direction, durationMs, pixels }) => {
+      const scroll = buildScrollGesturePlan({
+        direction,
+        pixels,
+        referenceWidth: viewport.width,
+        referenceHeight: viewport.height,
+      });
+      const plan = buildGesturePlan(
+        {
+          intent: 'pan',
+          origin: { x: viewport.x + scroll.x1, y: viewport.y + scroll.y1 },
+          delta: { x: scroll.x2 - scroll.x1, y: scroll.y2 - scroll.y1 },
+          durationMs,
+        },
+        viewport,
+        'android',
+      );
+      const controlled = lowerAndroidTouchPlan({ ...plan, releaseBehavior: 'controlled' });
+      const inertial = lowerAndroidTouchPlan({ ...plan, releaseBehavior: 'inertial' });
+      assert.equal(controlled.durationMs, durationMs);
+      assert.equal(inertial.durationMs, durationMs);
+      const samples = controlled.pointers[0].samples;
+      const linear = inertial.pointers[0].samples;
+      assert.deepEqual(
+        samples.map(({ offsetMs }) => offsetMs),
+        linear.map(({ offsetMs }) => offsetMs),
+      );
+      assert.deepEqual(samples[0], linear[0]);
+      assert.deepEqual(samples.at(-1), linear.at(-1));
+      for (const axis of ['x', 'y'] as const) {
+        const from = samples[0]!.point[axis];
+        const to = samples.at(-1)!.point[axis];
+        for (let i = 1; i < samples.length; i += 1) {
+          const sample = samples[i]!;
+          assert.ok(
+            sample.point[axis] >= Math.min(from, to) && sample.point[axis] <= Math.max(from, to),
+          );
+          assert.ok((sample.point[axis] - samples[i - 1]!.point[axis]) * (to - from) >= 0);
+          const expectedLinear = from + ((to - from) * sample.offsetMs) / durationMs;
+          assert.ok(Math.abs(linear[i]!.point[axis] - expectedLinear) < 1e-8);
+        }
+      }
+    }),
+    { numRuns: PROPERTY_RUNS_SMALL },
+  );
+});

website/docs/docs/commands.md

@@ -485,7 +485,7 @@ Target-authored drag is supported on Android touch devices and iOS/iPadOS. Backe
 `gesture transform` accepts `x y dx dy scale degrees [durationMs]` for one combined two-finger pan/zoom/rotate gesture on Android and iOS simulators. Pinch, rotate, two-finger pan, and transform use the same viewport-aware pointer planning; impossible paths fail before injection instead of clamping or distorting the requested motion.
 On iOS simulators it uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path, so verify app-level metrics instead of assuming the requested values map exactly to recognizer output.
 On Android, `gesture transform` injects a geometric two-finger path. App recognizers may report non-exact pan, scale, and rotation values, so verify qualitative state such as `pan changed yes`, `pinch changed yes`, and `rotate changed yes` unless the app explicitly promises exact centroid metrics. If exact app-state values matter, prefer isolated `gesture pan`, `gesture pinch`, or `gesture rotate` commands.
-`scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels <n>` for a fixed-distance gesture. The final content offset can differ because apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS.
+`scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels <n>` for a fixed-distance gesture. Directional scrolls decelerate through the drag on Android to reduce release momentum within the requested duration; `scroll top` and `scroll bottom` retain inertial release for edge traversal. Reduced momentum does not guarantee an exact content offset, especially for very short gestures: apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS.
 Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs.
 When a target only appears in an off-screen summary, use `scroll <direction> --settle`: the response waits for the UI to go quiet and returns the diff against the tree you last observed, with fresh refs on the added lines, so no follow-up `snapshot -i` is needed. `back --settle` does the same for navigation. Both are best-effort and never fail the action. For repeated checks without settle, a small shell loop is enough: