Skip to content

Commit 865145e

Browse files
author
agent
committed
refactor: ride the Interactor seam for the element read; drop the bespoke host
Two operations of the same class were reaching their mechanics two different ways: `findText` rides `Interactor` via `localInteractors.resolve`, while `readTextAtPoint` had its own host port. That is duplication of MECHANISM, so the read now rides the same seam. `Interactor` gains `readTextAtPoint?`, implemented on the Apple, Android and Linux interactors where those mechanics already live. `src/platform-runtime-element-text-host.ts` and its `elementText` host wiring are deleted; the contract binds through the resolver exactly as the snapshot runtime does. Size honesty: this removes an 89-line module but the four readers still have to exist, so they moved into the interactors rather than vanishing. Net production change is ~4 lines, not ~89. The duplication of mechanism is what is actually fixed; Wave 5/6 retires the seam for both operations together. Also from the size investigation: - `ElementTextRuntimeExecution` was byte-identical to `SnapshotRuntimeExecution`; removed and reused, as `find-text-runtime.ts` does. - Removed a stranded, stale comment in `selector-capture-binding.ts` that still claimed a duplication this branch had already retired. - `FrozenUnavailablePlatformRuntimeFacts` is derived from its input type rather than restated, removing a 14-line clone group my new cell had pushed over the detector threshold.
1 parent 7a55965 commit 865145e

15 files changed

Lines changed: 136 additions & 139 deletions

File tree

packages/contracts/src/element-text-runtime.ts

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
import type { DeviceInfo } from '@agent-device/kernel/device';
22
import type { Point } from '@agent-device/kernel/snapshot';
3-
import type { RunnerContext } from './interactor-types.ts';
3+
import type { Interactor, RunnerContext } from './interactor-types.ts';
44
import type { RuntimeOperationFact } from './platform-runtime.ts';
55
import type { SessionSurface } from './session-surface.ts';
6-
7-
/** Runner metadata the selected read implementation needs, without request-owned state. */
8-
export type ElementTextRuntimeExecution = Readonly<Omit<RunnerContext, 'appBundleId' | 'signal'>>;
6+
import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts';
97

108
/**
119
* Neutral intent for one point-addressed element read. The point is already resolved from the
@@ -14,7 +12,8 @@ export type ElementTextRuntimeExecution = Readonly<Omit<RunnerContext, 'appBundl
1412
export type ReadTextAtPointInput = Readonly<{
1513
point: Point;
1614
options?: Readonly<{ appBundleId?: string; surface?: SessionSurface }>;
17-
execution?: ElementTextRuntimeExecution;
15+
/** Same runner metadata a capture needs; reuses that type rather than restating it. */
16+
execution?: SnapshotRuntimeExecution;
1817
}>;
1918

2019
/**
@@ -71,24 +70,47 @@ export function elementTextRuntimeOperationFacts(
7170
return Object.freeze({ readTextAtPoint: input.readTextAtPoint });
7271
}
7372

73+
/** Resolves the selected owner's interactor, exactly as the snapshot runtime does. */
74+
export type ElementTextInteractorResolver = (
75+
device: DeviceInfo,
76+
runner: RunnerContext,
77+
) => Promise<Interactor>;
78+
7479
/**
75-
* The existing per-family read mechanics, injected by composition. Families reach their own
76-
* tools through this port rather than importing root modules, matching the snapshot runtime's
77-
* interactor-resolver seam.
80+
* Binds the owner's live point read for the lifetime of a request binding.
81+
*
82+
* Rides the same `Interactor` seam `findText` uses rather than a bespoke host port: two
83+
* operations of the same class reaching their mechanics two different ways is duplication of
84+
* mechanism, and Wave 5/6 retires the seam for both together.
7885
*/
79-
export type ElementTextRuntimeHost = Readonly<{
80-
readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise<ElementTextReadOutcome>;
81-
}>;
82-
83-
/** Captures one selected owner's read authority for the lifetime of a request binding. */
8486
export function bindElementTextRuntime(
8587
params: Readonly<{
8688
device: DeviceInfo;
87-
host: ElementTextRuntimeHost;
89+
signal: AbortSignal;
90+
resolveInteractor: ElementTextInteractorResolver;
8891
}>,
8992
): ElementTextRuntimeOperations {
9093
return Object.freeze({
91-
readTextAtPoint: async (input: ReadTextAtPointInput) =>
92-
await params.host.readTextAtPoint(params.device, input),
94+
readTextAtPoint: async (input: ReadTextAtPointInput) => {
95+
const signal = params.signal;
96+
signal.throwIfAborted();
97+
const interactor = await params.resolveInteractor(params.device, {
98+
...input.execution,
99+
appBundleId: input.options?.appBundleId,
100+
signal,
101+
});
102+
// An owner whose facts advertised the read but whose interactor has none is a runtime
103+
// contract error surfaced as a declined read, not a silent empty answer.
104+
if (!interactor.readTextAtPoint) {
105+
return Object.freeze({ status: 'unreadable', reason: 'surface-not-readable' } as const);
106+
}
107+
return elementTextRead(
108+
await interactor.readTextAtPoint(input.point, {
109+
appBundleId: input.options?.appBundleId,
110+
surface: input.options?.surface,
111+
signal,
112+
}),
113+
);
114+
},
93115
});
94116
}

packages/contracts/src/facades/platform.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,7 @@ export {
283283
} from '../element-text-runtime.ts';
284284
export type {
285285
ElementTextReadOutcome,
286-
ElementTextRuntimeExecution,
287-
ElementTextRuntimeHost,
286+
ElementTextInteractorResolver,
288287
ElementTextRuntimeOperationFacts,
289288
ElementTextRuntimeOperations,
290289
ElementTextUnreadableReason,

packages/contracts/src/interactor-types.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,16 @@ export type Interactor = {
222222
screenshot(outPath: string, options?: ScreenshotOptions): Promise<void>;
223223
setViewport?(width: number, height: number): Promise<Record<string, unknown> | void>;
224224
snapshot(options?: SnapshotOptions): Promise<SnapshotResult>;
225+
/**
226+
* Native reading of the live text at a point, when the backend has one. Answers the text the
227+
* owner can see right now, which can exceed what an already-captured node carries (an editable
228+
* field whose value is longer than its label). Optional: a backend without it leaves the
229+
* captured tree as the complete answer.
230+
*/
231+
readTextAtPoint?(
232+
point: Point,
233+
options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal },
234+
): Promise<string | undefined>;
225235
gestureViewport?(): Promise<Rect>;
226236
back(mode?: BackMode): Promise<void>;
227237
home(): Promise<void>;

packages/contracts/src/platform-runtime-operations.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,7 @@ import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtim
1515
import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts';
1616
import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts';
1717
import type { ViewportRuntimeOperations } from './viewport-runtime.ts';
18-
import type {
19-
ElementTextRuntimeHost,
20-
ElementTextRuntimeOperations,
21-
} from './element-text-runtime.ts';
18+
import type { ElementTextRuntimeOperations } from './element-text-runtime.ts';
2219
import type {
2320
DeviceReadinessRuntimeHost,
2421
DeviceReadinessRuntimeOperations,
@@ -291,7 +288,6 @@ export type PlatformRuntimeHost = AppLogRuntimeHost &
291288
}>;
292289
screenRecording: ScreenRecordingRuntimeHost;
293290
snapshot: SnapshotRuntimeHost;
294-
elementText: ElementTextRuntimeHost;
295291
deviceReadiness: DeviceReadinessRuntimeHost;
296292
deviceShutdown: DeviceShutdownRuntimeHost;
297293
localInteractors: LocalApplicationInteractorHost;

packages/contracts/src/platform-runtime-unavailable.ts

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,14 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{
3535
lifecycle: ApplicationLifecycleOperationFacts;
3636
}>;
3737

38-
type FrozenUnavailablePlatformRuntimeFacts = Readonly<{
39-
appLog: RuntimeOperationUnavailability;
40-
apps: RuntimeOperationUnavailability;
41-
appDeployment: RuntimeOperationUnavailability;
42-
appState: RuntimeOperationUnavailability;
43-
network: RuntimeOperationUnavailability;
44-
screenRecording: RuntimeOperationUnavailability;
45-
screenshot: RuntimeOperationUnavailability;
46-
snapshot: RuntimeOperationUnavailability;
47-
viewport: RuntimeOperationUnavailability;
48-
elementText: RuntimeOperationUnavailability;
49-
readiness: RuntimeOperationUnavailability;
50-
shutdown: RuntimeOperationUnavailability;
51-
lifecycle: ApplicationLifecycleOperationFacts;
52-
}>;
38+
/**
39+
* The same cells with every optional one resolved. Derived from the input type rather than
40+
* restated, so a new cell cannot be added to one and forgotten in the other.
41+
*/
42+
type FrozenUnavailablePlatformRuntimeFacts = Readonly<
43+
Required<Omit<UnavailablePlatformRuntimeFacts, 'lifecycle'>> &
44+
Readonly<{ lifecycle: ApplicationLifecycleOperationFacts }>
45+
>;
5346

5447
export function createUnavailablePlatformRuntimeBinding(
5548
device: DeviceInfo,

packages/platform-android/src/runtime.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,11 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor
220220
})
221221
: {}),
222222
...(facts.operations.readTextAtPoint.available
223-
? bindElementTextRuntime({ device: request.device, host: host.elementText })
223+
? bindElementTextRuntime({
224+
device: request.device,
225+
signal: request.scope.signal,
226+
resolveInteractor: host.localInteractors.resolve,
227+
})
224228
: {}),
225229
ensureReady: async (input: EnsureReadyInput) =>
226230
await ensureAndroidReady(

packages/platform-apple/src/runtime.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,11 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR
288288
})
289289
: {}),
290290
...(facts.operations.readTextAtPoint.available
291-
? bindElementTextRuntime({ device: request.device, host: host.elementText })
291+
? bindElementTextRuntime({
292+
device: request.device,
293+
signal: request.scope.signal,
294+
resolveInteractor: host.localInteractors.resolve,
295+
})
292296
: {}),
293297
...(facts.operations.ensureReady.available
294298
? {

packages/platform-linux/src/runtime.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,11 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR
9797
})
9898
: {}),
9999
...(facts.operations.readTextAtPoint.available
100-
? bindElementTextRuntime({ device: request.device, host: host.elementText })
100+
? bindElementTextRuntime({
101+
device: request.device,
102+
signal: request.scope.signal,
103+
resolveInteractor: host.localInteractors.resolve,
104+
})
101105
: {}),
102106
}),
103107
[Symbol.asyncDispose]: async () => undefined,

src/core/interactors/android.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ export function createAndroidInteractor(
6565
performGesture: (plan) => executeAndroidTouchPlan(device, plan),
6666
gestureViewport: () => readAndroidGestureViewport(device),
6767
screenshot: (outPath, options) => screenshotAndroid(device, outPath, options),
68+
// uiautomator reads the node covering a point; `undefined` means nothing covers it.
69+
readTextAtPoint: async (point) => {
70+
const { readAndroidTextAtPoint } = await import('../../platforms/android/input-actions.ts');
71+
return (await readAndroidTextAtPoint(device, point.x, point.y)) ?? undefined;
72+
},
6873
snapshot: async (options) => {
6974
const snapshotOptions = options ?? {};
7075
const result = await withDiagnosticTimer(

src/core/interactors/linux.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ export function createLinuxInteractor(): Interactor {
4545
await swipeLinux(start.x, start.y, end.x, end.y, plan.durationMs);
4646
},
4747
screenshot: (outPath, options) => screenshotLinux(outPath, options),
48+
// The Linux read is value-first (AXValue/title/description) where the captured tree is
49+
// label-first, so this genuinely reads differently from its snapshot text.
50+
readTextAtPoint: async (point, options) => {
51+
const { readLinuxTextAtPoint } = await import('../../platforms/linux/snapshot.ts');
52+
return await readLinuxTextAtPoint(point.x, point.y, options?.surface);
53+
},
4854
snapshot: async (options) => {
4955
return await withDiagnosticTimer(
5056
'snapshot_capture',

0 commit comments

Comments
 (0)