Skip to content

Commit 4959975

Browse files
committed
fix: stabilize private AX settle snapshots
1 parent 856ff38 commit 4959975

8 files changed

Lines changed: 329 additions & 130 deletions

File tree

.github/workflows/ios.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ jobs:
113113
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \
114114
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \
115115
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
116+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint \
117+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts \
116118
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
117119
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \
118120
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

Lines changed: 4 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -175,15 +175,10 @@ extension RunnerTests {
175175

176176
let rootFrame = privateAXRect(root["frame"])
177177
let viewport = privateAXSnapshotViewport(app: app, rootFrame: rootFrame)
178-
var nodes: [SnapshotNode] = []
179-
appendPrivateAXNode(
180-
root,
181-
to: &nodes,
178+
let nodes = privateAXPresentation(
179+
rawRoot: root,
182180
options: options,
183-
viewport: viewport,
184-
depth: 0,
185-
parentIndex: nil,
186-
insideMatchedScope: false
181+
viewport: viewport
187182
)
188183
if nodes.count <= 1 {
189184
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count)
@@ -250,110 +245,7 @@ extension RunnerTests {
250245
}
251246
}
252247

253-
private func appendPrivateAXNode(
254-
_ rawNode: [String: Any],
255-
to nodes: inout [SnapshotNode],
256-
options: SnapshotOptions,
257-
viewport: CGRect,
258-
depth: Int,
259-
parentIndex: Int?,
260-
insideMatchedScope: Bool
261-
) {
262-
if let limit = options.depth, depth > limit { return }
263-
264-
let rect = privateAXRect(rawNode["frame"])
265-
let label = privateAXString(rawNode["label"])
266-
let identifier = privateAXString(rawNode["identifier"])
267-
let value = privateAXString(rawNode["value"])
268-
let rawType = privateAXInt(rawNode["type"]) ?? 0
269-
let typeName = elementTypeName(rawElementType: rawType)
270-
let enabled = privateAXBool(rawNode["enabled"]) ?? true
271-
let visible = isVisibleInViewport(rect, viewport)
272-
let interactiveCandidate = privateAXInteractiveCandidate(rawElementType: rawType)
273-
let filterDecision = flatSnapshotFilterDecision(
274-
FlatSnapshotFilterNode(
275-
isRoot: parentIndex == nil,
276-
label: label,
277-
identifier: identifier,
278-
valueText: value.isEmpty ? nil : value,
279-
visible: visible
280-
),
281-
options: options,
282-
insideMatchedScope: insideMatchedScope
283-
)
284-
let include = filterDecision.include
285-
let nowInsideScope = filterDecision.insideMatchedScope
286-
287-
let currentIndex: Int?
288-
if include {
289-
currentIndex = nodes.count
290-
nodes.append(
291-
SnapshotNode(
292-
index: nodes.count,
293-
type: typeName,
294-
label: label.isEmpty ? nil : label,
295-
identifier: identifier.isEmpty ? nil : identifier,
296-
value: value.isEmpty ? nil : value,
297-
rect: snapshotRect(from: rect),
298-
enabled: enabled,
299-
focused: privateAXBool(rawNode["focused"]) == true ? true : nil,
300-
selected: privateAXBool(rawNode["selected"]) == true ? true : nil,
301-
hittable: visible && enabled && interactiveCandidate,
302-
depth: depth,
303-
parentIndex: parentIndex,
304-
hiddenContentAbove: nil,
305-
hiddenContentBelow: nil,
306-
actions: rawNode["actions"] as? [String]
307-
)
308-
)
309-
} else {
310-
currentIndex = parentIndex
311-
}
312-
313-
guard let children = rawNode["children"] as? [[String: Any]] else {
314-
return
315-
}
316-
for child in children {
317-
appendPrivateAXNode(
318-
child,
319-
to: &nodes,
320-
options: options,
321-
viewport: viewport,
322-
depth: depth + 1,
323-
parentIndex: currentIndex,
324-
insideMatchedScope: nowInsideScope
325-
)
326-
}
327-
}
328-
329-
private func elementTypeName(rawElementType: Int) -> String {
330-
if let type = flatSnapshotElementType(rawElementType: rawElementType) {
331-
return elementTypeName(type)
332-
}
333-
return "Element(\(rawElementType))"
334-
}
335-
336-
private func privateAXString(_ value: Any?) -> String {
337-
guard let value else { return "" }
338-
if let string = value as? String {
339-
return string.trimmingCharacters(in: .whitespacesAndNewlines)
340-
}
341-
return String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
342-
}
343-
344-
private func privateAXInt(_ value: Any?) -> Int? {
345-
if let value = value as? Int { return value }
346-
if let value = value as? NSNumber { return value.intValue }
347-
return nil
348-
}
349-
350-
private func privateAXBool(_ value: Any?) -> Bool? {
351-
if let value = value as? Bool { return value }
352-
if let value = value as? NSNumber { return value.boolValue }
353-
return nil
354-
}
355-
356-
private func privateAXRect(_ value: Any?) -> CGRect {
248+
func privateAXRect(_ value: Any?) -> CGRect {
357249
guard let frame = value as? [String: Any] else {
358250
return .zero
359251
}
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import XCTest
2+
3+
extension RunnerTests {
4+
func privateAXPresentation(rawRoot: [String: Any], options: SnapshotOptions, viewport: CGRect)
5+
-> [SnapshotNode]
6+
{
7+
var nodes: [SnapshotNode] = []
8+
var hints: [Int: (above: Bool, below: Bool)] = [:]
9+
appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
10+
depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil)
11+
guard !hints.isEmpty else { return nodes }
12+
return nodes.map { node in
13+
guard let hint = hints[node.index] else { return node }
14+
return SnapshotNode(index: node.index, type: node.type, label: node.label,
15+
identifier: node.identifier, value: node.value, rect: node.rect, enabled: node.enabled,
16+
focused: node.focused, selected: node.selected, hittable: node.hittable, depth: node.depth,
17+
parentIndex: node.parentIndex, hiddenContentAbove: hint.above ? true : node.hiddenContentAbove,
18+
hiddenContentBelow: hint.below ? true : node.hiddenContentBelow, actions: node.actions)
19+
}
20+
}
21+
22+
private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [SnapshotNode],
23+
hints: inout [Int: (above: Bool, below: Bool)], options: SnapshotOptions, viewport: CGRect,
24+
depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
25+
scrollContext: (index: Int, rect: CGRect)?)
26+
{
27+
if let limit = options.depth, depth > limit { return }
28+
let rect = privateAXRect(raw["frame"])
29+
let label = privateAXPresentationString(raw["label"])
30+
let identifier = privateAXPresentationString(raw["identifier"])
31+
let value = privateAXPresentationString(raw["value"])
32+
let rawType = privateAXPresentationInt(raw["type"]) ?? 0
33+
let enabled = privateAXPresentationBool(raw["enabled"]) ?? true
34+
let hasFrame = !rect.isNull && !rect.isEmpty
35+
let visible = (!hasFrame || isVisibleInViewport(rect, viewport))
36+
&& (!hasFrame || scrollContext.map { isVisibleInViewport(rect, $0.rect) } ?? true)
37+
let decision = flatSnapshotFilterDecision(
38+
FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier,
39+
valueText: value.isEmpty ? nil : value, visible: visible),
40+
options: options, insideMatchedScope: insideMatchedScope)
41+
let include = decision.include && (parentIndex == nil || visible)
42+
43+
if !include, !visible, let scrollContext, hasFrame {
44+
var hint = hints[scrollContext.index] ?? (above: false, below: false)
45+
if rect.maxY <= scrollContext.rect.minY { hint.above = true }
46+
if rect.minY >= scrollContext.rect.maxY { hint.below = true }
47+
hints[scrollContext.index] = hint
48+
}
49+
50+
let currentIndex: Int?
51+
if include {
52+
currentIndex = nodes.count
53+
let typeName = flatSnapshotElementType(rawElementType: rawType).map(elementTypeName)
54+
?? "Element(\(rawType))"
55+
nodes.append(SnapshotNode(index: nodes.count, type: typeName,
56+
label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
57+
value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
58+
focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
59+
selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
60+
hittable: hasFrame && visible && enabled && privateAXInteractiveCandidate(rawElementType: rawType),
61+
depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
62+
actions: raw["actions"] as? [String]))
63+
} else { currentIndex = parentIndex }
64+
65+
let nextScrollContext: (index: Int, rect: CGRect)?
66+
if include, hasFrame, let type = flatSnapshotElementType(rawElementType: rawType),
67+
Self.scrollContainerTypes.contains(type), let currentIndex {
68+
nextScrollContext = (currentIndex, rect)
69+
} else { nextScrollContext = scrollContext }
70+
for child in raw["children"] as? [[String: Any]] ?? [] {
71+
appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport,
72+
depth: depth + 1, parentIndex: currentIndex,
73+
insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext)
74+
}
75+
}
76+
77+
func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [SnapshotNode],
78+
options: SnapshotOptions, viewport: CGRect, depth: Int, parentIndex: Int?,
79+
insideMatchedScope: Bool)
80+
{
81+
var hints: [Int: (above: Bool, below: Bool)] = [:]
82+
appendPrivateAXNode(raw, to: &nodes, hints: &hints, options: options, viewport: viewport,
83+
depth: depth, parentIndex: parentIndex, insideMatchedScope: insideMatchedScope,
84+
scrollContext: nil)
85+
}
86+
87+
private func privateAXPresentationString(_ value: Any?) -> String {
88+
guard let value else { return "" }
89+
return (value as? String ?? String(describing: value))
90+
.trimmingCharacters(in: .whitespacesAndNewlines)
91+
}
92+
private func privateAXPresentationInt(_ value: Any?) -> Int? {
93+
(value as? Int) ?? (value as? NSNumber)?.intValue
94+
}
95+
private func privateAXPresentationBool(_ value: Any?) -> Bool? {
96+
(value as? Bool) ?? (value as? NSNumber)?.boolValue
97+
}
98+
}
99+
100+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
101+
extension RunnerTests {
102+
func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() {
103+
func frame(_ x: Double, _ y: Double, _ width: Double, _ height: Double) -> [String: Any] {
104+
["x": x, "y": y, "width": width, "height": height]
105+
}
106+
let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue),
107+
"label": "Element", "frame": frame(0, 0, 402, 874), "children": [[
108+
"type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700),
109+
"children": [["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", "frame": frame(16, 120, 44, 44)],
110+
["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": frame(16, 900, 360, 44)]]]]]
111+
let nodes = privateAXPresentation(rawRoot: root,
112+
options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
113+
viewport: CGRect(x: 0, y: 0, width: 402, height: 874))
114+
XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"])
115+
XCTAssertEqual(nodes.first { $0.type == "ScrollView" }?.hiddenContentBelow, true)
116+
}
117+
118+
func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() {
119+
let zero = ["x": 0, "y": 0, "width": 0, "height": 0]
120+
let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue),
121+
"label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874],
122+
"children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue),
123+
"label": "Settings semantics", "frame": zero, "children": [[
124+
"type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero]]]]]
125+
let nodes = privateAXPresentation(rawRoot: root,
126+
options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
127+
viewport: CGRect(x: 0, y: 0, width: 402, height: 874))
128+
XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"])
129+
XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false])
130+
XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil })
131+
}
132+
}
133+
#endif

src/commands/interaction/runtime/selector-read-shared.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ export async function captureSelectorSnapshot(
5252
includeRects?: boolean;
5353
interactiveOnly?: boolean;
5454
includeHiddenContentHints?: boolean;
55+
preferredBackend?: 'private-ax';
5556
} = {
5657
updateSession: true,
5758
},
@@ -68,6 +69,9 @@ export async function captureSelectorSnapshot(
6869
scope: captureOptions.scope ?? options.scope,
6970
raw: options.raw,
7071
includeRects: captureOptions.includeRects,
72+
...(captureOptions.preferredBackend
73+
? { preferredBackend: captureOptions.preferredBackend }
74+
: {}),
7175
...(captureOptions.includeHiddenContentHints !== undefined
7276
? { includeHiddenContentHints: captureOptions.includeHiddenContentHints }
7377
: {}),

src/commands/interaction/runtime/settle.test.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,40 @@ test('private-ax recovery resets the settle budget once', async () => {
208208
assert.equal(captures, 4);
209209
});
210210

211+
test('press --settle pins an already-established private-ax observation backend', async () => {
212+
const established = welcomeSnapshot();
213+
established.snapshotQuality = {
214+
state: 'recovered',
215+
backend: 'private-ax',
216+
reasonCode: 'deferred',
217+
reason: 'penalty',
218+
};
219+
const preferredBackends: Array<string | undefined> = [];
220+
const device = createAgentDevice({
221+
backend: {
222+
platform: 'ios',
223+
captureSnapshot: async (_context, options) => {
224+
preferredBackends.push(options?.preferredBackend);
225+
return { snapshot: established };
226+
},
227+
tap: async () => ({ ok: true }),
228+
fill: async () => ({ ok: true }),
229+
longPress: async () => ({ ok: true }),
230+
typeText: async () => {},
231+
} satisfies AgentDeviceBackend,
232+
artifacts: createLocalArtifactAdapter(),
233+
sessions: createMemorySessionStore([{ name: 'default', snapshot: established }]),
234+
policy: localCommandPolicy(),
235+
clock: createFakeClock(),
236+
});
237+
await device.interactions.press(selector('label=Next'), {
238+
session: 'default',
239+
settle: { quietMs: 500, timeoutMs: 2_000 },
240+
});
241+
assert.equal(preferredBackends[0], undefined);
242+
assert.deepEqual(new Set(preferredBackends.slice(1)), new Set(['private-ax']));
243+
});
244+
211245
test('penalty-deferred private-ax captures do not reset the settle budget', async () => {
212246
// Same shape as the reset test above, but the verdict says the circuit breaker PRE-selected
213247
// private AX ('deferred'): the capture paid no grind, so the loop keeps its original budget
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts';
4+
import { stableCaptureSignal, stableCaptureSignalsEqual } from './stable-capture-signal.ts';
5+
6+
function snapshot(jitter: number, hiddenY: number) {
7+
return makeSnapshotState(
8+
[
9+
{
10+
index: 0,
11+
depth: 0,
12+
type: 'Application',
13+
label: 'Element',
14+
rect: { x: 0, y: 0, width: 402, height: 874 },
15+
hittable: false,
16+
},
17+
{
18+
index: 1,
19+
depth: 1,
20+
parentIndex: 0,
21+
type: 'ScrollView',
22+
rect: { x: 0, y: 96, width: 402, height: 700 },
23+
hittable: true,
24+
},
25+
{
26+
index: 2,
27+
depth: 2,
28+
parentIndex: 1,
29+
type: 'Button',
30+
label: 'Profile',
31+
rect: { x: 17 + jitter, y: 120, width: 44, height: 44 },
32+
hittable: true,
33+
},
34+
{
35+
index: 3,
36+
depth: 2,
37+
parentIndex: 1,
38+
type: 'Button',
39+
label: 'Theme',
40+
rect: { x: 16, y: hiddenY, width: 360, height: 44 },
41+
hittable: false,
42+
},
43+
],
44+
{
45+
snapshotQuality: {
46+
state: 'recovered',
47+
backend: 'private-ax',
48+
reasonCode: 'deferred',
49+
reason: 'penalty',
50+
},
51+
},
52+
);
53+
}
54+
55+
test('private-ax stability ignores offscreen churn and boundary-crossing one-pixel jitter', () => {
56+
assert.equal(
57+
stableCaptureSignalsEqual(
58+
stableCaptureSignal(snapshot(0, 900)),
59+
stableCaptureSignal(snapshot(1, 980)),
60+
),
61+
true,
62+
);
63+
});

0 commit comments

Comments
 (0)