fix(ios): keep surface identity in verify and settle comparisons

`--verify` compared node digests and `--settle` diffed node-only baselines, so an app
baseline and an in-place system-surface capture (a web sign-in sheet) were treated as one
presentation: a meaningless changed verdict, and a whole-surface replacement presented as an
in-surface diff with refs.

The pre-action baseline now travels with the surface its capture described, from the resolution
and the session frame through to the settled capture, and one module owns the comparison for
both routes. Across a surface change no same-surface claim is made: evidence reports the
transition instead of a digest comparison, the settled diff and its refs are withheld, and both
payloads disclose the transition.

Author: thymikee

packages/contracts/src/interaction.ts

@@ -121,15 +121,35 @@ export type PreresolvedInteractionTarget = {
   node: SnapshotNode;
   /** The tree `node` came from — the guards read its siblings for occlusion/viewport. */
   nodes: SnapshotNode[];
+  /**
+   * The in-place iOS system surface `nodes` describes (#2438), absent for ordinary app content.
+   * Travels with the tree so the adopting consumer's post-action comparison knows which surface
+   * its baseline came from.
+   */
+  iosSystemSurfaceBundleId?: string;
+};
+
+/**
+ * The pre-action tree a post-action observation compares against, and the SURFACE the capture it
+ * came from described (#2438: the bundle id of an in-place iOS system surface such as a web
+ * sign-in sheet, absent for ordinary app content).
+ *
+ * One value, never two channels: a capture of the sheet and a capture of the app describe
+ * different surfaces, so a `--verify` digest comparison or a `--settle` diff across that boundary
+ * is not about one presentation. The identity is attached where the nodes are taken, so every
+ * comparison site can ask the question.
+ */
+type PreActionBaselineFields = {
+  preActionNodes?: SnapshotNode[];
+  preActionSurfaceBundleId?: string;
 };
 
 export type ResolvedInteractionTarget =
-  | {
+  | ({
       kind: 'point';
       point: Point;
-      preActionNodes?: SnapshotNode[];
-    }
-  | {
+    } & PreActionBaselineFields)
+  | ({
       kind: 'ref';
       point?: Point;
       target: Extract<ResolvedTarget, { kind: 'ref' }>;
@@ -138,11 +158,10 @@ export type ResolvedInteractionTarget =
       refLabel?: string;
       targetHittable?: boolean;
       hint?: string;
-      preActionNodes?: SnapshotNode[];
       resolution?: ResolutionDisclosure;
       recordingTarget?: RecordingTargetOverride;
-    }
-  | {
+    } & PreActionBaselineFields)
+  | ({
       kind: 'selector';
       point: Point;
       target: Extract<ResolvedTarget, { kind: 'selector' }>;
@@ -151,10 +170,26 @@ export type ResolvedInteractionTarget =
       refLabel?: string;
       targetHittable?: boolean;
       hint?: string;
-      preActionNodes?: SnapshotNode[];
       resolution?: ResolutionDisclosure;
       recordingTarget?: RecordingTargetOverride;
-    };
+    } & PreActionBaselineFields);
+
+/**
+ * A post-action capture that describes a DIFFERENT surface than the pre-action baseline (#2438): an
+ * in-place iOS system surface (a web sign-in sheet, hosted out of the app's process) was presented
+ * over the app, or left it. `from`/`to` name the two surfaces — a host bundle id, or `app` for
+ * ordinary app content.
+ *
+ * Its presence IS the refusal of a same-surface claim: the two captures are not one presentation,
+ * so `--verify` reports `changedFromBefore` from this transition instead of from a digest
+ * comparison across it, and `--settle` attaches no settled diff (and therefore no refs) across it.
+ */
+export type PostActionSurfaceChange = {
+  from: string;
+  to: string;
+  /** The one agent-facing sentence for this transition (`@agent-device/contracts/ios-system-surface`). */
+  disclosure: string;
+};
 
 /**
  * Opt-in (`--verify`) cheap post-condition evidence for mutating interaction
@@ -164,13 +199,18 @@ export type ResolvedInteractionTarget =
  * held, so no extra device round trip is spent beyond the one verify capture.
  * `changedFromBefore: false` is evidence, not failure — the command still
  * succeeded.
+ *
+ * When `surfaceChange` is present the two captures describe different surfaces, so the digest
+ * comparison is not made at all: `changedFromBefore` then reports that transition, which replaced
+ * the whole observed surface.
  */
 export type InteractionEvidence = {
   foregroundApp?: string;
   nodeCount: number;
   interactiveNodeCount: number;
   digest: string;
   changedFromBefore: boolean;
+  surfaceChange?: PostActionSurfaceChange;
 };
 
 export type SettleDiffLine = {
@@ -249,7 +289,17 @@ export type SettleObservation = {
    * intentionally omitted.
    */
   refs?: Array<{ ref: string }>;
-  /** Present only for `settled: true` observations that stored the settled tree. */
+  /**
+   * Present when the settled capture describes a different surface than the pre-action baseline
+   * (#2438). The settled tree then replaced the whole surface rather than changing within one, so
+   * `diff` is omitted: its lines (and their refs) would present a surface replacement as an
+   * in-surface change. `hint` says what to do instead.
+   */
+  surfaceChange?: PostActionSurfaceChange;
+  /**
+   * Present only for `settled: true` observations that stored the settled tree, and never across a
+   * `surfaceChange` — a diff describes change WITHIN one surface.
+   */
   diff?: {
     summary: { additions: number; removals: number; unchanged: number };
     lines: SettleDiffLine[];

packages/contracts/src/ios-system-surface.ts

@@ -79,3 +79,16 @@ export function iosSystemSurfaceDisclosure(
 ): string | undefined {
   return provenance ? IOS_SYSTEM_SURFACE_DISCLOSURE : undefined;
 }
+
+/**
+ * The agent-facing sentence for a surface TRANSITION between two captures — the post-action
+ * observation's case, where the pre-action baseline and the capture taken after the action describe
+ * different surfaces. `to` is the surface the AFTER capture describes: a host bundle id when the
+ * sheet is now on screen (the standing disclosure applies verbatim), or `undefined` when the sheet
+ * has left and the capture shows app content again, which the standing sentence cannot say.
+ */
+export function iosSystemSurfaceTransitionDisclosure(to: string | undefined): string {
+  return to === undefined
+    ? 'A system web sign-in sheet was presented over the app before this action and is gone now, so this observation describes app content while the pre-action tree described that sheet.'
+    : IOS_SYSTEM_SURFACE_DISCLOSURE;
+}

src/commands/interaction/runtime/interactions.test.ts

@@ -8,7 +8,9 @@ import {
   createMemorySessionStore,
   localCommandPolicy,
 } from '../../../runtime.ts';
-import type { Point } from '@agent-device/kernel/snapshot';
+import type { Point, SnapshotState } from '@agent-device/kernel/snapshot';
+import { summarizeAxEvidence } from '@agent-device/capture-kit/snapshot-evidence';
+import { IOS_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/ios-system-surface';
 import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
 import {
   coveredByTabBarSnapshot,
@@ -491,6 +493,92 @@ test('runtime press with verify reports changedFromBefore true when the post-act
   assert.equal(result.evidence?.nodeCount, 2);
 });
 
+// #2438: iOS serves a web sign-in sheet (com.apple.SafariViewService) IN PLACE over a
+// still-foreground app, so a capture of the sheet and a capture of the app describe DIFFERENT
+// surfaces. `--verify` must not answer "did this change?" by comparing their node digests.
+const WEB_SIGN_IN_SHEET_BUNDLE_ID = 'com.apple.SafariViewService';
+
+function webSignInSheetSnapshot(labels: string[]): SnapshotState {
+  return {
+    ...makeSnapshotState(
+      labels.map((label, index) => ({
+        index,
+        depth: 0,
+        type: 'Button',
+        label,
+        rect: { x: 10, y: 20 + index * 50, width: 200, height: 40 },
+        hittable: true,
+      })),
+    ),
+    iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
+  };
+}
+
+test('runtime press with verify discloses the surface change when a sign-in sheet replaces the app', async () => {
+  let captureCount = 0;
+  const device = createInteractionDevice(selectorSnapshot(), {
+    captureSnapshot: async () => {
+      captureCount += 1;
+      // Capture 1 resolves the target against app content; capture 2 is the verify capture, taken
+      // after the sheet was presented in place over that app.
+      return {
+        snapshot:
+          captureCount === 1
+            ? selectorSnapshot()
+            : webSignInSheetSnapshot(['Sign in with Example', 'Cancel']),
+      };
+    },
+    tap: async () => ({ ok: true }),
+  });
+
+  const result = await device.interactions.press(selector('label=Continue'), {
+    session: 'default',
+    verify: true,
+  });
+
+  assert.equal(result.kind, 'selector');
+  assert.deepEqual(result.evidence?.surfaceChange, {
+    from: 'app',
+    to: WEB_SIGN_IN_SHEET_BUNDLE_ID,
+    disclosure: IOS_SYSTEM_SURFACE_DISCLOSURE,
+  });
+  // The transition is what changed, not a digest comparison between two different surfaces.
+  assert.equal(result.evidence?.changedFromBefore, true);
+});
+
+test('runtime press with verify reports the sheet leaving even when the two digests coincide', async () => {
+  const sheet: SnapshotState = {
+    ...selectorSnapshot(),
+    iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
+  };
+  const app = selectorSnapshot();
+  // The premise this test exists for: the sheet tree and the app tree that replaces it digest
+  // identically, so a digest comparison would report "nothing changed" across a whole-surface
+  // replacement. The surfaces, not the digests, answer the question.
+  assert.equal(summarizeAxEvidence(sheet.nodes).digest, summarizeAxEvidence(app.nodes).digest);
+  let captureCount = 0;
+  const device = createInteractionDevice(sheet, {
+    captureSnapshot: async () => {
+      captureCount += 1;
+      return { snapshot: captureCount === 1 ? sheet : app };
+    },
+    tap: async () => ({ ok: true }),
+  });
+
+  const result = await device.interactions.press(selector('label=Continue'), {
+    session: 'default',
+    verify: true,
+  });
+
+  assert.equal(result.kind, 'selector');
+  assert.equal(result.evidence?.surfaceChange?.from, WEB_SIGN_IN_SHEET_BUNDLE_ID);
+  assert.equal(result.evidence?.surfaceChange?.to, 'app');
+  assert.match(result.evidence?.surfaceChange?.disclosure ?? '', /sign-in sheet/);
+  // The sheet is gone, so the standing "is presented over the app" sentence cannot be the one used.
+  assert.notEqual(result.evidence?.surfaceChange?.disclosure, IOS_SYSTEM_SURFACE_DISCLOSURE);
+  assert.equal(result.evidence?.changedFromBefore, true);
+});
+
 test('runtime fill without verify omits evidence entirely', async () => {
   const device = createInteractionDevice(fillableSnapshot(), {
     fill: async () => ({ ok: true }),

src/commands/interaction/runtime/post-action-observation.ts

@@ -1,13 +1,16 @@
-import type { SnapshotNode } from '@agent-device/kernel/snapshot';
 import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts';
-import { summarizeAxEvidence } from '@agent-device/capture-kit/snapshot-evidence';
 import type {
   InteractionEvidence,
   ResolvedInteractionTarget,
   SettleObservation,
   SettleParams,
 } from '@agent-device/contracts/interaction';
 import { captureInteractionSnapshot } from './resolution.ts';
+import {
+  preActionBaseline,
+  summarizePostActionEvidence,
+  surfaceScopedNodes,
+} from './post-action-surface.ts';
 import { settleAfterInteraction, settleEvidence } from './settle.ts';
 
 type ObservedResult<T extends object> = T & {
@@ -82,10 +85,7 @@ async function observeAfterInteraction(
       resolved,
     });
     const evidence = params.verify
-      ? settleEvidence(
-          outcome.settledNodes,
-          'preActionNodes' in resolved ? resolved.preActionNodes : undefined,
-        )
+      ? settleEvidence(outcome.settledCapture, preActionBaseline(resolved))
       : undefined;
     return { settle: outcome.observation, ...(evidence ? { evidence } : {}) };
   }
@@ -99,22 +99,24 @@ async function observeAfterInteraction(
  * the same capture helper the resolution path already uses, digested and then
  * discarded. The node tree itself is never attached to the result, only the
  * cheap summary.
+ *
+ * Both sides of the comparison carry the surface they describe (#2438), so a
+ * capture of an in-place system surface is never digest-compared against an app
+ * baseline: `summarizePostActionEvidence` owns that rule for this route and the
+ * `--settle --verify` route alike. A missing baseline still yields
+ * `changedFromBefore: false` — no baseline, no claim.
  */
 async function captureVerifyEvidence(
   runtime: AgentDeviceRuntime,
   options: CommandContext,
   resolved: ResolvedInteractionTarget,
 ): Promise<InteractionEvidence | undefined> {
-  const preActionNodes: SnapshotNode[] | undefined =
-    'preActionNodes' in resolved ? resolved.preActionNodes : undefined;
   try {
     const capture = await captureInteractionSnapshot(runtime, options, true);
-    const after = summarizeAxEvidence(capture.snapshot.nodes);
-    // No pre-action baseline means we cannot claim a change happened; default
-    // to false rather than asserting a change we did not actually observe.
-    const changedFromBefore =
-      preActionNodes !== undefined && after.digest !== summarizeAxEvidence(preActionNodes).digest;
-    return { ...after, changedFromBefore };
+    return summarizePostActionEvidence(
+      surfaceScopedNodes(capture.snapshot),
+      preActionBaseline(resolved),
+    );
   } catch {
     return undefined;
   }
@@ -150,6 +152,9 @@ function hasMaterialPostActionChange(view: {
   settle?: SettleObservation;
 }): boolean {
   if (view.evidence?.changedFromBefore === true) return true;
+  // A surface replacement (#2438) carries no settled diff by design, and it is the most material
+  // post-action change there is: the screen is now a different surface.
+  if (view.settle?.surfaceChange !== undefined) return true;
   const summary = view.settle?.diff?.summary;
   return !!summary && (summary.additions > 0 || summary.removals > 0);
 }