Add viewport-aware element tracking, screen change detection, and bounding box fixes

- Add screen change detection via frame comparison in web UI (Image + Canvas thumbnail diff)
- Auto-refresh element tree when screen navigation detected (>15% pixel change)
- Filter elements by active route (SceneView.isFocused) to exclude frozen screens
- Add screenId to annotations for screen-scoped display
- Add scroll-aware annotation pin repositioning
- Fix bounding box misalignment: remove incorrect viewport-to-screen normalization
- Add iPhone 17 series to iOS screen size lookup table
- Fix device dimensions for iPhone 17 Pro (402x874 instead of default 393x852)
- Add accessibility merge utility for cross-source element enrichment
- Add MCP annotation tool and bridge-core getScreenId interface
- Improve Flutter bounding box extraction (fix early-return bug)
- Add iOS multi-strategy accessibility fallback

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: web-flow

packages/bridges/bridge-core/src/accessibility-merge.ts

@@ -0,0 +1,334 @@
+import type { MobileElement, Platform } from "@agentation-mobile/core";
+
+/**
+ * Maps iOS AX accessibility roles to semantic role names.
+ */
+const IOS_ROLE_MAP: Record<string, string> = {
+	AXButton: "button",
+	AXStaticText: "text",
+	AXTextField: "textfield",
+	AXSecureTextField: "textfield",
+	AXTextView: "textfield",
+	AXImage: "image",
+	AXCheckBox: "checkbox",
+	AXRadioButton: "radio",
+	AXSwitch: "switch",
+	AXSlider: "slider",
+	AXProgressIndicator: "progressbar",
+	AXPopUpButton: "combobox",
+	AXTable: "list",
+	AXCollectionView: "list",
+	AXScrollView: "scrollbar",
+	AXWebView: "web",
+	AXTabBar: "tablist",
+	AXTabButton: "tab",
+	AXNavigationBar: "navigation",
+	AXToolbar: "toolbar",
+	AXLink: "link",
+	AXCell: "cell",
+	AXGroup: "group",
+	AXWindow: "window",
+	AXApplication: "application",
+};
+
+/**
+ * Maps Android widget class names to semantic role names.
+ */
+const ANDROID_ROLE_MAP: Record<string, string> = {
+	"android.widget.Button": "button",
+	"android.widget.TextView": "text",
+	"android.widget.EditText": "textfield",
+	"android.widget.ImageView": "image",
+	"android.widget.ImageButton": "button",
+	"android.widget.CheckBox": "checkbox",
+	"android.widget.RadioButton": "radio",
+	"android.widget.Switch": "switch",
+	"android.widget.ToggleButton": "switch",
+	"android.widget.SeekBar": "slider",
+	"android.widget.ProgressBar": "progressbar",
+	"android.widget.Spinner": "combobox",
+	"android.widget.ListView": "list",
+	"android.widget.RecyclerView": "list",
+	"android.widget.ScrollView": "scrollbar",
+	"android.widget.HorizontalScrollView": "scrollbar",
+	"android.widget.TabHost": "tablist",
+	"android.widget.TabWidget": "tablist",
+	"android.webkit.WebView": "web",
+	"android.widget.LinearLayout": "group",
+	"android.widget.RelativeLayout": "group",
+	"android.widget.FrameLayout": "group",
+	"android.view.ViewGroup": "group",
+	"android.view.View": "view",
+};
+
+/**
+ * Maps a platform-specific role string to a unified semantic role.
+ * Handles both iOS AX roles and Android widget class names.
+ */
+export function mapRole(role: string): string {
+	// Check iOS mappings first
+	if (role in IOS_ROLE_MAP) return IOS_ROLE_MAP[role];
+
+	// Check Android mappings
+	if (role in ANDROID_ROLE_MAP) return ANDROID_ROLE_MAP[role];
+
+	// Fallback: strip AX prefix for iOS, extract class name for Android
+	if (role.startsWith("AX")) return role.replace(/^AX/, "").toLowerCase();
+	if (role.includes(".")) return role.split(".").pop()?.toLowerCase() ?? role.toLowerCase();
+
+	return role.toLowerCase();
+}
+
+/**
+ * Finds the best SDK element match for a given native element using bounding box
+ * overlap. Returns null if no sufficient overlap is found.
+ *
+ * Requires >50% overlap relative to the smaller element's area.
+ */
+export function findBestMatch(
+	target: MobileElement,
+	sdkElements: MobileElement[],
+): MobileElement | null {
+	let bestMatch: MobileElement | null = null;
+	let bestOverlap = 0;
+
+	const tb = target.boundingBox;
+	const targetArea = tb.width * tb.height;
+
+	for (const sdk of sdkElements) {
+		const sb = sdk.boundingBox;
+
+		const overlapX = Math.max(0, Math.min(tb.x + tb.width, sb.x + sb.width) - Math.max(tb.x, sb.x));
+		const overlapY = Math.max(
+			0,
+			Math.min(tb.y + tb.height, sb.y + sb.height) - Math.max(tb.y, sb.y),
+		);
+		const overlapArea = overlapX * overlapY;
+
+		const sdkArea = sb.width * sb.height;
+		const minArea = Math.min(targetArea, sdkArea);
+
+		if (minArea > 0 && overlapArea / minArea > 0.5 && overlapArea > bestOverlap) {
+			bestOverlap = overlapArea;
+			bestMatch = sdk;
+		}
+	}
+
+	// Fallback: when both have zero-area bounding boxes, try text-content matching
+	if (!bestMatch && targetArea === 0 && target.textContent) {
+		const targetText = target.textContent.trim().toLowerCase();
+		if (targetText.length > 0) {
+			for (const sdk of sdkElements) {
+				const sdkText = sdk.textContent?.trim().toLowerCase();
+				if (sdkText && sdkText === targetText) {
+					return sdk;
+				}
+			}
+		}
+	}
+
+	return bestMatch;
+}
+
+/**
+ * Enriches "native" elements (from accessibility tree / UIAutomator) with data from
+ * "SDK" elements (from fiber tree / DevTools) using bounding box overlap matching.
+ *
+ * Native elements provide accurate bounding boxes; SDK elements provide component
+ * names, file paths, and source locations.
+ */
+export function mergeElements(
+	nativeElements: MobileElement[],
+	sdkElements: MobileElement[],
+): MobileElement[] {
+	if (sdkElements.length === 0) return nativeElements;
+
+	return nativeElements.map((nativeEl) => {
+		const match = findBestMatch(nativeEl, sdkElements);
+		if (!match) return nativeEl;
+
+		return {
+			...nativeEl,
+			sourceLocation: match.sourceLocation ?? nativeEl.sourceLocation,
+			componentFile: match.componentFile ?? nativeEl.componentFile,
+			componentName: match.componentName || nativeEl.componentName,
+			animations: match.animations ?? nativeEl.animations,
+		};
+	});
+}
+
+export interface AccessibilityNode {
+	label: string;
+	role: string;
+	value: string;
+	traits: string[];
+	frame: { x: number; y: number; width: number; height: number } | null;
+	depth: number;
+}
+
+function normalizeAccessibilityNode(
+	partial: Partial<AccessibilityNode>,
+	depth: number,
+): AccessibilityNode {
+	return {
+		label: partial.label ?? "",
+		role: partial.role ?? "Unknown",
+		value: partial.value ?? "",
+		traits: partial.traits ?? [],
+		frame: partial.frame ?? null,
+		depth,
+	};
+}
+
+/**
+ * Parses the raw text output from `xcrun simctl ui <deviceId> accessibility`
+ * into structured accessibility nodes.
+ */
+export function parseAccessibilityOutput(output: string): AccessibilityNode[] {
+	const nodes: AccessibilityNode[] = [];
+	const lines = output.split("\n");
+
+	let currentNode: Partial<AccessibilityNode> | null = null;
+	let currentDepth = 0;
+
+	for (const line of lines) {
+		const trimmed = line.trimEnd();
+		if (!trimmed) continue;
+
+		// Measure indentation depth
+		const indent = line.length - line.trimStart().length;
+		const depth = Math.floor(indent / 2);
+
+		// Check for element/component start
+		const elementMatch = trimmed.match(/^\s*(?:Element|SBElement|AX\w+):\s*(?:<(\w+)>)?/);
+		if (elementMatch) {
+			// Save previous node if it exists
+			if (currentNode) {
+				nodes.push(normalizeAccessibilityNode(currentNode, currentDepth));
+			}
+
+			currentNode = {
+				role: elementMatch[1] ?? "Unknown",
+			};
+			currentDepth = depth;
+			continue;
+		}
+
+		// Parse properties of the current node
+		if (currentNode) {
+			const labelMatch = trimmed.match(/^\s*Label:\s*"?([^"]*)"?/);
+			if (labelMatch) {
+				currentNode.label = labelMatch[1];
+				continue;
+			}
+
+			const valueMatch = trimmed.match(/^\s*Value:\s*"?([^"]*)"?/);
+			if (valueMatch) {
+				currentNode.value = valueMatch[1];
+				continue;
+			}
+
+			const traitsMatch = trimmed.match(/^\s*Traits?:\s*(.*)/);
+			if (traitsMatch) {
+				currentNode.traits = traitsMatch[1]
+					.split(",")
+					.map((t) => t.trim())
+					.filter(Boolean);
+				continue;
+			}
+
+			// Frame: {{x, y}, {width, height}}
+			const frameMatch = trimmed.match(
+				/^\s*Frame:\s*\{\{([\d.]+),\s*([\d.]+)\},\s*\{([\d.]+),\s*([\d.]+)\}\}/,
+			);
+			if (frameMatch) {
+				currentNode.frame = {
+					x: Number.parseFloat(frameMatch[1]),
+					y: Number.parseFloat(frameMatch[2]),
+					width: Number.parseFloat(frameMatch[3]),
+					height: Number.parseFloat(frameMatch[4]),
+				};
+			}
+		}
+	}
+
+	// Don't forget the last node
+	if (currentNode) {
+		nodes.push(normalizeAccessibilityNode(currentNode, currentDepth));
+	}
+
+	return nodes;
+}
+
+/**
+ * Converts parsed iOS accessibility nodes into MobileElement[].
+ * Uses the shared `mapRole` for semantic role mapping.
+ */
+export function accessibilityNodesToElements(
+	nodes: AccessibilityNode[],
+	platform: Platform,
+): MobileElement[] {
+	const elements: MobileElement[] = [];
+	const pathStack: string[] = [];
+
+	for (let i = 0; i < nodes.length; i++) {
+		const node = nodes[i];
+		if (!node.frame) continue;
+
+		const componentName = node.role ? node.role.replace(/^AX/, "") : "Unknown";
+
+		while (pathStack.length > node.depth) {
+			pathStack.pop();
+		}
+		pathStack.push(componentName);
+
+		const componentPath = pathStack.join("/");
+		const id = node.label
+			? `ios:${componentName}:${node.label}:${i}`
+			: `ios:${componentName}:${node.frame.x},${node.frame.y}:${i}`;
+
+		const element: MobileElement = {
+			id,
+			platform,
+			componentPath,
+			componentName,
+			boundingBox: node.frame,
+		};
+
+		const accessibility: MobileElement["accessibility"] = {};
+		let hasAccessibility = false;
+
+		if (node.label) {
+			accessibility.label = node.label;
+			hasAccessibility = true;
+		}
+
+		const semanticRole = mapRole(node.role);
+		if (semanticRole) {
+			accessibility.role = semanticRole;
+			hasAccessibility = true;
+		}
+
+		if (node.value) {
+			accessibility.value = node.value;
+			hasAccessibility = true;
+		}
+
+		if (node.traits.length > 0) {
+			accessibility.traits = node.traits;
+			hasAccessibility = true;
+		}
+
+		if (hasAccessibility) {
+			element.accessibility = accessibility;
+		}
+
+		if (node.label && node.role === "AXStaticText") {
+			element.textContent = node.label;
+		}
+
+		elements.push(element);
+	}
+
+	return elements;
+}

packages/bridges/bridge-core/src/index.ts

@@ -2,6 +2,14 @@ import type { AnimationInfo, MobileElement, Platform } from "@agentation-mobile/
 
 export { parseUiAutomatorXml, hitTestElement } from "./uiautomator";
 export { parseWmSize } from "./android-utils";
+export {
+	mergeElements,
+	findBestMatch,
+	mapRole,
+	parseAccessibilityOutput,
+	accessibilityNodesToElements,
+} from "./accessibility-merge";
+export type { AccessibilityNode } from "./accessibility-merge";
 export {
 	IOS_UDID_REGEX,
 	isIosSimulatorId,
@@ -33,6 +41,9 @@ export interface IPlatformBridge {
 	/** Get the UI element tree for the current screen */
 	getElementTree(deviceId: string): Promise<MobileElement[]>;
 
+	/** Get the current screen/route identifier (optional) */
+	getScreenId?(deviceId: string): Promise<string | null>;
+
 	/** Inspect a specific element at screen coordinates */
 	inspectElement(deviceId: string, x: number, y: number): Promise<MobileElement | null>;
 

packages/bridges/bridge-core/src/ios-devices.ts

@@ -15,6 +15,11 @@ export function isIosSimulatorId(deviceId: string): boolean {
  * Used as a fallback when dynamic resolution detection is not available.
  */
 export const IOS_SCREEN_SIZES: Record<string, { width: number; height: number }> = {
+	"iPhone 17 Pro Max": { width: 440, height: 956 },
+	"iPhone 17 Pro": { width: 402, height: 874 },
+	"iPhone 17 Plus": { width: 430, height: 932 },
+	"iPhone 17 Air": { width: 402, height: 874 },
+	"iPhone 17": { width: 393, height: 852 },
 	"iPhone 16 Pro Max": { width: 440, height: 956 },
 	"iPhone 16 Pro": { width: 402, height: 874 },
 	"iPhone 16 Plus": { width: 430, height: 932 },