fix(browser): resolve aria snapshot refs via DOM markers

Co-authored-by: MrKipler <mrkipler@kiphausen.com>

Author: steipete

CHANGELOG.md

@@ -15,6 +15,7 @@ Docs: https://docs.openclaw.ai
 ### Fixes
 
 - Browser/Playwright: ignore benign already-handled route races during guarded navigation so browser-page tasks no longer fail when Playwright tears down a route mid-flight. (#68708) Thanks @Steady-ai.
+- Browser/aria snapshots: bind `format=aria` `axN` refs to live DOM nodes through backend DOM ids when Playwright is available, so follow-up browser actions can use those refs without timing out. (#62434) Thanks @MrKipler.
 - Telegram: prevent duplicate in-process long pollers for the same bot token and add clearer `getUpdates` conflict diagnostics for external duplicate pollers. Fixes #56230.
 - Browser/Linux: detect Chromium-based installs under `/opt/google`, `/opt/brave.com`, `/usr/lib/chromium`, and `/usr/lib/chromium-browser` before asking users to set `browser.executablePath`. (#48563) Thanks @lupuletic.
 - Sessions/browser: close tracked browser tabs when idle, daily, `/new`, or `/reset` session rollover archives the previous transcript, preventing tabs from leaking past the old session. Thanks @jakozloski.

extensions/browser/src/browser/pw-ai.ts

@@ -53,6 +53,7 @@ export {
   snapshotAiViaPlaywright,
   snapshotAriaViaPlaywright,
   snapshotRoleViaPlaywright,
+  storeAriaSnapshotRefsViaPlaywright,
   screenshotWithLabelsViaPlaywright,
   storageClearViaPlaywright,
   storageGetViaPlaywright,

extensions/browser/src/browser/pw-session.page-cdp.test.ts

@@ -1,5 +1,9 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { withPageScopedCdpClient } from "./pw-session.page-cdp.js";
+import {
+  BROWSER_REF_MARKER_ATTRIBUTE,
+  markBackendDomRefsOnPage,
+  withPageScopedCdpClient,
+} from "./pw-session.page-cdp.js";
 
 describe("pw-session page-scoped CDP client", () => {
   beforeEach(() => {
@@ -32,4 +36,107 @@ describe("pw-session page-scoped CDP client", () => {
     expect(sessionSend).toHaveBeenCalledWith("Emulation.setLocaleOverride", { locale: "en-US" });
     expect(sessionDetach).toHaveBeenCalledTimes(1);
   });
+
+  it("marks backend DOM refs on the page", async () => {
+    const sessionSend = vi.fn(async (method: string, params?: Record<string, unknown>) => {
+      if (method === "DOM.pushNodesByBackendIdsToFrontend") {
+        expect(params).toEqual({ backendNodeIds: [42, 84] });
+        return { nodeIds: [101, 202] };
+      }
+      return {};
+    });
+    const sessionDetach = vi.fn(async () => {});
+    const newCDPSession = vi.fn(async () => ({
+      send: sessionSend,
+      detach: sessionDetach,
+    }));
+    const evaluateAll = vi.fn(async () => {});
+    const page = {
+      context: () => ({
+        newCDPSession,
+      }),
+      locator: vi.fn(() => ({ evaluateAll })),
+    };
+
+    const marked = await markBackendDomRefsOnPage({
+      page: page as never,
+      refs: [
+        { ref: "ax1", backendDOMNodeId: 42 },
+        { ref: "ax2", backendDOMNodeId: 84 },
+      ],
+    });
+
+    expect(page.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`);
+    expect(evaluateAll).toHaveBeenCalledTimes(1);
+    expect(sessionSend).toHaveBeenNthCalledWith(1, "DOM.enable", undefined);
+    expect(sessionSend).toHaveBeenNthCalledWith(2, "DOM.pushNodesByBackendIdsToFrontend", {
+      backendNodeIds: [42, 84],
+    });
+    expect(sessionSend).toHaveBeenNthCalledWith(3, "DOM.setAttributeValue", {
+      nodeId: 101,
+      name: BROWSER_REF_MARKER_ATTRIBUTE,
+      value: "ax1",
+    });
+    expect(sessionSend).toHaveBeenNthCalledWith(4, "DOM.setAttributeValue", {
+      nodeId: 202,
+      name: BROWSER_REF_MARKER_ATTRIBUTE,
+      value: "ax2",
+    });
+    expect(marked).toEqual(new Set(["ax1", "ax2"]));
+    expect(sessionDetach).toHaveBeenCalledTimes(1);
+  });
+
+  it("clears stale markers even when no backend refs are valid", async () => {
+    const newCDPSession = vi.fn();
+    const evaluateAll = vi.fn(async () => {});
+    const page = {
+      context: () => ({
+        newCDPSession,
+      }),
+      locator: vi.fn(() => ({ evaluateAll })),
+    };
+
+    const marked = await markBackendDomRefsOnPage({
+      page: page as never,
+      refs: [{ ref: "e1", backendDOMNodeId: 0 }],
+    });
+
+    expect(page.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`);
+    expect(evaluateAll).toHaveBeenCalledTimes(1);
+    expect(newCDPSession).not.toHaveBeenCalled();
+    expect(marked).toEqual(new Set());
+  });
+
+  it("keeps unmarked refs out of the marked set when marker writes fail", async () => {
+    const sessionSend = vi.fn(async (method: string) => {
+      if (method === "DOM.pushNodesByBackendIdsToFrontend") {
+        return { nodeIds: [101, 202] };
+      }
+      if (method === "DOM.setAttributeValue") {
+        throw new Error("detached");
+      }
+      return {};
+    });
+    const sessionDetach = vi.fn(async () => {});
+    const page = {
+      context: () => ({
+        newCDPSession: vi.fn(async () => ({
+          send: sessionSend,
+          detach: sessionDetach,
+        })),
+      }),
+      locator: vi.fn(() => ({ evaluateAll: vi.fn(async () => {}) })),
+    };
+
+    const marked = await markBackendDomRefsOnPage({
+      page: page as never,
+      refs: [
+        { ref: "ax1", backendDOMNodeId: 42 },
+        { ref: "ax2", backendDOMNodeId: 84 },
+      ],
+    });
+
+    expect(marked).toEqual(new Set());
+    expect(sessionDetach).toHaveBeenCalledTimes(1);
+  });
 });

extensions/browser/src/browser/pw-session.page-cdp.ts

@@ -1,6 +1,9 @@
 import type { CDPSession, Page } from "playwright-core";
 
 type PageCdpSend = (method: string, params?: Record<string, unknown>) => Promise<unknown>;
+type MarkBackendDomRef = { ref: string; backendDOMNodeId: number };
+
+export const BROWSER_REF_MARKER_ATTRIBUTE = "data-openclaw-browser-ref";
 
 async function withPlaywrightPageCdpSession<T>(
   page: Page,
@@ -31,3 +34,75 @@ export async function withPageScopedCdpClient<T>(opts: {
     );
   });
 }
+
+export async function markBackendDomRefsOnPage(opts: {
+  page: Page;
+  refs: MarkBackendDomRef[];
+}): Promise<Set<string>> {
+  await opts.page
+    .locator(`[${BROWSER_REF_MARKER_ATTRIBUTE}]`)
+    .evaluateAll((elements, attr) => {
+      for (const element of elements) {
+        if (element instanceof Element) {
+          element.removeAttribute(attr);
+        }
+      }
+    }, BROWSER_REF_MARKER_ATTRIBUTE)
+    .catch(() => {});
+
+  const refs = opts.refs.filter(
+    (entry) =>
+      /^ax\d+$/.test(entry.ref) &&
+      Number.isFinite(entry.backendDOMNodeId) &&
+      Math.floor(entry.backendDOMNodeId) > 0,
+  );
+  const marked = new Set<string>();
+  if (!refs.length) {
+    return marked;
+  }
+
+  return await withPlaywrightPageCdpSession(opts.page, async (session) => {
+    const send = async (method: string, params?: Record<string, unknown>) =>
+      await (
+        session.send as unknown as (
+          method: string,
+          params?: Record<string, unknown>,
+        ) => Promise<unknown>
+      )(method, params);
+
+    await send("DOM.enable").catch(() => {});
+
+    const backendNodeIds = [...new Set(refs.map((entry) => Math.floor(entry.backendDOMNodeId)))];
+    const pushed = (await send("DOM.pushNodesByBackendIdsToFrontend", {
+      backendNodeIds,
+    }).catch(() => ({}))) as { nodeIds?: number[] };
+    const nodeIds = Array.isArray(pushed.nodeIds) ? pushed.nodeIds : [];
+    const nodeIdByBackendId = new Map<number, number>();
+    for (let index = 0; index < backendNodeIds.length; index += 1) {
+      const backendNodeId = backendNodeIds[index];
+      const nodeId = nodeIds[index];
+      if (backendNodeId && typeof nodeId === "number" && nodeId > 0) {
+        nodeIdByBackendId.set(backendNodeId, nodeId);
+      }
+    }
+
+    for (const entry of refs) {
+      const nodeId = nodeIdByBackendId.get(Math.floor(entry.backendDOMNodeId));
+      if (!nodeId) {
+        continue;
+      }
+      try {
+        await send("DOM.setAttributeValue", {
+          nodeId,
+          name: BROWSER_REF_MARKER_ATTRIBUTE,
+          value: entry.ref,
+        });
+        marked.add(entry.ref);
+      } catch {
+        // Best-effort marker write. Unmarked refs fall back to role metadata.
+      }
+    }
+
+    return marked;
+  });
+}

extensions/browser/src/browser/pw-session.test.ts

@@ -6,6 +6,7 @@ import {
   rememberRoleRefsForTarget,
   restoreRoleRefsForTarget,
 } from "./pw-session.js";
+import { BROWSER_REF_MARKER_ATTRIBUTE } from "./pw-session.page-cdp.js";
 
 function fakePage(): {
   page: Page;
@@ -27,6 +28,7 @@ function fakePage(): {
   const getByRole = vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) }));
   const frameLocator = vi.fn(() => ({
     getByRole: vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })),
+    locator: vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) })),
   }));
   const locator = vi.fn(() => ({ nth: vi.fn(() => ({ ok: true })) }));
 
@@ -72,10 +74,30 @@ describe("pw-session refLocator", () => {
     expect(mocks.locator).toHaveBeenCalledWith("aria-ref=e1");
   });
 
-  it("rejects axN refs from format=aria snapshots instead of timing out", () => {
+  it("uses backend-marked DOM locators for ax refs", () => {
+    const { page, mocks } = fakePage();
+    const state = ensurePageState(page);
+    state.roleRefs = { ax12: { role: "button", name: "OK", domMarker: true } };
+
+    refLocator(page, "ax12");
+
+    expect(mocks.locator).toHaveBeenCalledWith(`[${BROWSER_REF_MARKER_ATTRIBUTE}="ax12"]`);
+  });
+
+  it("falls back to role heuristics for ax refs without backend markers", () => {
+    const { page, mocks } = fakePage();
+    const state = ensurePageState(page);
+    state.roleRefs = { ax12: { role: "button", name: "OK" } };
+
+    refLocator(page, "ax12");
+
+    expect(mocks.getByRole).toHaveBeenCalledWith("button", { name: "OK", exact: true });
+  });
+
+  it("rejects unknown ax refs instead of timing out on aria-ref locators", () => {
     const { page, mocks } = fakePage();
 
-    expect(() => refLocator(page, "ax12")).toThrow(/format=aria snapshot/);
+    expect(() => refLocator(page, "ax12")).toThrow(/Unknown ref/);
     expect(mocks.locator).not.toHaveBeenCalled();
   });
 });

extensions/browser/src/browser/pw-session.ts

@@ -31,7 +31,7 @@ import {
   InvalidBrowserNavigationUrlError,
   withBrowserNavigationPolicy,
 } from "./navigation-guard.js";
-import { withPageScopedCdpClient } from "./pw-session.page-cdp.js";
+import { BROWSER_REF_MARKER_ATTRIBUTE, withPageScopedCdpClient } from "./pw-session.page-cdp.js";
 
 export type BrowserConsoleMessage = {
   type: string;
@@ -84,7 +84,7 @@ type PageState = {
    * Mode "role" refs are generated from ariaSnapshot and resolved via getByRole.
    * Mode "aria" refs are Playwright aria-ref ids and resolved via `aria-ref=...`.
    */
-  roleRefs?: Record<string, { role: string; name?: string; nth?: number }>;
+  roleRefs?: Record<string, { role: string; name?: string; nth?: number; domMarker?: boolean }>;
   roleRefsMode?: "role" | "aria";
   roleRefsFrameSelector?: string;
 };
@@ -935,10 +935,29 @@ export function refLocator(page: Page, ref: string) {
   }
 
   if (AX_REF_PATTERN.test(normalized)) {
-    throw new Error(
-      `Ref "${normalized}" comes from a format=aria snapshot and cannot be used with act. ` +
-        `Re-snapshot with format=ai and use the eN refs from that snapshot.`,
-    );
+    const state = pageStates.get(page);
+    const info = state?.roleRefs?.[normalized];
+    if (!info) {
+      throw new Error(
+        `Unknown ref "${normalized}". Run a new snapshot and use a ref from that snapshot.`,
+      );
+    }
+    const scope = state.roleRefsFrameSelector
+      ? page.frameLocator(state.roleRefsFrameSelector)
+      : page;
+    if (info.domMarker) {
+      return scope.locator(`[${BROWSER_REF_MARKER_ATTRIBUTE}="${normalized}"]`);
+    }
+    const locAny = scope as unknown as {
+      getByRole: (
+        role: never,
+        opts?: { name?: string; exact?: boolean },
+      ) => ReturnType<Page["getByRole"]>;
+    };
+    const locator = info.name
+      ? locAny.getByRole(info.role as never, { name: info.name, exact: true })
+      : locAny.getByRole(info.role as never);
+    return info.nth !== undefined ? locator.nth(info.nth) : locator;
   }
 
   return page.locator(`aria-ref=${normalized}`);

extensions/browser/src/browser/pw-tools-core.browser-ssrf-guard.test.ts

@@ -27,6 +27,7 @@ const sessionMocks = vi.hoisted(() => ({
 }));
 
 const pageCdpMocks = vi.hoisted(() => ({
+  markBackendDomRefsOnPage: vi.fn(async () => new Set<string>()),
   withPageScopedCdpClient: vi.fn(
     async ({ fn }: { fn: (send: () => Promise<unknown>) => unknown }) =>
       await fn(async () => ({ nodes: [] })),