From 9b48a1b7040a98ed4278d3452bfc947e16af6c7f Mon Sep 17 00:00:00 2001 From: Jake Gaylor Date: Sun, 6 Sep 2026 06:28:08 -0400 Subject: [PATCH 1/3] Wait for LiveView hydration in deployed browser checks --- deployed/lib/browser-liveview.mjs | 6 ++++++ deployed/profiles/browser-console.mjs | 5 +++++ deployed/profiles/browser-credentials.mjs | 3 +++ 3 files changed, 14 insertions(+) create mode 100644 deployed/lib/browser-liveview.mjs diff --git a/deployed/lib/browser-liveview.mjs b/deployed/lib/browser-liveview.mjs new file mode 100644 index 000000000..c90e77a02 --- /dev/null +++ b/deployed/lib/browser-liveview.mjs @@ -0,0 +1,6 @@ +// Server-rendered controls can be visible before LiveView installs its event +// handlers. Wait for the connection before filling or submitting those forms; +// Playwright's visibility/actionability checks do not establish hydration. +export async function waitForLiveView(page) { + await page.locator('[data-phx-main].phx-connected').waitFor({ state: 'attached' }); +} diff --git a/deployed/profiles/browser-console.mjs b/deployed/profiles/browser-console.mjs index c30dea0f4..4832cfee4 100644 --- a/deployed/profiles/browser-console.mjs +++ b/deployed/profiles/browser-console.mjs @@ -1,6 +1,7 @@ import { reserveBrowserFixture, adoptBrowserFixture } from '../lib/browser-fixtures.mjs'; import { ensure } from '../lib/execution.mjs'; import { browserCredentials } from './browser-credentials.mjs'; +import { waitForLiveView } from '../lib/browser-liveview.mjs'; // Called with a fresh browser context by the browser driver. All writes go // through visible console forms. Public API reads independently establish @@ -29,6 +30,7 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { await evidence.step('console/create-agent', async () => { const intent = reserveBrowserFixture(fixtures, 'agent'); await page.goto(`${config.base_url}/agents/new`); + await waitForLiveView(page); await page.getByLabel('Name', { exact: true }).fill(intent.name); await page.getByLabel('Description', { exact: true }).fill('Dedicated deployed browser fixture'); await page.getByLabel('System prompt', { exact: true }).fill('Perform only the requested small file task. Use a shell tool. Do not access the network or start background work.'); @@ -49,6 +51,7 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { await evidence.step('console/edit-agent', async () => { await page.goto(`${config.base_url}/agents/${agent.id}/edit`); + await waitForLiveView(page); const description = `Browser edit verified for ${report.run_id}`; await page.getByLabel('Description', { exact: true }).fill(description); await page.getByRole('button', { name: 'Save', exact: true }).click(); @@ -61,6 +64,7 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { await evidence.step('console/api-key-lifecycle', async () => { const intent = reserveBrowserFixture(fixtures, 'api_key'); await page.goto(`${config.base_url}/api-keys`); + await waitForLiveView(page); await page.getByLabel('Key label', { exact: true }).fill(intent.name); await page.getByRole('button', { name: 'Create key', exact: true }).click(); await page.getByText('New API key created', { exact: true }).waitFor({ state: 'visible' }); @@ -73,6 +77,7 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { ensure(me.body.id === report.owner_id, 'UI key authenticated as another account'); await page.getByRole('button', { name: "I've copied it, dismiss", exact: true }).click(); if (settings.conversations) await handoff(agent, key); + await waitForLiveView(page); const row = page.getByRole('row').filter({ hasText: intent.name }); page.once('dialog', dialog => dialog.accept()); await row.getByRole('button', { name: 'Revoke', exact: true }).click(); diff --git a/deployed/profiles/browser-credentials.mjs b/deployed/profiles/browser-credentials.mjs index c2bfebd4b..76050fff0 100644 --- a/deployed/profiles/browser-credentials.mjs +++ b/deployed/profiles/browser-credentials.mjs @@ -1,5 +1,6 @@ import { credentialPath, credentialStatus, reserveCredential } from '../lib/browser-credentials.mjs'; import { ensure } from '../lib/execution.mjs'; +import { waitForLiveView } from '../lib/browser-liveview.mjs'; export async function browserCredentials(ctx, page, evidence) { const { config, fixtures, client, report, env, redactor, signal } = ctx; @@ -10,6 +11,7 @@ export async function browserCredentials(ctx, page, evidence) { await evidence.step('console/credential-validation', async () => { const before = await client.request('GET', credentialPath, { expected: 200, recordBody: false }); await page.goto(`${config.base_url}/account/inference-credentials`); + await waitForLiveView(page); ensure(await form().locator('input[name="value"]').getAttribute('type') === 'password', 'Credential field must hide its value'); await form().getByRole('button', { name: 'Save', exact: true }).click(); await page.getByText('Paste a value before saving.', { exact: true }).waitFor({ state: 'visible' }); @@ -37,6 +39,7 @@ export async function browserCredentials(ctx, page, evidence) { return async () => evidence.step('console/credential-clear', async () => { await page.goto(`${config.base_url}/account/inference-credentials`); + await waitForLiveView(page); await form().getByRole('button', { name: 'Clear', exact: true }).click(); await page.getByText('Credential cleared.', { exact: true }).waitFor({ state: 'visible' }); ensure(!await credentialStatus(client, provider, signal), 'Cleared provider credential remains set'); From 0e8e253659ae6a6d4ddf724417b0b60ab80d26d0 Mon Sep 17 00:00:00 2001 From: Jake Gaylor Date: Sun, 6 Sep 2026 06:32:39 -0400 Subject: [PATCH 2/3] Wait for the catalog-required browser provider selector --- deployed/profiles/browser-console.mjs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/deployed/profiles/browser-console.mjs b/deployed/profiles/browser-console.mjs index 4832cfee4..82d0a6097 100644 --- a/deployed/profiles/browser-console.mjs +++ b/deployed/profiles/browser-console.mjs @@ -37,15 +37,18 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { await page.locator('select[name="agent[runtime]"]').selectOption(settings.agent.runtime); await page.getByLabel('Model', { exact: true }).fill(settings.agent.model); const provider = page.locator('select[name="agent[sandbox_provider]"]'); - if (await provider.count()) await provider.selectOption(settings.agent.sandbox_provider); - else { - const { body } = await client.request('GET', '/api/catalog', { expected: 200 }); - ensure(JSON.stringify(body.data.sandbox_providers.enabled) === JSON.stringify([settings.agent.sandbox_provider]), 'Hidden provider selector does not imply the pinned sole provider'); - } + const { body: catalog } = await client.request('GET', '/api/catalog', { expected: 200 }); + const enabledProviders = catalog.data.sandbox_providers.enabled; + ensure(enabledProviders.includes(settings.agent.sandbox_provider), 'Pinned browser provider is not enabled'); + // Runtime/model validation can temporarily replace the form. A zero count + // during that patch is not evidence of a single-provider deployment. + if (enabledProviders.length > 1 || await provider.count()) await provider.selectOption(settings.agent.sandbox_provider); + else ensure(JSON.stringify(enabledProviders) === JSON.stringify([settings.agent.sandbox_provider]), 'Hidden provider selector does not imply the pinned sole provider'); await page.getByRole('button', { name: 'Save', exact: true }).click(); await page.waitForURL(`${config.base_url}/agents`); agent = await adoptBrowserFixture(fixtures, intent, ctx.signal); ensure(agent.runtime === settings.agent.runtime && agent.model === settings.agent.model, 'UI agent runtime/model did not persist'); + ensure((agent.sandbox_provider ?? catalog.data.sandbox_providers.default) === settings.agent.sandbox_provider, 'UI agent sandbox provider did not persist'); report.browser.console.agent_id = agent.id; }); From 492c8e7e15ae64d22379965994962a169bd5c358 Mon Sep 17 00:00:00 2001 From: Jake Gaylor Date: Mon, 7 Sep 2026 07:51:25 -0400 Subject: [PATCH 3/3] Require an explicit provider selection to persist in the console check A multi-provider deployment, or a sole provider whose selector is visible, selects the pinned provider explicitly, so the saved agent must store it. The `?? default` fallback let a null stored provider pass whenever the instance default happened to equal the pin. Only a hidden sole-provider selector may rely on the effective instance default. The selection and persistence steps are lifted into selectBrowserProvider and verifyBrowserProvider so browser-console.test.mjs can drive them against a fake selector; the two fix-targeting tests fail against the old fallback. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JRqdEnAYoRN73f7qjpizjL --- deployed/profiles/browser-console.mjs | 32 +++++++++++++++++----- deployed/test/browser-console.test.mjs | 38 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 deployed/test/browser-console.test.mjs diff --git a/deployed/profiles/browser-console.mjs b/deployed/profiles/browser-console.mjs index 82d0a6097..c3176c076 100644 --- a/deployed/profiles/browser-console.mjs +++ b/deployed/profiles/browser-console.mjs @@ -3,6 +3,29 @@ import { ensure } from '../lib/execution.mjs'; import { browserCredentials } from './browser-credentials.mjs'; import { waitForLiveView } from '../lib/browser-liveview.mjs'; +// Picks the pinned provider in the new-agent form. Returns whether an explicit +// selection was made; a hidden selector is only acceptable when the catalog +// enables exactly the pinned provider. +export async function selectBrowserProvider(provider, catalog, pinned) { + const enabledProviders = catalog.data.sandbox_providers.enabled; + ensure(enabledProviders.includes(pinned), 'Pinned browser provider is not enabled'); + // Runtime/model validation can temporarily replace the form. A zero count + // during that patch is not evidence of a single-provider deployment. + const selected = enabledProviders.length > 1 || (await provider.count()) > 0; + if (selected) await provider.selectOption(pinned); + else ensure(JSON.stringify(enabledProviders) === JSON.stringify([pinned]), 'Hidden provider selector does not imply the pinned sole provider'); + return selected; +} + +// An explicit selection must persist as the stored value. Only a hidden +// sole-provider selector may rely on the instance default. +export function verifyBrowserProvider(agent, catalog, pinned, selected) { + const savedProvider = selected + ? agent.sandbox_provider + : (agent.sandbox_provider ?? catalog.data.sandbox_providers.default); + ensure(savedProvider === pinned, 'UI agent sandbox provider did not persist'); +} + // Called with a fresh browser context by the browser driver. All writes go // through visible console forms. Public API reads independently establish // identity, persistence and revocation; they never create the UI fixtures. @@ -38,17 +61,12 @@ export async function browserConsole(ctx, page, evidence, { handoff } = {}) { await page.getByLabel('Model', { exact: true }).fill(settings.agent.model); const provider = page.locator('select[name="agent[sandbox_provider]"]'); const { body: catalog } = await client.request('GET', '/api/catalog', { expected: 200 }); - const enabledProviders = catalog.data.sandbox_providers.enabled; - ensure(enabledProviders.includes(settings.agent.sandbox_provider), 'Pinned browser provider is not enabled'); - // Runtime/model validation can temporarily replace the form. A zero count - // during that patch is not evidence of a single-provider deployment. - if (enabledProviders.length > 1 || await provider.count()) await provider.selectOption(settings.agent.sandbox_provider); - else ensure(JSON.stringify(enabledProviders) === JSON.stringify([settings.agent.sandbox_provider]), 'Hidden provider selector does not imply the pinned sole provider'); + const selectedProvider = await selectBrowserProvider(provider, catalog, settings.agent.sandbox_provider); await page.getByRole('button', { name: 'Save', exact: true }).click(); await page.waitForURL(`${config.base_url}/agents`); agent = await adoptBrowserFixture(fixtures, intent, ctx.signal); ensure(agent.runtime === settings.agent.runtime && agent.model === settings.agent.model, 'UI agent runtime/model did not persist'); - ensure((agent.sandbox_provider ?? catalog.data.sandbox_providers.default) === settings.agent.sandbox_provider, 'UI agent sandbox provider did not persist'); + verifyBrowserProvider(agent, catalog, settings.agent.sandbox_provider, selectedProvider); report.browser.console.agent_id = agent.id; }); diff --git a/deployed/test/browser-console.test.mjs b/deployed/test/browser-console.test.mjs new file mode 100644 index 000000000..4f35107f0 --- /dev/null +++ b/deployed/test/browser-console.test.mjs @@ -0,0 +1,38 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { selectBrowserProvider, verifyBrowserProvider } from '../profiles/browser-console.mjs'; + +const pinned = 'sprites'; + +// Drives the console create-agent provider steps against a fake selector: +// `visible` is whether the form currently renders the provider select, +// `stored` is what the public API reads back for the saved agent. +async function createAgent({ enabled, visible, stored, defaultProvider = pinned }) { + const selections = []; + const provider = { async count() { return visible ? 1 : 0; }, async selectOption(value) { selections.push(value); } }; + const catalog = { data: { sandbox_providers: { enabled, default: defaultProvider } } }; + const selected = await selectBrowserProvider(provider, catalog, pinned); + const agent = { sandbox_provider: stored }; + verifyBrowserProvider(agent, catalog, pinned, selected); + return { selections, agent }; +} + +test('multiple providers require the explicit selection to persist even when it equals the default', async () => { + const scenario = { enabled: ['sprites', 'runner'], visible: true }; + await assert.rejects(createAgent({ ...scenario, stored: null }), /UI agent sandbox provider did not persist/); + const result = await createAgent({ ...scenario, stored: 'sprites' }); + assert.deepEqual(result.selections, ['sprites']); + assert.equal(result.agent.sandbox_provider, 'sprites'); +}); + +test('a visible sole-provider selector also requires explicit persistence', async () => { + await assert.rejects(createAgent({ enabled: ['sprites'], visible: true, stored: null }), /UI agent sandbox provider did not persist/); +}); + +test('a hidden sole-provider selector permits the effective instance default', async () => { + const scenario = { enabled: ['sprites'], visible: false, stored: null }; + const result = await createAgent(scenario); + assert.deepEqual(result.selections, []); + assert.equal(result.agent.sandbox_provider, null); + await assert.rejects(createAgent({ ...scenario, defaultProvider: 'runner' }), /UI agent sandbox provider did not persist/); +});