From c795da3d007434fd7b59dc98084d2c716878e144 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Sun, 13 Sep 2026 20:18:09 -0700 Subject: [PATCH] test(website): cover live production approval flows --- .../e2e/platform-production-smoke.spec.ts | 144 +++++++++++++++++- 1 file changed, 141 insertions(+), 3 deletions(-) diff --git a/apps/website/e2e/platform-production-smoke.spec.ts b/apps/website/e2e/platform-production-smoke.spec.ts index 5f259163a..c7aa270f8 100644 --- a/apps/website/e2e/platform-production-smoke.spec.ts +++ b/apps/website/e2e/platform-production-smoke.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test, type Page, type Response } from '@playwright/test'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { validateRuntimeParentOrigins } from '@threadplane/cockpit-runtime-bridge'; @@ -13,7 +13,10 @@ import { * * Requires: * EXAMPLES_URL - e.g. https://examples.threadplane.ai - * OPENAI_API_KEY - optional; enables the single live-provider canary + * OPENAI_API_KEY - optional; enables the canonical LangGraph telemetry canary + * + * AG-UI canaries always exercise the deployed services' own provider keys. + * They must not skip when the test runner has no local provider key. * * Run: * PRODUCTION_SMOKE=true \ @@ -25,6 +28,8 @@ const EXAMPLES_URL = process.env['EXAMPLES_URL'] ?? 'https://examples.threadplane.ai'; const DEMO_URL = process.env['DEMO_URL'] ?? 'https://demo.threadplane.ai'; const WEBSITE_URL = process.env['WEBSITE_URL'] ?? 'https://threadplane.ai'; +const AG_UI_DEMO_URL = + process.env['AG_UI_DEMO_URL'] ?? 'https://ag-ui.threadplane.ai'; // Playwright transpiles specs to CJS, so `import.meta.url` here compiles to a // `require` the ESM-loaded output cannot resolve and the whole file fails to // load. `__dirname` is what the emitted module actually has. Don't "modernise" @@ -393,8 +398,141 @@ test.describe('examples langgraph proxy hardening', () => { }); }); +// A provider authentication error can close an HTTP-200 stream before an +// interrupt or reply arrives. Require protocol completion, not just transport +// success, so this fails promptly with a useful diagnostic in that case. +async function completedAgentEvents(response: Response) { + expect(response.status(), response.url()).toBe(200); + const events = (await response.text()) + .split(/\r?\n/) + .filter((line) => line.startsWith('data:')) + .map((line) => JSON.parse(line.slice(5).trim()) as Record); + const types = events.map((event) => event['type']); + expect(types, 'Agent returned RUN_ERROR').not.toContain('RUN_ERROR'); + expect( + types, + 'Agent stream ended without RUN_FINISHED; check deployed provider credentials and runtime logs' + ).toContain('RUN_FINISHED'); + return events; +} + +function nextAgentResponse(page: Page) { + return page.waitForResponse( + (response) => + response.request().method() === 'POST' && + new URL(response.url()).pathname.endsWith('/agent'), + { timeout: 60_000 } + ); +} + +test.describe('Production: live AG-UI provider canaries', () => { + // These are synthetic demo operations. Run sequentially and bound retries + // separately from dev-server startup retries to limit live provider usage. + test.describe.configure({ mode: 'default', timeout: 120_000, retries: 1 }); + + for (const runtime of ['refund', 'mastra'] as const) { + for (const approved of [true, false]) { + const action = approved ? 'Approve' : 'Cancel'; + test(`${runtime}: ${action} completes the live approval flow`, async ({ + page, + }) => { + const mastra = runtime === 'mastra'; + await page.goto( + `${EXAMPLES_URL}/${mastra ? 'runtimes/mastra' : 'ag-ui/interrupts'}/` + ); + const initialResponse = nextAgentResponse(page); + await page + .getByText( + mastra ? 'Reserve the campsite' : 'Refund a duplicate charge', + { exact: true } + ) + .click(); + const initialEvents = await completedAgentEvents(await initialResponse); + expect(initialEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'CUSTOM', name: 'on_interrupt' }), + ]) + ); + const dialog = page.locator('dialog.chat-approval-card'); + await expect(dialog).toBeVisible(); + await expect(dialog).toContainText( + mastra ? 'Reservation approval required' : 'Refund approval required' + ); + await expect(dialog).toContainText( + mastra ? 'North Pines' : 'cus_a8x2k' + ); + await expect(dialog).toContainText(mastra ? '$90.00' : '$47.50'); + + const resumedResponse = nextAgentResponse(page); + await dialog.getByRole('button', { name: action, exact: true }).click(); + const response = await resumedResponse; + const command = response.request().postDataJSON() + .forwardedProps.command; + expect(command.resume.approved).toBe(approved); + const events = await completedAgentEvents(response); + await expect(dialog).not.toBeVisible(); + const reply = page + .locator('chat-message[data-role="assistant"]') + .last(); + + if (mastra) { + expect(command.interruptEvent.toolCallId).toEqual(expect.any(String)); + expect(command.interruptEvent.runId).toEqual(expect.any(String)); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'TOOL_CALL_RESULT', + toolCallId: command.interruptEvent.toolCallId, + content: expect.stringContaining( + approved ? 'Reserved North Pines' : 'Nothing was booked.' + ), + }), + ]) + ); + const cancellation = + /declined|cancelled|canceled|not.{0,20}(booked|completed|confirmed|reserved)|nothing.{0,20}booked/i; + await expect(reply).toContainText( + approved ? /reserved|confirmed|booked/i : cancellation + ); + if (approved) await expect(reply).not.toContainText(cancellation); + else await expect(reply).not.toContainText('TP-0288'); + } else { + await expect(reply).toContainText( + approved + ? 'Refund of $47.50 issued to' + : 'Refund cancelled by operator. No charge issued.' + ); + if (approved) await expect(reply).toContainText('cus_a8x2k'); + else await expect(reply).not.toContainText('Refund ID:'); + } + }); + } + } + + test('AG-UI demo completes a live reply', async ({ page }) => { + await page.goto(`${AG_UI_DEMO_URL}/embed`); + await page + .locator('textarea[name="messageText"]') + .fill('Say hello in one sentence.'); + const response = nextAgentResponse(page); + await page.getByRole('button', { name: /send message/i }).click(); + const events = await completedAgentEvents(await response); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'TEXT_MESSAGE_CONTENT', + delta: expect.stringMatching(/\S/), + }), + ]) + ); + await expect( + page.locator('chat-message[data-role="assistant"]').last() + ).toContainText(/\S/); + }); +}); + test.describe('AG-UI demo (ag-ui.threadplane.ai)', () => { - const DEMO = process.env['AG_UI_DEMO_URL'] ?? 'https://ag-ui.threadplane.ai'; + const DEMO = AG_UI_DEMO_URL; test('demo SPA is reachable', async ({ page }) => { const res = await page.goto(`${DEMO}/`);