From bacabf31d8cf3773c6ad2e343cf09c59b13ad5dc Mon Sep 17 00:00:00 2001 From: Brian Love Date: Tue, 8 Sep 2026 21:36:13 -0700 Subject: [PATCH] fix(mastra): verify cancellation outcomes with aimock --- cockpit/runtimes/mastra/angular/e2e/README.md | 43 +++++++++++++++++ .../mastra/angular/e2e/fixtures/mastra.json | 10 +++- .../mastra/angular/e2e/global-setup-impl.ts | 7 +-- .../mastra/angular/e2e/mastra.spec.ts | 47 +++++++++++++++---- 4 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 cockpit/runtimes/mastra/angular/e2e/README.md diff --git a/cockpit/runtimes/mastra/angular/e2e/README.md b/cockpit/runtimes/mastra/angular/e2e/README.md new file mode 100644 index 000000000..49cf7e653 --- /dev/null +++ b/cockpit/runtimes/mastra/angular/e2e/README.md @@ -0,0 +1,43 @@ +# Mastra interrupt verification + +The reservation tests exercise both Approve and Cancel through the browser, +check the correlated resume command and backend tool result, and assert that +the visible assistant reply agrees with that result. + +Run the full suite with aimock replay: + +```sh +npx nx e2e cockpit-runtimes-mastra-angular +``` + +To verify the same decisions against the live model through aimock, configure +`OPENAI_API_KEY` in the environment and run: + +```sh +AIMOCK_MODE=record AIMOCK_RECORD_DIR=/tmp/mastra-approve-recordings \ + npx nx e2e cockpit-runtimes-mastra-angular --grep='Approve sends' +AIMOCK_MODE=record AIMOCK_RECORD_DIR=/tmp/mastra-cancel-recordings \ + npx nx e2e cockpit-runtimes-mastra-angular --grep='Cancel sends' +``` + +Use a fresh process for each decision. Aimock caches newly recorded responses +in memory, and its generated matchers do not distinguish these two tool-result +payloads. Running both decisions in one recording process can reuse the first +decision's reply instead of verifying the second against the live model. + +Record mode forwards unmatched model requests to the provider and captures responses. +The campsite tool remains local: it returns a synthetic confirmation or decline +and does not make an external reservation. Each run uses a fresh local database. + +Keep approval and cancellation continuation fixtures separate using +`toolResultContains`. Matching only the user prompt and `hasToolResult` can +replay an approval after a cancellation because both decisions share the same +prompt. The initial reservation fixture excludes tool results, so an unknown +continuation cannot restart the tool call silently. + +The cancellation reply was captured from `gpt-4o-mini` through aimock record +mode on 2026-09-09, then given a stable tool-result matcher for replay. + +Review captures before incorporating them into fixtures. Retain stable outcome +matchers rather than generated tool-call IDs, and do not commit credentials or +raw browser traces. diff --git a/cockpit/runtimes/mastra/angular/e2e/fixtures/mastra.json b/cockpit/runtimes/mastra/angular/e2e/fixtures/mastra.json index a06e26920..278dbf970 100644 --- a/cockpit/runtimes/mastra/angular/e2e/fixtures/mastra.json +++ b/cockpit/runtimes/mastra/angular/e2e/fixtures/mastra.json @@ -13,11 +13,17 @@ } }, { - "match": { "userMessage": "reserve", "hasToolResult": true }, + "match": { "userMessage": "reserve", "hasToolResult": true, "toolResultContains": "Reserved North Pines" }, "response": { "content": "North Pines is reserved for 2 nights — confirmation TP-0288." } }, + { + "match": { "userMessage": "reserve", "hasToolResult": true, "toolResultContains": "Nothing was booked." }, + "response": { + "content": "The reservation for the North Pines campsite was not booked." + } + }, { "match": { "userMessage": "Bear Lake", "hasToolResult": true }, "response": { @@ -63,7 +69,7 @@ } }, { - "match": { "userMessage": "reserve" }, + "match": { "userMessage": "reserve", "hasToolResult": false }, "response": { "toolCalls": [ { diff --git a/cockpit/runtimes/mastra/angular/e2e/global-setup-impl.ts b/cockpit/runtimes/mastra/angular/e2e/global-setup-impl.ts index 7af039acf..44aabe236 100644 --- a/cockpit/runtimes/mastra/angular/e2e/global-setup-impl.ts +++ b/cockpit/runtimes/mastra/angular/e2e/global-setup-impl.ts @@ -12,7 +12,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { portsFor } from '../../../../../cockpit/ports.mjs'; -import { startAimock, type AimockHandle } from '@threadplane-internal/e2e-harness'; +import { resolveAimockLaunch, startAimock, type AimockHandle } from '@threadplane-internal/e2e-harness'; const ports = portsFor('cockpit-runtimes-mastra-angular'); const angularProject = 'cockpit-runtimes-mastra-angular'; @@ -57,7 +57,8 @@ export default async function globalSetup(): Promise { const fixturesDir = resolve(__dirname, 'fixtures'); const serviceDir = resolve(root, backendCwd); - const aimock = await startAimock({ mode: 'replay', fixturePath: fixturesDir }); + const launch = resolveAimockLaunch(fixturesDir); + const aimock = await startAimock(launch.startOptions); console.log(`[mastra-harness] aimock listening at ${aimock.baseUrl}`); // The service is self-contained (own package.json + lockfile, deps NOT in @@ -77,7 +78,7 @@ export default async function globalSetup(): Promise { ...process.env, PORT: String(backendPort), AG_UI_INTERNAL_TOKEN: INTERNAL_TOKEN, - OPENAI_API_KEY: 'test-not-used', + OPENAI_API_KEY: launch.openaiApiKey, OPENAI_BASE_URL: aimock.baseUrl, AG_UI_MASTRA_DB_PATH: join(dbDir, 'mastra.db'), }, diff --git a/cockpit/runtimes/mastra/angular/e2e/mastra.spec.ts b/cockpit/runtimes/mastra/angular/e2e/mastra.spec.ts index 3cdb2423b..7fa6e1b98 100644 --- a/cockpit/runtimes/mastra/angular/e2e/mastra.spec.ts +++ b/cockpit/runtimes/mastra/angular/e2e/mastra.spec.ts @@ -35,14 +35,45 @@ test.describe('cockpit runtimes/mastra: camping trip planner', () => { await expect(dialog).toContainText('$90.00'); }); - test('Approve resumes the suspended run and the reservation completes', async ({ page }) => { - await page.goto('/'); - await page.getByText('Reserve the campsite').click(); - const dialog = page.locator('dialog.chat-approval-card'); - await expect(dialog).toBeVisible({ timeout: 30_000 }); - await dialog.getByRole('button', { name: 'Approve' }).click(); - await expect(page.getByText(/reserved for 2 nights/i)).toBeVisible({ timeout: 30_000 }); - }); + for (const approved of [true, false]) { + const action = approved ? 'Approve' : 'Cancel'; + test(`${action} sends the decision and displays the matching reservation outcome`, async ({ page }) => { + if (process.env['AIMOCK_MODE'] === 'record') test.setTimeout(120_000); + await page.goto('/'); + await page.getByText('Reserve the campsite').click(); + const dialog = page.locator('dialog.chat-approval-card'); + await expect(dialog).toBeVisible({ timeout: 30_000 }); + await expect(dialog).toContainText('North Pines'); + await expect(dialog).toContainText('$90.00'); + + const resumed = page.waitForResponse(response => { + if (response.request().method() !== 'POST' || !response.url().endsWith('/agent')) return false; + return response.request().postDataJSON()?.forwardedProps?.command?.resume?.approved === approved; + }); + await dialog.getByRole('button', { name: action, exact: true }).click(); + const response = await resumed; + expect(response.status()).toBe(200); + const command = response.request().postDataJSON().forwardedProps.command; + expect(command.interruptEvent.toolCallId).toEqual(expect.any(String)); + expect(command.interruptEvent.runId).toEqual(expect.any(String)); + const events = (await response.text()).split('\n') + .filter(line => line.startsWith('data: ')) + .map(line => JSON.parse(line.slice(6))); + const result = events.find(event => event.type === 'TOOL_CALL_RESULT'); + expect(result?.toolCallId).toBe(command.interruptEvent.toolCallId); + expect(result?.content).toContain(approved ? 'Reserved North Pines' : 'Nothing was booked.'); + await expect(dialog).not.toBeVisible(); + const reply = page.locator('chat-message[data-role="assistant"]').last(); + await expect(reply).toContainText(approved + ? /reserved|confirmed|booked/i + : /declined|cancelled|canceled|not.{0,20}(booked|completed|confirmed|reserved)|nothing.{0,20}booked/i); + if (approved) { + await expect(reply).not.toContainText(/declined|cancelled|canceled|nothing.{0,20}booked|not.{0,20}(booked|completed|confirmed|reserved)/i); + } else { + await expect(reply).not.toContainText('TP-0288'); + } + }); + } // Delegation: the supervisor calls the registered weather_forecaster // sub-agent (wire tool `agent-weather_forecaster`); the server-side