Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions cockpit/runtimes/mastra/angular/e2e/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Mastra interrupt verification

The reservation tests exercise both Approve and Cancel through the browser,
check the correlated resume command and backend tool result, and assert that
the visible assistant reply agrees with that result.

Run the full suite with aimock replay:

```sh
npx nx e2e cockpit-runtimes-mastra-angular
```

To verify the same decisions against the live model through aimock, configure
`OPENAI_API_KEY` in the environment and run:

```sh
AIMOCK_MODE=record AIMOCK_RECORD_DIR=/tmp/mastra-approve-recordings \
npx nx e2e cockpit-runtimes-mastra-angular --grep='Approve sends'
AIMOCK_MODE=record AIMOCK_RECORD_DIR=/tmp/mastra-cancel-recordings \
npx nx e2e cockpit-runtimes-mastra-angular --grep='Cancel sends'
```

Use a fresh process for each decision. Aimock caches newly recorded responses
in memory, and its generated matchers do not distinguish these two tool-result
payloads. Running both decisions in one recording process can reuse the first
decision's reply instead of verifying the second against the live model.

Record mode forwards unmatched model requests to the provider and captures responses.
The campsite tool remains local: it returns a synthetic confirmation or decline
and does not make an external reservation. Each run uses a fresh local database.

Keep approval and cancellation continuation fixtures separate using
`toolResultContains`. Matching only the user prompt and `hasToolResult` can
replay an approval after a cancellation because both decisions share the same
prompt. The initial reservation fixture excludes tool results, so an unknown
continuation cannot restart the tool call silently.

The cancellation reply was captured from `gpt-4o-mini` through aimock record
mode on 2026-09-09, then given a stable tool-result matcher for replay.

Review captures before incorporating them into fixtures. Retain stable outcome
matchers rather than generated tool-call IDs, and do not commit credentials or
raw browser traces.
10 changes: 8 additions & 2 deletions cockpit/runtimes/mastra/angular/e2e/fixtures/mastra.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,17 @@
}
},
{
"match": { "userMessage": "reserve", "hasToolResult": true },
"match": { "userMessage": "reserve", "hasToolResult": true, "toolResultContains": "Reserved North Pines" },
"response": {
"content": "North Pines is reserved for 2 nights — confirmation TP-0288."
}
},
{
"match": { "userMessage": "reserve", "hasToolResult": true, "toolResultContains": "Nothing was booked." },
"response": {
"content": "The reservation for the North Pines campsite was not booked."
}
},
{
"match": { "userMessage": "Bear Lake", "hasToolResult": true },
"response": {
Expand Down Expand Up @@ -63,7 +69,7 @@
}
},
{
"match": { "userMessage": "reserve" },
"match": { "userMessage": "reserve", "hasToolResult": false },
"response": {
"toolCalls": [
{
Expand Down
7 changes: 4 additions & 3 deletions cockpit/runtimes/mastra/angular/e2e/global-setup-impl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { portsFor } from '../../../../../cockpit/ports.mjs';
import { startAimock, type AimockHandle } from '@threadplane-internal/e2e-harness';
import { resolveAimockLaunch, startAimock, type AimockHandle } from '@threadplane-internal/e2e-harness';

const ports = portsFor('cockpit-runtimes-mastra-angular');
const angularProject = 'cockpit-runtimes-mastra-angular';
Expand Down Expand Up @@ -57,7 +57,8 @@ export default async function globalSetup(): Promise<void> {
const fixturesDir = resolve(__dirname, 'fixtures');
const serviceDir = resolve(root, backendCwd);

const aimock = await startAimock({ mode: 'replay', fixturePath: fixturesDir });
const launch = resolveAimockLaunch(fixturesDir);
const aimock = await startAimock(launch.startOptions);
console.log(`[mastra-harness] aimock listening at ${aimock.baseUrl}`);

// The service is self-contained (own package.json + lockfile, deps NOT in
Expand All @@ -77,7 +78,7 @@ export default async function globalSetup(): Promise<void> {
...process.env,
PORT: String(backendPort),
AG_UI_INTERNAL_TOKEN: INTERNAL_TOKEN,
OPENAI_API_KEY: 'test-not-used',
OPENAI_API_KEY: launch.openaiApiKey,
OPENAI_BASE_URL: aimock.baseUrl,
AG_UI_MASTRA_DB_PATH: join(dbDir, 'mastra.db'),
},
Expand Down
47 changes: 39 additions & 8 deletions cockpit/runtimes/mastra/angular/e2e/mastra.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,45 @@ test.describe('cockpit runtimes/mastra: camping trip planner', () => {
await expect(dialog).toContainText('$90.00');
});

test('Approve resumes the suspended run and the reservation completes', async ({ page }) => {
await page.goto('/');
await page.getByText('Reserve the campsite').click();
const dialog = page.locator('dialog.chat-approval-card');
await expect(dialog).toBeVisible({ timeout: 30_000 });
await dialog.getByRole('button', { name: 'Approve' }).click();
await expect(page.getByText(/reserved for 2 nights/i)).toBeVisible({ timeout: 30_000 });
});
for (const approved of [true, false]) {
const action = approved ? 'Approve' : 'Cancel';
test(`${action} sends the decision and displays the matching reservation outcome`, async ({ page }) => {
if (process.env['AIMOCK_MODE'] === 'record') test.setTimeout(120_000);
await page.goto('/');
await page.getByText('Reserve the campsite').click();
const dialog = page.locator('dialog.chat-approval-card');
await expect(dialog).toBeVisible({ timeout: 30_000 });
await expect(dialog).toContainText('North Pines');
await expect(dialog).toContainText('$90.00');

const resumed = page.waitForResponse(response => {
if (response.request().method() !== 'POST' || !response.url().endsWith('/agent')) return false;
return response.request().postDataJSON()?.forwardedProps?.command?.resume?.approved === approved;
});
await dialog.getByRole('button', { name: action, exact: true }).click();
const response = await resumed;
expect(response.status()).toBe(200);
const command = response.request().postDataJSON().forwardedProps.command;
expect(command.interruptEvent.toolCallId).toEqual(expect.any(String));
expect(command.interruptEvent.runId).toEqual(expect.any(String));
const events = (await response.text()).split('\n')
.filter(line => line.startsWith('data: '))
.map(line => JSON.parse(line.slice(6)));
const result = events.find(event => event.type === 'TOOL_CALL_RESULT');
expect(result?.toolCallId).toBe(command.interruptEvent.toolCallId);
expect(result?.content).toContain(approved ? 'Reserved North Pines' : 'Nothing was booked.');
await expect(dialog).not.toBeVisible();
const reply = page.locator('chat-message[data-role="assistant"]').last();
await expect(reply).toContainText(approved
? /reserved|confirmed|booked/i
: /declined|cancelled|canceled|not.{0,20}(booked|completed|confirmed|reserved)|nothing.{0,20}booked/i);
if (approved) {
await expect(reply).not.toContainText(/declined|cancelled|canceled|nothing.{0,20}booked|not.{0,20}(booked|completed|confirmed|reserved)/i);
} else {
await expect(reply).not.toContainText('TP-0288');
}
});
}

// Delegation: the supervisor calls the registered weather_forecaster
// sub-agent (wire tool `agent-weather_forecaster`); the server-side
Expand Down
Loading