From ede093d07a19ac43f5a4268e6bf66c95fcd61dfa Mon Sep 17 00:00:00 2001 From: Raffael Prem Date: Mon, 30 Mar 2026 09:47:37 +0200 Subject: [PATCH] Add multi-agent capabilties --- README.md | 57 +++++++- docs/examples/multi-turn.md | 190 +++++++++++++++++++++++++- docs/guide/pass-fail-logic.md | 2 +- docs/guide/scenarios.md | 96 +++++++++++-- docs/guide/trajectory-assertions.md | 6 +- docs/reference/config-api.md | 13 +- docs/reference/scenario-api.md | 43 +++++- docs/reference/types.md | 16 ++- package.json | 2 +- src/config/schema.ts | 1 + src/evaluator/evaluator.ts | 10 +- src/index.ts | 4 + src/runner/reporters/console.ts | 23 ++++ src/runner/reporters/githubActions.ts | 7 + src/runner/reporters/json.ts | 5 + src/runner/runner.ts | 52 ++++++- src/scenario/scenario.ts | 2 + src/scenario/types.ts | 44 +++++- src/simulator/agentClient.ts | 8 +- src/simulator/simulatedUser.ts | 8 +- src/simulator/simulator.ts | 70 +++++++--- src/vitest.ts | 16 +++ 22 files changed, 619 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index b5ba263..aa74408 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ npm install @agentesting/agentest --save-dev | | | |---|---| | Scenario-based tests | Define user personas, goals, and knowledge — Agentest generates realistic multi-turn conversations | +| Scripted multi-turn | Predetermined user messages with per-turn trajectory assertions for deterministic regression tests | | Tool-call mocks | Intercept and control tool calls with functions, sequences, and error simulation | | Trajectory assertions | Verify tool call order and arguments with `strict`, `contains`, `unordered`, and `within` match modes | | LLM-as-judge metrics | Helpfulness, coherence, relevance, faithfulness, goal completion, behavior failure detection | @@ -205,6 +206,44 @@ scenario('user books a morning slot', { }) ``` +#### Scripted alternative + +For deterministic tests with predetermined user messages and per-turn assertions: + +```ts +// tests/context.sim.ts +import { scenario } from '@agentesting/agentest' + +scenario('follow-up reuses vehicle context', { + turns: [ + { + userMessage: 'How fast was Leo (12345678) last week?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'get_speed', args: { id: '12345678' }, argMatchMode: 'partial' }], + }, + }, + }, + { + userMessage: 'And what about its failure count?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'get_failures', args: { id: '12345678' }, argMatchMode: 'partial' }], + }, + }, + }, + ], + mocks: { + tools: { + get_speed: () => ({ speed: 0.8, unit: 'm/s' }), + get_failures: () => ({ count: 5 }), + }, + }, +}) +``` + ### 3. Run ```bash @@ -337,11 +376,27 @@ export default defineConfig({ }) ``` -The handler receives the full message history (same `ChatMessage` format used internally) and must return an assistant message. If the response includes `tool_calls`, Agentest runs them through mocks and calls your handler again with the tool results — the same loop as with HTTP endpoints. +The handler receives the full message history and a `ctx` object. If the response includes `tool_calls`, Agentest runs them through mocks and calls your handler again with the tool results — the same loop as with HTTP endpoints. + +The `ctx` object provides `resolveTool()` for agents that handle tools internally (e.g., multi-agent supervisors): + +```ts +handler: async (messages, ctx) => { + const mockClient = { + async get(endpoint, params) { + return ctx.resolveTool(endpoint, params) // uses scenario mocks, records for trajectory + }, + } + const agent = createSupervisor({ client: mockClient }) + const result = await agent.invoke({ messages }) + return { role: 'assistant' as const, content: result.content } +} +``` This is useful when your agent: - Uses a non-OpenAI API (Anthropic, Google, custom protocols) - Runs in-process (no HTTP server needed) +- Has multi-agent routing that handles tools internally - Needs custom request/response mapping - Uses an SDK or framework with its own calling convention diff --git a/docs/examples/multi-turn.md b/docs/examples/multi-turn.md index 5e5de86..9a496a6 100644 --- a/docs/examples/multi-turn.md +++ b/docs/examples/multi-turn.md @@ -2,7 +2,11 @@ Test complex scenarios that require multiple interactions. -## Example: Multi-step Booking Flow +## Simulated Multi-turn (LLM-driven) + +Use `profile` and `goal` to let the simulated user drive the conversation autonomously. Agentest's LLM-powered user generates realistic messages and decides when the goal is met. + +### Example: Multi-step Booking Flow ```ts import { scenario, sequence } from '@agentesting/agentest' @@ -61,7 +65,7 @@ scenario('user completes multi-step booking with questions', { }) ``` -## Expected Conversation Flow +### Expected Conversation Flow ``` Turn 1: @@ -89,7 +93,7 @@ Agent: *calls create_booking* Agent: Perfect! Booked for Tuesday at 09:00. Confirmation: BK-001 ``` -## Testing Conversation Depth +### Testing Conversation Depth Use `maxTurns` to accommodate longer conversations: @@ -101,7 +105,7 @@ scenario('complex troubleshooting scenario', { }) ``` -## Multiple Conversations +### Multiple Conversations Run more conversations to test variance: @@ -112,3 +116,181 @@ scenario('unpredictable user behavior', { conversationsPerScenario: 10, // Run 10 times }) ``` + +## Scripted Multi-turn (Deterministic) + +Use `turns` to define exact user messages for each turn. This skips the simulated user entirely — no LLM is used to generate messages. Each turn can have its own trajectory assertions. + +This is ideal for testing **context carry-forward**, **conversation continuity**, and **deterministic regression tests**. + +### Example: Context Carry-forward + +```ts +import { scenario } from '@agentesting/agentest' + +scenario('follow-up reuses vehicle context', { + turns: [ + { + userMessage: 'How fast was Leo (12345678) last week?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'performance_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + ], + }, + }, + }, + { + userMessage: 'And what about its failure count in the same period?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'failure_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + ], + }, + }, + }, + ], + + mocks: { + tools: { + performance_agent: () => ({ speed: 0.8, unit: 'm/s' }), + failure_agent: () => ({ count: 5, severity_breakdown: { warning: 3, error: 2 } }), + }, + }, +}) +``` + +The second turn says "its failure count" — the agent must carry forward that "it" refers to vehicle `12345678` from the previous turn. The per-turn assertion verifies this. + +### Example: Domain Switch with Follow-up Export + +```ts +scenario('cross-domain pivot then export', { + turns: [ + { + userMessage: 'What was the energy consumption of Leo (12345678) from Jan 1 to Jan 7?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'performance_agent', argMatchMode: 'ignore' }], + }, + }, + }, + { + userMessage: 'Were there any errors during that period?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'failure_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + ], + }, + }, + }, + { + userMessage: 'Export that to CSV', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'export_to_csv', argMatchMode: 'ignore' }], + }, + }, + }, + ], + + mocks: { + tools: { + performance_agent: () => ({ energy: 12.4, unit: 'kWh' }), + failure_agent: () => ({ count: 3 }), + export_to_csv: () => ({ fileId: 'export-001', url: '/download/export-001' }), + }, + }, +}) +``` + +### Key Differences from Simulated Multi-turn + +| | Simulated (`profile` + `goal`) | Scripted (`turns`) | +|---|---|---| +| User messages | Generated by LLM | Predetermined | +| Deterministic | No (LLM variance) | Yes | +| `conversationsPerScenario` default | From config (usually 3) | 1 | +| LLM evaluation | Full metrics + goal completion | Only if `goal` is provided | +| Per-turn assertions | No (cumulative only) | Yes | +| Best for | Exploratory testing, persona variance | Regression tests, context carry-forward | + +### Scripted Scenarios with Evaluation + +To enable LLM-as-judge evaluation on scripted scenarios, provide a `goal`: + +```ts +scenario('follow-up with quality evaluation', { + goal: 'Get speed and failure data for vehicle Leo.', + + turns: [ + { userMessage: 'How fast was Leo (12345678) last week?' }, + { userMessage: 'And what about its failure count?' }, + ], + + mocks: { + tools: { + performance_agent: () => ({ speed: 0.8 }), + failure_agent: () => ({ count: 5 }), + }, + }, +}) +``` + +Without a `goal`, scripted scenarios skip LLM evaluation entirely and rely on trajectory assertions for pass/fail. + +### Cumulative + Per-turn Assertions + +You can combine scenario-level cumulative assertions with per-turn assertions: + +```ts +scenario('full booking flow', { + // Cumulative: these tools must all be called across the whole conversation + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'check_availability', argMatchMode: 'ignore' }, + { name: 'create_booking', argMatchMode: 'ignore' }, + ], + }, + }, + + turns: [ + { + userMessage: 'Is Tuesday morning available?', + // Per-turn: only check_availability should be called in this turn + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'check_availability', argMatchMode: 'ignore' }], + }, + }, + }, + { + userMessage: 'Book the 9am slot.', + // Per-turn: create_booking should be called in this turn + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'create_booking', argMatchMode: 'ignore' }], + }, + }, + }, + ], + + mocks: { + tools: { + check_availability: () => ({ available: true, slots: ['09:00', '10:30'] }), + create_booking: () => ({ success: true, bookingId: 'BK-001' }), + }, + }, +}) +``` diff --git a/docs/guide/pass-fail-logic.md b/docs/guide/pass-fail-logic.md index 21f4ed1..6dc5ea9 100644 --- a/docs/guide/pass-fail-logic.md +++ b/docs/guide/pass-fail-logic.md @@ -11,7 +11,7 @@ A scenario either passes or fails based on multiple criteria. The overall run pa A scenario **passes** when all of the following conditions are true: 1. ✅ No conversation threw an error -2. ✅ All trajectory assertions matched (if configured) +2. ✅ All trajectory assertions matched (if configured) — both scenario-level cumulative and per-turn 3. ✅ No errors at or above the configured `failOnErrorSeverity` 4. ✅ All metric averages meet their configured thresholds diff --git a/docs/guide/scenarios.md b/docs/guide/scenarios.md index d5a7e79..c786b30 100644 --- a/docs/guide/scenarios.md +++ b/docs/guide/scenarios.md @@ -2,21 +2,35 @@ Learn how to define test scenarios for your agent. -## Basic Structure +## Two Modes + +Agentest supports two modes for defining scenarios: + +**Simulated mode** — An LLM-powered user drives the conversation based on a `profile` and `goal`. Best for exploratory testing and persona-based variance. ```ts -import { scenario } from '@agentesting/agentest' +scenario('simulated booking', { + profile: 'Busy professional who prefers mornings.', + goal: 'Book a haircut for Tuesday morning.', +}) +``` -scenario('descriptive name', { - profile: 'User personality and context', - goal: 'What the user wants to accomplish', - // ... options +**Scripted mode** — You define exact user messages with `turns`. No LLM is used for the user side. Best for deterministic regression tests and context carry-forward testing. + +```ts +scenario('scripted follow-up', { + turns: [ + { userMessage: 'How fast was Leo (12345678) last week?' }, + { userMessage: 'And what about its failure count?' }, + ], }) ``` -## Profile & Goal +See [Multi-turn Conversations](/examples/multi-turn) for detailed scripted examples. + +## Simulated Mode: Profile & Goal -The `profile` and `goal` are the foundation of every scenario. They define who the simulated user is and what they're trying to accomplish. +The `profile` and `goal` are the foundation of simulated scenarios. They define who the simulated user is and what they're trying to accomplish. These fields are **required** for simulated scenarios and **optional** for scripted scenarios. ### Profile @@ -339,6 +353,72 @@ export default defineConfig({ }) ``` +## Testing Multi-Agent Routing + +Agents with supervisor/sub-agent architectures handle tool routing internally — the supervisor decides which domain agent to call, and tool calls never leave the agent process. To test these with agentest's mock system, use a `custom` handler with `ctx.resolveTool()`. + +The custom handler creates the agent in-process with mock dependencies that delegate to agentest's mock resolver: + +```ts +// agentest.config.ts +import { defineConfig } from '@agentesting/agentest' +import { createSupervisor } from './src/agents/supervisor.js' + +export default defineConfig({ + agent: { + type: 'custom', + name: 'my-supervisor', + handler: async (messages, ctx) => { + // Create a mock API client that uses agentest's mock resolver + const mockClient = { + async get(endpoint, params) { + return ctx.resolveTool(endpoint, params) + }, + } + + // Create the agent with mocked dependencies + const agent = createSupervisor({ client: mockClient }) + const result = await agent.invoke({ messages }) + return { role: 'assistant', content: result.content } + }, + }, +}) +``` + +```ts +// scenarios/routing.sim.ts +import { scenario } from '@agentesting/agentest' + +scenario('supervisor routes to performance agent', { + turns: [ + { + userMessage: 'How fast was vehicle 12345678 last week?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'get_report_data', args: { serials: '12345678' }, argMatchMode: 'partial' }, + ], + }, + }, + }, + ], + + mocks: { + tools: { + get_report_data: (args) => ({ speed_avg: 0.8, unit: 'm/s' }), + }, + }, +}) +``` + +When the agent internally calls `get_report_data`, the mock client calls `ctx.resolveTool('get_report_data', args)`, which: +1. Resolves through agentest's per-scenario mock definitions +2. Records the tool call for trajectory assertions +3. Returns the mock result to the agent + +This gives you full control over tool responses while testing the actual routing logic of your supervisor. + ## Complete Example See [Basic Scenario Example](/examples/basic-scenario) for a full walkthrough. diff --git a/docs/guide/trajectory-assertions.md b/docs/guide/trajectory-assertions.md index 853919a..1f902a3 100644 --- a/docs/guide/trajectory-assertions.md +++ b/docs/guide/trajectory-assertions.md @@ -37,7 +37,11 @@ scenario('user books a morning slot', { }) ``` -Assertions are checked **per-conversation**. If any conversation in the scenario fails the assertion, the entire scenario fails. +Scenario-level assertions are checked **per-conversation** across all turns combined. If any conversation fails the assertion, the entire scenario fails. + +::: tip Per-turn Assertions +When using [scripted multi-turn scenarios](/examples/multi-turn#scripted-multi-turn-deterministic), each turn can define its own trajectory assertions that are checked against only that turn's tool calls. This is useful for verifying context carry-forward — e.g., that a follow-up question routes to the right domain agent with the correct vehicle serial from the previous turn. +::: ## Match Modes diff --git a/docs/reference/config-api.md b/docs/reference/config-api.md index 60c6029..b060cde 100644 --- a/docs/reference/config-api.md +++ b/docs/reference/config-api.md @@ -43,10 +43,21 @@ agent: { agent: { type: 'custom', name: string, - handler: (messages: ChatMessage[]) => Promise, + handler: (messages: ChatMessage[], ctx: CustomHandlerContext) => Promise, } ``` +The handler receives a `ctx` object with: + +| Field | Type | Description | +|-------|------|-------------| +| `resolveTool` | `(name: string, args: Record) => Promise` | Resolve a tool call through the scenario's mock system. Records the call for trajectory assertions. | +| `turnIndex` | `number` | Current turn index | +| `conversationId` | `string` | Current conversation ID | +| `scenarioName` | `string` | Name of the running scenario | + +Use `ctx.resolveTool()` to wire agentest's per-scenario mocks into agents that handle tools internally (e.g., multi-agent supervisors). See [Testing Multi-Agent Routing](/guide/scenarios#testing-multi-agent-routing). + ## LLM Provider ### `provider` diff --git a/docs/reference/scenario-api.md b/docs/reference/scenario-api.md index 8cf8817..3488e7f 100644 --- a/docs/reference/scenario-api.md +++ b/docs/reference/scenario-api.md @@ -14,11 +14,13 @@ Define a test scenario for your agent. ## Required Options +Either `profile` + `goal` (simulated mode) or `turns` (scripted mode) must be provided. + ### `profile` **Type:** `string` -**Required** +**Required** for simulated mode. Optional for scripted mode. Simulated user's personality and context. @@ -26,9 +28,44 @@ Simulated user's personality and context. **Type:** `string` -**Required** +**Required** for simulated mode. Optional for scripted mode. + +What the user wants to accomplish. When provided with scripted `turns`, enables LLM-as-judge evaluation. + +### `turns` + +**Type:** `ScriptedTurn[]` + +**Required** for scripted mode. + +Scripted conversation turns with predetermined user messages. When provided, the simulated user is skipped entirely and messages are replayed in order. + +Each turn can include per-turn trajectory assertions. + +```ts +turns: [ + { + userMessage: 'How fast was vehicle 12345678?', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'performance_agent', argMatchMode: 'ignore' }], + }, + }, + }, + { + userMessage: 'Export that to CSV', + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'export_to_csv', argMatchMode: 'ignore' }], + }, + }, + }, +] +``` -What the user wants to accomplish. +Defaults: `conversationsPerScenario` defaults to `1` (deterministic). `maxTurns` is set to `turns.length`. ## Optional Options diff --git a/docs/reference/types.md b/docs/reference/types.md index 4478211..6817e12 100644 --- a/docs/reference/types.md +++ b/docs/reference/types.md @@ -39,8 +39,13 @@ interface Config { ```ts interface ScenarioOptions { - profile: string - goal: string + // Required for simulated mode, optional for scripted mode + profile?: string + goal?: string + + // Scripted mode: predetermined user messages with per-turn assertions + turns?: ScriptedTurn[] + knowledge?: KnowledgeItem[] mocks?: { tools: Record } assertions?: { toolCalls?: TrajectoryAssertion } @@ -49,6 +54,13 @@ interface ScenarioOptions { userPromptTemplate?: string } +interface ScriptedTurn { + userMessage: string + assertions?: { + toolCalls?: TrajectoryAssertion + } +} + interface KnowledgeItem { content: string } diff --git a/package.json b/package.json index 2433160..79c2245 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentesting/agentest", - "version": "0.0.8", + "version": "0.0.9", "description": "Embedded agent simulation & evaluation framework for Node.js/TypeScript", "type": "module", "main": "./dist/index.js", diff --git a/src/config/schema.ts b/src/config/schema.ts index 58c3413..f0f4cca 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -61,6 +61,7 @@ export type CustomAgentHandler = ( function: { name: string; arguments: string } }> }>, + ctx: import('../scenario/types.js').CustomHandlerContext, ) => Promise<{ role: 'assistant' content: string | null diff --git a/src/evaluator/evaluator.ts b/src/evaluator/evaluator.ts index 8544966..e40b545 100644 --- a/src/evaluator/evaluator.ts +++ b/src/evaluator/evaluator.ts @@ -104,12 +104,12 @@ export class Evaluator { turns.map((turn, i) => this.evaluateTurn(turn, turns.slice(0, i), scenario)), ) - // Goal completion runs once per conversation + // Goal completion runs once per conversation (skip when no goal is defined) let goalCompletion: QuantResult | null = null - if (this.goalCompletionMetric) { + if (this.goalCompletionMetric && scenario.goal) { const input: ConversationScoreInput = { goal: scenario.goal, - profile: scenario.profile, + profile: scenario.profile ?? 'A user testing the agent', knowledge: scenario.knowledge ?? [], turns: turns.map((t) => ({ userMessage: t.userMessage, @@ -129,8 +129,8 @@ export class Evaluator { scenario: ScenarioOptions, ): Promise { const input: ScoreInput = { - goal: scenario.goal, - profile: scenario.profile, + goal: scenario.goal ?? 'Respond to the user query accurately', + profile: scenario.profile ?? 'A user testing the agent', knowledge: scenario.knowledge ?? [], userMessage: turn.userMessage, agentMessage: turn.agentMessage, diff --git a/src/index.ts b/src/index.ts index f79d80e..873f51d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,9 +7,12 @@ export type { AgentestConfig, AgentestConfigInput, CustomAgentHandler } from './ export { scenario, getRegisteredScenarios, clearScenarioRegistry } from './scenario/scenario.js' export { sequence } from './scenario/mocks.js' export { AgentestError, MockResolver } from './scenario/mocks.js' +export { validateScenarioOptions } from './scenario/types.js' export type { Scenario, ScenarioOptions, + ScriptedTurn, + CustomHandlerContext, ToolMockFn, ToolCallRecord, ToolCallAssertion, @@ -86,6 +89,7 @@ export type { ThresholdViolation } from './evaluator/scoring.js' // Runner export { Runner } from './runner/runner.js' export type { + PerTurnTrajectoryResult, ScenarioResult, RunResult, ComparisonAgentResult, diff --git a/src/runner/reporters/console.ts b/src/runner/reporters/console.ts index 7a3ca46..e5b2a84 100644 --- a/src/runner/reporters/console.ts +++ b/src/runner/reporters/console.ts @@ -171,6 +171,29 @@ export class ConsoleReporter implements Reporter { } } + // Per-turn trajectory + const perTurnResults = result.perTurnTrajectoryResults.get(conv.conversationId) + if (perTurnResults) { + const failed = perTurnResults.filter((r) => !r.result.matched) + if (failed.length === 0) { + parts.push(`${GREEN}per-turn trajectories matched${RESET}`) + } else { + for (const f of failed) { + const details: string[] = [`turn ${f.turnIndex}`] + if (f.result.missingCalls.length > 0) { + details.push(`missing: ${f.result.missingCalls.join(', ')}`) + } + if (f.result.extraCalls.length > 0) { + details.push(`extra: ${f.result.extraCalls.join(', ')}`) + } + if (f.result.forbiddenCalls.length > 0) { + details.push(`forbidden: ${f.result.forbiddenCalls.join(', ')}`) + } + parts.push(`${RED}${details.join(' — ')}${RESET}`) + } + } + } + // Average scores if (evaluation) { const avgScores = computeTurnAverages(evaluation.turnEvaluations) diff --git a/src/runner/reporters/githubActions.ts b/src/runner/reporters/githubActions.ts index f580f4a..278293b 100644 --- a/src/runner/reporters/githubActions.ts +++ b/src/runner/reporters/githubActions.ts @@ -277,6 +277,13 @@ export class GitHubActionsReporter implements Reporter { reasons.push(`${failedTrajectories.length} trajectory assertion(s) failed`) } + const failedPerTurn = [...result.perTurnTrajectoryResults.values()].flatMap((rs) => + rs.filter((r) => !r.result.matched), + ) + if (failedPerTurn.length > 0) { + reasons.push(`${failedPerTurn.length} per-turn trajectory assertion(s) failed`) + } + if (result.errors.length > 0) { const critical = result.errors.filter( (e) => e.severity === 'critical' || e.severity === 'high', diff --git a/src/runner/reporters/json.ts b/src/runner/reporters/json.ts index 93bde3f..7ec6fab 100644 --- a/src/runner/reporters/json.ts +++ b/src/runner/reporters/json.ts @@ -117,6 +117,11 @@ export class JsonReporter implements Reporter { } : null, trajectory: trajectory ?? null, + perTurnTrajectory: + sr.perTurnTrajectoryResults.get(conv.conversationId)?.map((r) => ({ + turnIndex: r.turnIndex, + ...r.result, + })) ?? null, } }), errors: sr.errors, diff --git a/src/runner/runner.ts b/src/runner/runner.ts index 4180da8..60cdc78 100644 --- a/src/runner/runner.ts +++ b/src/runner/runner.ts @@ -16,11 +16,17 @@ import { computeThresholdViolations } from '../evaluator/scoring.js' import type { DiscoveryResult } from './discovery.js' import type { Reporter, ProgressEvent } from './reporters/types.js' +export interface PerTurnTrajectoryResult { + turnIndex: number + result: TrajectoryResult +} + export interface ScenarioResult { scenario: Scenario simulation: SimulationResult evaluations: Map trajectoryResults: Map + perTurnTrajectoryResults: Map errors: UniqueError[] passed: boolean } @@ -280,17 +286,22 @@ export class Runner { const label = agentLabel ? `[${agentLabel}] ` : '' // 1. Simulate conversations + const isScripted = Array.isArray(scenario.options.turns) && scenario.options.turns.length > 0 + const conversationCount = + scenario.options.conversationsPerScenario ?? + (isScripted ? 1 : this.config.conversationsPerScenario) this.emitProgress({ scenario: scenario.name, phase: 'simulating', - detail: `${label}${scenario.options.conversationsPerScenario ?? this.config.conversationsPerScenario} conversation(s)`, + detail: `${label}${conversationCount} conversation(s)`, }) const simulation = await simulator.runScenario(scenario) - // 2. Evaluate each conversation + // 2. Evaluate each conversation (skip for scripted scenarios without a goal) const evaluations = new Map() for (const conv of simulation.conversations) { if (conv.error) continue // skip errored conversations + if (isScripted && !scenario.options.goal) continue // no meaningful goal to evaluate against this.emitProgress({ scenario: scenario.name, @@ -318,6 +329,28 @@ export class Runner { } } + // 3b. Per-turn trajectory assertions (scripted scenarios) + const perTurnTrajectoryResults = new Map() + if (scenario.options.turns) { + for (const conv of simulation.conversations) { + if (conv.error) continue + const turnResults: PerTurnTrajectoryResult[] = [] + for (const turn of conv.turns) { + const scriptedTurn = scenario.options.turns[turn.turnIndex] + if (scriptedTurn?.assertions?.toolCalls) { + const result = trajectoryMatcher.match( + turn.toolCalls, + scriptedTurn.assertions.toolCalls, + ) + turnResults.push({ turnIndex: turn.turnIndex, result }) + } + } + if (turnResults.length > 0) { + perTurnTrajectoryResults.set(conv.conversationId, turnResults) + } + } + } + // 4. Collect and deduplicate errors const allFailures: FailureTurn[] = [] for (const [convId, evaluation] of evaluations) { @@ -340,13 +373,20 @@ export class Runner { } // 5. Determine pass/fail - const passed = this.determinePassFail(simulation, evaluations, trajectoryResults, errors) + const passed = this.determinePassFail( + simulation, + evaluations, + trajectoryResults, + perTurnTrajectoryResults, + errors, + ) const scenarioResult: ScenarioResult = { scenario, simulation, evaluations, trajectoryResults, + perTurnTrajectoryResults, errors, passed, } @@ -379,6 +419,7 @@ export class Runner { simulation: SimulationResult, evaluations: Map, trajectoryResults: Map, + perTurnTrajectoryResults: Map, errors: UniqueError[], ): boolean { // Fail if any conversation errored @@ -391,6 +432,11 @@ export class Runner { if (!result.matched) return false } + // Fail if any per-turn trajectory assertion failed + for (const turnResults of perTurnTrajectoryResults.values()) { + if (turnResults.some((r) => !r.result.matched)) return false + } + // Fail if any errors meet or exceed the configured severity const severityOrder = ['low', 'medium', 'high', 'critical'] as const const failAtIndex = severityOrder.indexOf(this.config.failOnErrorSeverity) diff --git a/src/scenario/scenario.ts b/src/scenario/scenario.ts index ec50c58..2033666 100644 --- a/src/scenario/scenario.ts +++ b/src/scenario/scenario.ts @@ -1,8 +1,10 @@ import type { Scenario, ScenarioOptions } from './types.js' +import { validateScenarioOptions } from './types.js' const scenarioRegistry: Scenario[] = [] export function scenario(name: string, options: ScenarioOptions): Scenario { + validateScenarioOptions(options) const s: Scenario = { name, options } scenarioRegistry.push(s) return s diff --git a/src/scenario/types.ts b/src/scenario/types.ts index c62f39b..c16f7a5 100644 --- a/src/scenario/types.ts +++ b/src/scenario/types.ts @@ -29,15 +29,25 @@ export interface KnowledgeItem { content: string } +export interface ScriptedTurn { + userMessage: string + assertions?: { + toolCalls?: TrajectoryAssertions + } +} + export interface ScenarioOptions { - profile: string - goal: string + profile?: string + goal?: string knowledge?: KnowledgeItem[] userPromptTemplate?: string conversationsPerScenario?: number maxTurns?: number + /** Scripted multi-turn conversation. When provided, profile/goal are optional and SimulatedUser is skipped. */ + turns?: ScriptedTurn[] + mocks?: { tools?: Record> } @@ -47,6 +57,28 @@ export interface ScenarioOptions { } } +export function validateScenarioOptions(options: ScenarioOptions): void { + const isScripted = Array.isArray(options.turns) + + if (isScripted) { + if (options.turns!.length === 0) { + throw new Error('Scenario with "turns" must have at least one turn') + } + for (let i = 0; i < options.turns!.length; i++) { + if (!options.turns![i].userMessage?.trim()) { + throw new Error(`Turn ${i} must have a non-empty "userMessage" string`) + } + } + } else { + if (!options.profile || typeof options.profile !== 'string') { + throw new Error('Scenario without "turns" requires a non-empty "profile" string') + } + if (!options.goal || typeof options.goal !== 'string') { + throw new Error('Scenario without "turns" requires a non-empty "goal" string') + } + } +} + export interface Scenario { name: string options: ScenarioOptions @@ -58,3 +90,11 @@ export interface ToolCallRecord { result: unknown turnIndex: number } + +export interface CustomHandlerContext { + /** Resolve a tool call through the scenario's mock system. Records the call for trajectory assertions. */ + resolveTool: (name: string, args: Record) => Promise + turnIndex: number + conversationId: string + scenarioName: string +} diff --git a/src/simulator/agentClient.ts b/src/simulator/agentClient.ts index 17be24a..e721f8f 100644 --- a/src/simulator/agentClient.ts +++ b/src/simulator/agentClient.ts @@ -1,4 +1,5 @@ import type { AgentestConfig, CustomAgentHandler } from '../config/schema.js' +import type { CustomHandlerContext } from '../scenario/types.js' export interface ChatMessage { role: 'system' | 'user' | 'assistant' | 'tool' @@ -86,6 +87,7 @@ export class AgentClient { private requestTimeoutMs: number private customHandler?: CustomAgentHandler private streaming: boolean + private handlerContext?: CustomHandlerContext constructor(config: AgentestConfig) { const agent = config.agent @@ -111,6 +113,10 @@ export class AgentClient { this.requestTimeoutMs = config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS } + setHandlerContext(ctx: CustomHandlerContext): void { + this.handlerContext = ctx + } + private validateEndpoint(endpoint: string): void { const url = new URL(endpoint) if (!ALLOWED_PROTOCOLS.includes(url.protocol)) { @@ -136,7 +142,7 @@ export class AgentClient { } private async sendCustom(messages: ChatMessage[]): Promise { - const result = await this.customHandler!(messages) + const result = await this.customHandler!(messages, this.handlerContext!) const message = result as ChatMessage if (!message || typeof message.role !== 'string') { diff --git a/src/simulator/simulatedUser.ts b/src/simulator/simulatedUser.ts index 0c8c49e..3f4361c 100644 --- a/src/simulator/simulatedUser.ts +++ b/src/simulator/simulatedUser.ts @@ -68,8 +68,8 @@ function renderTemplate(template: string, scenario: ScenarioOptions): string { : '' return template - .replaceAll('{{profile}}', scenario.profile) - .replaceAll('{{goal}}', scenario.goal) + .replaceAll('{{profile}}', scenario.profile ?? '') + .replaceAll('{{goal}}', scenario.goal ?? '') .replaceAll('{{knowledge}}', knowledgeStr) } @@ -78,10 +78,10 @@ function buildDefaultSystemPrompt(scenario: ScenarioOptions): string { 'You are a simulated user interacting with an AI agent. Your job is to play the role described below and work toward the stated goal.', '', `## Your Profile`, - scenario.profile, + scenario.profile ?? '', '', `## Your Goal`, - scenario.goal, + scenario.goal ?? '', '', '## Instructions', '- Stay in character according to your profile.', diff --git a/src/simulator/simulator.ts b/src/simulator/simulator.ts index b62e66f..e359868 100644 --- a/src/simulator/simulator.ts +++ b/src/simulator/simulator.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto' import type { AgentestConfig } from '../config/schema.js' -import type { Scenario, ToolCallRecord } from '../scenario/types.js' +import type { Scenario, ToolCallRecord, CustomHandlerContext } from '../scenario/types.js' import type { ToolMockFn } from '../scenario/types.js' import { MockResolver, AgentestError } from '../scenario/mocks.js' import { AgentClient, type ChatMessage, type ToolCall } from './agentClient.js' @@ -68,8 +68,10 @@ export class Simulator { } async runScenario(scenario: Scenario): Promise { + const isScripted = Array.isArray(scenario.options.turns) && scenario.options.turns.length > 0 const conversationCount = - scenario.options.conversationsPerScenario ?? this.config.conversationsPerScenario + scenario.options.conversationsPerScenario ?? + (isScripted ? 1 : this.config.conversationsPerScenario) const conversations: ConversationRecord[] = [] for (let i = 0; i < conversationCount; i++) { @@ -85,8 +87,12 @@ export class Simulator { scenario: Scenario, conversationId: string, ): Promise { - const maxTurns = scenario.options.maxTurns ?? this.config.maxTurns - const simulatedUser = new SimulatedUser(this.llmProvider, scenario.options) + const isScripted = Array.isArray(scenario.options.turns) && scenario.options.turns.length > 0 + const scriptedTurns = scenario.options.turns + const maxTurns = isScripted + ? scriptedTurns!.length + : (scenario.options.maxTurns ?? this.config.maxTurns) + const simulatedUser = isScripted ? null : new SimulatedUser(this.llmProvider, scenario.options) const mockResolver = new MockResolver( scenario.options.mocks?.tools as Record | undefined, this.config.unmockedTools, @@ -101,23 +107,32 @@ export class Simulator { try { for (let turnIndex = 0; turnIndex < maxTurns; turnIndex++) { - // 1. Simulated user generates a message - this.onProgress?.({ - scenario: scenario.name, - conversationId, - turn: turnIndex + 1, - maxTurns, - phase: 'user-message', - }) - const userResponse = await simulatedUser.generateMessage(conversationHistory) + // 1. Get the next user message + let userMessage: string + let shouldStop = false + + if (isScripted) { + userMessage = scriptedTurns![turnIndex].userMessage + } else { + this.onProgress?.({ + scenario: scenario.name, + conversationId, + turn: turnIndex + 1, + maxTurns, + phase: 'user-message', + }) + const userResponse = await simulatedUser!.generateMessage(conversationHistory) + + // On subsequent turns, if the simulated user signals stop *before* + // sending a new message, the goal is already met — end the conversation. + if (userResponse.shouldStop && turnIndex > 0) { + break + } - // On subsequent turns, if the simulated user signals stop *before* - // sending a new message, the goal is already met — end the conversation. - if (userResponse.shouldStop && turnIndex > 0) { - break + userMessage = userResponse.message + shouldStop = userResponse.shouldStop } - const userMessage = userResponse.message conversationHistory.push({ role: 'user', content: userMessage }) agentMessages.push({ role: 'user', content: userMessage }) @@ -125,6 +140,20 @@ export class Simulator { const turnToolCalls: ToolCallRecord[] = [] let agentFinalText = '' + // Set up context for custom handler (provides resolveTool for in-process agents) + const contextToolCalls: ToolCallRecord[] = [] + const handlerCtx: CustomHandlerContext = { + resolveTool: async (name, args) => { + const resolved = await mockResolver.resolve(name, args, turnIndex) + contextToolCalls.push({ name, args, result: resolved.result, turnIndex }) + return resolved.result + }, + turnIndex, + conversationId, + scenarioName: scenario.name, + } + this.agentClient.setHandlerContext(handlerCtx) + this.onProgress?.({ scenario: scenario.name, conversationId, @@ -173,6 +202,9 @@ export class Simulator { agentMessages.push(response.message) conversationHistory.push({ role: 'assistant', content: agentFinalText }) + // Merge tool calls resolved via ctx.resolveTool (custom handler in-process mocks) + turnToolCalls.push(...contextToolCalls) + turns.push({ turnIndex, userMessage, @@ -182,7 +214,7 @@ export class Simulator { // On turn 0 the user signalled stop with their first message — now // that the agent has replied once, honour that signal and end. - if (userResponse.shouldStop) { + if (shouldStop) { break } } diff --git a/src/vitest.ts b/src/vitest.ts index 0dfdd27..a286332 100644 --- a/src/vitest.ts +++ b/src/vitest.ts @@ -157,6 +157,22 @@ function formatScenarioFailure(sr: ScenarioResult): string { } } + // Per-turn trajectory failures + for (const [convId, turnResults] of sr.perTurnTrajectoryResults) { + for (const tr of turnResults) { + if (!tr.result.matched) { + const details: string[] = [] + if (tr.result.missingCalls.length > 0) + details.push(`missing: ${tr.result.missingCalls.join(', ')}`) + if (tr.result.extraCalls.length > 0) + details.push(`extra: ${tr.result.extraCalls.join(', ')}`) + if (tr.result.forbiddenCalls.length > 0) + details.push(`forbidden: ${tr.result.forbiddenCalls.join(', ')}`) + lines.push(` ${convId}: turn ${tr.turnIndex} trajectory failed — ${details.join('; ')}`) + } + } + } + // Metric averages const averages = computeMetricAverages(sr.evaluations.values()) if (Object.keys(averages).length > 0) {