From 0308a6cd167e37bf0c92ff3b24e239d35838e6f1 Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Wed, 8 Jul 2026 01:42:13 +0530 Subject: [PATCH] test(e2e): convert chat-harness-subagent to llmKeywordRules (#4517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #4519 for the residual chat-harness-subagent spec (the orchestrator -> researcher fan-out). The delegated researcher runs its own agent-harness loop, so the mock's global forced-response FIFO can still be shifted by an off-turn call even after #4519's ancillary no-tools gating — the scripted final canary then lands on the wrong turn or never renders. Switch this spec to `llmKeywordRules`: matches are case-insensitive substrings of the latest user/tool message per call and are never consumed, so any extra ancillary call falls through to the mock's dynamic default and leaves the scripted turns intact. Rules: - 'llama-research' -> orchestrator turn 1 -> research(prompt=...) - 'coded marker phrase' -> researcher turn -> RESEARCHER_REPLY text - 'researcher trace signal' -> orchestrator turn 2 -> final canary The sub-agent's user message is rendered as "Task:\n" by render_structured_handoff in archetype_delegation.rs, so DELEGATE_PROMPT surfaces verbatim to pickProbeText. The orchestrator's final turn sees RESEARCHER_REPLY as the role: tool message content via ToolResult::success(outcome.output) in dispatch.rs. The fire-and-forget title-gen call from threadSlice.ts shares the turn-1 probe but only consumes `content` (chat_with_system ignores tool_calls when tools were not requested) — benign for both callers. E2E-mock fixture only; no product/runtime change. Refs #4517 (does not close — chat-harness-subagent-continue.spec.ts has a separate {{DYNAMIC_TASK_ID}} placeholder blocker tracked separately). --- .../e2e/specs/chat-harness-subagent.spec.ts | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/app/test/e2e/specs/chat-harness-subagent.spec.ts b/app/test/e2e/specs/chat-harness-subagent.spec.ts index b9780c155c..c6756ec94a 100644 --- a/app/test/e2e/specs/chat-harness-subagent.spec.ts +++ b/app/test/e2e/specs/chat-harness-subagent.spec.ts @@ -10,12 +10,24 @@ * to a sub-agent which runs the agent harness loop a level deeper — * which means the LLM gets hit at least once more for the sub-agent. * + * Scripting: `llmKeywordRules` (content-addressed, never depleted). + * A prior version of this spec used the `llmForcedResponses` FIFO but + * that queue drains one entry per `/chat/completions` regardless of who + * called (#4517): the sub-agent's own harness loop plus any ancillary + * summarisation/memory-prep call shifts responses out of order and the + * scripted final canary lands on the wrong turn (or never renders). + * Keyword rules route each call by a substring of its latest + * user/tool message, so extra calls that don't match any rule fall + * through to the mock's dynamic default — the scripted turns are never + * consumed by an off-turn caller. + * * What this spec scripts and verifies: * - * 1. Configure `llmForcedResponses` with THREE responses in order: - * A) orchestrator turn — emits `research` tool_call - * B) researcher turn — answers with a plain text finding - * C) orchestrator turn — final synthesis text (canary marker) + * 1. Configure `llmKeywordRules` with three rules keyed on + * per-turn-unique tokens: + * A) orchestrator turn (user PROMPT) — emits `research` tool_call + * B) researcher turn (delegate prompt) — plain text finding + * C) orchestrator turn (tool result) — final synthesis (canary) * * 2. Send the user prompt and watch the runtime: * UI: @@ -52,27 +64,41 @@ import { navigateViaHash } from '../helpers/shared-flows'; import { getRequestLog, setMockBehavior, startMockServer, stopMockServer } from '../mock-server'; const USER_ID = 'e2e-chat-harness-subagent'; -const PROMPT = 'Research the answer to life and tell me a marker phrase.'; +// Per-turn tokens chosen so pickProbeText's substring match routes each +// call to exactly one rule regardless of any extra ancillary +// tool/context/summary call the harness may issue on top of the three +// happy-path turns. +const PROMPT = 'Please delegate a llama-research task and return the marker.'; +const DELEGATE_PROMPT = 'Return the coded marker phrase.'; +const RESEARCHER_REPLY = 'The researcher trace signal is FORTY-TWO.'; const CANARY_FINAL = 'subagent-canary-final-7afe2'; -const RESEARCHER_REPLY = 'The researcher answer is 42.'; -// Three forced responses, popped in order by the mock LLM streamer. -const FORCED_RESPONSES = [ - // 1. Orchestrator: emit a research tool call. +// Content-addressed keyword rules — never depleted, immune to extra +// ancillary /chat/completions calls (#4517). +const KEYWORD_RULES = [ + // Sub-agent turn — dispatch_subagent renders the arg as "Task:\n" + // (archetype_delegation.rs render_structured_handoff), so DELEGATE_PROMPT + // surfaces verbatim in the researcher's user message. + { keyword: 'coded marker phrase', content: RESEARCHER_REPLY }, + // Orchestrator's post-delegation turn — the sub-agent's output is handed + // back as the `role: tool` message content + // (dispatch.rs `ToolResult::success(outcome.output)`). + { keyword: 'researcher trace signal', content: `Done. The result is: ${CANARY_FINAL}` }, + // Orchestrator's initial turn — the fire-and-forget thread-title-gen + // call (threadSlice.ts, tools: None) sees the same probe, but + // chat_with_system consumes `content` and ignores unexpected tool_calls, + // so a delegation-triggering rule here is safe for both callers. { - content: '', + keyword: 'llama-research', + content: 'Delegating to researcher.', toolCalls: [ { id: 'call_research_1', name: 'research', - arguments: JSON.stringify({ prompt: 'Tell me a marker phrase' }), + arguments: JSON.stringify({ prompt: DELEGATE_PROMPT }), }, ], }, - // 2. Researcher sub-agent: produces a text answer. - { content: RESEARCHER_REPLY }, - // 3. Orchestrator final synthesis containing the canary. - { content: `Done. The result is: ${CANARY_FINAL}` }, ]; interface RuntimeSnapshot { @@ -132,14 +158,14 @@ describe('Chat harness — orchestrator → subagent flow', () => { '[chat-harness-subagent] Disabled super context for deterministic scripted LLM calls' ); - setMockBehavior('llmForcedResponses', JSON.stringify(FORCED_RESPONSES)); + setMockBehavior('llmKeywordRules', JSON.stringify(KEYWORD_RULES)); // Faster streaming for non-tool-call responses so this spec doesn't // need 30s of patience for three full streams. setMockBehavior('llmStreamChunkDelayMs', '10'); }); after(async () => { - setMockBehavior('llmForcedResponses', ''); + setMockBehavior('llmKeywordRules', ''); setMockBehavior('llmStreamChunkDelayMs', ''); await stopMockServer(); });