Skip to content

Commit 600957b

Browse files
stephentoubCopilot
stephentoub
and
Copilot
committed
Normalize 1.0.72 multi-turn read_agent lifecycle framing in replay proxy
The 1.0.72 CLI reports a finished background agent as "Agent is idle (waiting for messages)." with status: idle and total_turns >= 1, and prefixes each turn's output with a "[Turn N]" marker, whereas the snapshots were recorded against a runtime that reported "Agent completed." with status: completed, total_turns: 0, and a trailing duration. This caused subagent_hooks to miss the cached read_agent result and fail. Add normalizeReadAgentLifecycle, applied symmetrically on stored snapshots and incoming requests and guarded on the leading status header, to collapse the lifecycle fields to stable placeholders, drop the trailing duration, and strip the per-turn markers so snapshots keep matching across runtime versions. The agent's actual response body is replayed from the snapshot, so only this CLI-generated framing drifts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e94bab38-c22e-4a38-8d8b-c82c1991a33c
1 parent 307fe21 commit 600957b

2 files changed

Lines changed: 165 additions & 2 deletions

File tree

‎test/harness/replayingCapiProxy.test.ts‎

Lines changed: 134 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -624,7 +624,7 @@ Always include PINEAPPLE_COCONUT_42.
624624
);
625625
});
626626

627-
test("normalizes read_agent timing metadata", async () => {
627+
test("normalizes read_agent timing and lifecycle metadata", async () => {
628628
const requestBody = JSON.stringify({
629629
messages: [
630630
{ role: "user", content: "Help me" },
@@ -662,7 +662,49 @@ Always include PINEAPPLE_COCONUT_42.
662662
(m) => m.role === "tool",
663663
);
664664
expect(toolMessage?.content).toBe(
665-
"Agent completed. agent_id: read-file, agent_type: explore, status: completed, description: Reading subagent-test.txt, elapsed: 0s, total_turns: 0, duration: 0s\n\nDone.",
665+
"Agent ${agent_state}. agent_id: read-file, agent_type: explore, status: ${agent_status}, description: Reading subagent-test.txt, elapsed: 0s, total_turns: ${turns}\n\nDone.",
666+
);
667+
});
668+
669+
test("normalizes the 1.0.72 idle-phrasing read_agent result to match the older completed form", async () => {
670+
const requestBody = JSON.stringify({
671+
messages: [
672+
{ role: "user", content: "Help me" },
673+
{
674+
role: "assistant",
675+
tool_calls: [
676+
{
677+
id: "tc1",
678+
type: "function",
679+
function: {
680+
name: "read_agent",
681+
arguments: '{"agent_id":"read-file","wait":true}',
682+
},
683+
},
684+
],
685+
},
686+
{
687+
role: "tool",
688+
tool_call_id: "tc1",
689+
content:
690+
"Agent is idle (waiting for messages). agent_id: read-file, agent_type: explore, status: idle, description: Reading subagent-test.txt, elapsed: 1.25s, total_turns: 1\n\n[Turn 0]\nDone.",
691+
},
692+
],
693+
});
694+
const responseBody = JSON.stringify({
695+
choices: [{ message: { role: "assistant", content: "Done" } }],
696+
});
697+
698+
const outputPath = await createProxy([
699+
{ url: "/chat/completions", requestBody, responseBody },
700+
]);
701+
702+
const result = await readYamlOutput(outputPath);
703+
const toolMessage = result.conversations[0].messages.find(
704+
(m) => m.role === "tool",
705+
);
706+
expect(toolMessage?.content).toBe(
707+
"Agent ${agent_state}. agent_id: read-file, agent_type: explore, status: ${agent_status}, description: Reading subagent-test.txt, elapsed: 0s, total_turns: ${turns}\n\nDone.",
666708
);
667709
});
668710

@@ -1414,6 +1456,96 @@ Always include PINEAPPLE_COCONUT_42.
14141456
}
14151457
});
14161458

1459+
test("matches cached read_agent result against the 1.0.72 idle-phrasing runtime", async () => {
1460+
const cachePath = path.join(tempDir, "cache.yaml");
1461+
// Snapshot recorded against the older runtime: an agent that finished its
1462+
// work reported "Agent completed." with a completed status and a trailing
1463+
// duration, and no per-turn markers.
1464+
const storedResult =
1465+
"Agent completed. agent_id: read-file, agent_type: explore, status: completed, description: Reading subagent-test.txt, elapsed: 0s, total_turns: 0, duration: 0s\n\nThe file says hello.";
1466+
// The 1.0.72 runtime reports the same finished agent as idle, with a
1467+
// total_turns count of at least 1 and a "[Turn N]" marker before the body.
1468+
const runtimeResult =
1469+
"Agent is idle (waiting for messages). agent_id: read-file, agent_type: explore, status: idle, description: Reading subagent-test.txt, elapsed: 0s, total_turns: 1\n\n[Turn 0]\nThe file says hello.";
1470+
1471+
const cacheContent = yaml.stringify({
1472+
models: ["test-model"],
1473+
conversations: [
1474+
{
1475+
messages: [
1476+
{ role: "system", content: "${system}" },
1477+
{ role: "user", content: "Read the file" },
1478+
{
1479+
role: "assistant",
1480+
tool_calls: [
1481+
{
1482+
id: "toolcall_0",
1483+
type: "function",
1484+
function: {
1485+
name: "read_agent",
1486+
arguments: '{"agent_id":"read-file","wait":true}',
1487+
},
1488+
},
1489+
],
1490+
},
1491+
{
1492+
role: "tool",
1493+
tool_call_id: "toolcall_0",
1494+
content: storedResult,
1495+
},
1496+
{ role: "assistant", content: "The file was read successfully." },
1497+
],
1498+
},
1499+
],
1500+
} satisfies NormalizedData);
1501+
await writeFile(cachePath, cacheContent);
1502+
1503+
const proxy = new ReplayingCapiProxy(
1504+
"http://localhost:9999",
1505+
cachePath,
1506+
workDir,
1507+
);
1508+
const proxyUrl = await proxy.start();
1509+
1510+
try {
1511+
const response = await makeRequest(proxyUrl, "/chat/completions", {
1512+
body: {
1513+
model: "test-model",
1514+
messages: [
1515+
{ role: "system", content: "Be helpful" },
1516+
{ role: "user", content: "Read the file" },
1517+
{
1518+
role: "assistant",
1519+
tool_calls: [
1520+
{
1521+
id: "runtime-call-id",
1522+
type: "function",
1523+
function: {
1524+
name: "read_agent",
1525+
arguments: '{"agent_id":"read-file","wait":true}',
1526+
},
1527+
},
1528+
],
1529+
},
1530+
{
1531+
role: "tool",
1532+
tool_call_id: "runtime-call-id",
1533+
content: runtimeResult,
1534+
},
1535+
],
1536+
},
1537+
});
1538+
1539+
expect(response.status).toBe(200);
1540+
expect(
1541+
(JSON.parse(response.body) as ChatCompletion).choices[0].message
1542+
.content,
1543+
).toBe("The file was read successfully.");
1544+
} finally {
1545+
await proxy.stop();
1546+
}
1547+
});
1548+
14171549
test("matches parallel tool results regardless of arrival order", async () => {
14181550
const cachePath = path.join(tempDir, "cache.yaml");
14191551
const cacheContent = yaml.stringify({

‎test/harness/replayingCapiProxy.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy {
129129
{ toolName: "*", normalizer: normalizeAvailableToolNames },
130130
{ toolName: "task", normalizer: normalizeBackgroundAgentAdvice },
131131
{ toolName: "read_agent", normalizer: normalizeReadAgentTimings },
132+
{ toolName: "read_agent", normalizer: normalizeReadAgentLifecycle },
132133
];
133134

134135
/**
@@ -1322,6 +1323,7 @@ function normalizeStoredToolMessages(conversations: NormalizedConversation[]) {
13221323
if (message.role === "tool" && typeof message.content === "string") {
13231324
message.content = normalizeAvailableToolNames(message.content);
13241325
message.content = normalizeBackgroundAgentAdvice(message.content);
1326+
message.content = normalizeReadAgentLifecycle(message.content);
13251327
}
13261328
}
13271329
}
@@ -1417,6 +1419,35 @@ function normalizeReadAgentTimings(result: string): string {
14171419
.replace(/\bduration: \d+(?:\.\d+)?s\b/g, "duration: 0s");
14181420
}
14191421

1422+
// Background-agent lifecycle framing changed with 1.0.72's multi-turn agents.
1423+
// An agent that has finished its work now reports
1424+
// "Agent is idle (waiting for messages)." with status: idle, total_turns: >=1
1425+
// whereas older runtimes reported
1426+
// "Agent completed." with status: completed, total_turns: 0, duration: <d>.
1427+
// 1.0.72 also prefixes each turn's output with a "[Turn N]" marker. None of this
1428+
// changes what the read_agent-based tests assert (e.g. subagent_hooks checks that
1429+
// hooks fire, not the completion wording), and the agent's actual response body is
1430+
// itself replayed from the snapshot, so only this CLI-generated framing drifts.
1431+
// Collapse the lifecycle fields to stable placeholders, drop the trailing
1432+
// duration, and strip the turn markers so snapshots keep matching across runtime
1433+
// versions. Applied symmetrically to stored snapshots and incoming requests, and
1434+
// guarded on the leading status header so unrelated tool results are never
1435+
// touched.
1436+
function normalizeReadAgentLifecycle(result: string): string {
1437+
if (!/^Agent (?:completed|is idle \(waiting for messages\))\./.test(result)) {
1438+
return result;
1439+
}
1440+
return result
1441+
.replace(
1442+
/^Agent (?:completed|is idle \(waiting for messages\))\./,
1443+
"Agent ${agent_state}.",
1444+
)
1445+
.replace(/\bstatus: \w+/, "status: ${agent_status}")
1446+
.replace(/\btotal_turns: \d+/, "total_turns: ${turns}")
1447+
.replace(/, duration: \d+(?:\.\d+)?s\b/g, "")
1448+
.replace(/\n\[Turn \d+\]\n/g, "\n");
1449+
}
1450+
14201451
// When a model calls a tool that doesn't exist (e.g., the removed report_intent
14211452
// tool), the runtime replies with "Tool '<name>' does not exist. Available tools
14221453
// that can be called are <list>." Some runtime versions omit that second sentence

0 commit comments

Comments
 (0)