diff --git a/proxy/extensions.json b/proxy/extensions.json index 316b06ce..080185b1 100644 --- a/proxy/extensions.json +++ b/proxy/extensions.json @@ -1,5 +1,6 @@ { "bootstrap-defense": { "enabled": true, "order": 45 }, + "output-guard-stash": { "enabled": true, "order": 55 }, "request-capture": { "enabled": true, "order": 60 }, "ttl-tier-detect": { "enabled": true, "order": 75 }, "cc-version-normalize": { "enabled": true, "order": 90 }, @@ -25,6 +26,7 @@ "overage-warning": { "enabled": true, "order": 610 }, "upstream-error-log": { "enabled": true, "order": 670 }, "session-budget-breaker": { "enabled": true, "order": 690 }, + "output-guard": { "enabled": true, "order": 690 }, "request-log": { "enabled": false, "order": 700 }, "jsonl-session-mirror": { "enabled": true, "order": 720 } } diff --git a/proxy/extensions/output-guard-stash.mjs b/proxy/extensions/output-guard-stash.mjs new file mode 100644 index 00000000..9eb1271d --- /dev/null +++ b/proxy/extensions/output-guard-stash.mjs @@ -0,0 +1,32 @@ +// output-guard-stash — first half of the output guard (directive: +// docs/directives/proxy-output-guard.md). Order 55: before the first +// body-mutating extension (cc-version-normalize, 90), so the stash is +// what CC actually sent. The validating half is output-guard.mjs +// (order 690). Two files because the pipeline loads one default export +// per file and the two halves must run at opposite ends of the chain. + +export function isGuardEnabled(env = process.env) { + return env.CACHE_FIX_OUTPUT_GUARD === "1"; +} + +export default { + name: "output-guard-stash", + description: + "Stash a pre-mutation clone of the request body for output-guard's " + + "restore path", + enabled: false, // overridden by extensions.json + order: 55, + + async onRequest(ctx) { + if (!isGuardEnabled()) return; + if (!ctx || !ctx.body || !Array.isArray(ctx.body.messages)) return; + try { + ctx.meta = ctx.meta || {}; + ctx.meta._preMutationBody = structuredClone(ctx.body); + } catch { + // A failed stash disables the restore path for this request only; + // output-guard treats a missing stash as "cannot restore" and + // passes through with a WARN. + } + }, +}; diff --git a/proxy/extensions/output-guard.mjs b/proxy/extensions/output-guard.mjs new file mode 100644 index 00000000..e6f05bc8 --- /dev/null +++ b/proxy/extensions/output-guard.mjs @@ -0,0 +1,201 @@ +// output-guard — last-line invariant check on the outgoing body. +// Directive: docs/directives/proxy-output-guard.md. +// +// Protects against US, not CC: the pipeline mutates bodies (reorder, +// rewrite, inject, and since phase 3 substitute first-seen bytes), and +// the composition of individually-correct extensions is where the one +// real shipped defect lived. On any hard-invariant violation the guard +// forwards CC's ORIGINAL bytes (stashed by output-guard-stash at order +// 55) — valid by definition, they are what the API would have received +// with the proxy absent. +// +// Order 690: after the last body mutator (ttl-management, 500), before +// the pure observers (request-log 700, jsonl-session-mirror 720) so +// what they record is what actually goes out. +// +// Fail-open at the mutation level, fail-safe at the request level: a +// validator crash counts as "cannot verify" and passes the mutated body +// through with a WARN — the guard must never be able to break a request +// or silently disable the pipeline's value. + +import { appendFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { claudeHome } from "../claude-home.mjs"; +import { resolveSessionId } from "./cache-telemetry.mjs"; +import { validateToolAdjacency } from "./insertion-normalization.mjs"; +import { isGuardEnabled } from "./output-guard-stash.mjs"; + +const MAX_MARKERS = 4; + +function isDebug(env = process.env) { + return env.CACHE_FIX_DEBUG === "1"; +} + +function getSnapshotDir() { + return join(claudeHome(), "cache-fix-snapshots"); +} + +// --- Invariant validators (pure; each returns null or a violation string) --- + +function checkToolAdjacency(body) { + return validateToolAdjacency(body.messages) + ? null + : "tool-adjacency: a tool_result user message is not preceded by its matching tool_use assistant message"; +} + +function countMarkers(body) { + let n = 0; + const scan = (blocks) => { + if (!Array.isArray(blocks)) return; + for (const b of blocks) { + if (b && typeof b === "object" && b.cache_control) n++; + } + }; + scan(body.system); + for (const m of body.messages) scan(m?.content); + return n; +} + +function checkMarkerBudget(body) { + const n = countMarkers(body); + return n <= MAX_MARKERS ? null : `marker-budget: ${n} cache_control markers exceed the API cap of ${MAX_MARKERS}`; +} + +// "system" is legal mid-conversation (mid-conversation system messages, +// and deferred-tool-rewrite's injected tool_addition messages) — but never +// as messages[0], per the documented placement constraint. +function checkRoles(body) { + for (let i = 0; i < body.messages.length; i++) { + const r = body.messages[i]?.role; + if (r !== "user" && r !== "assistant" && r !== "system") { + return `roles: messages[${i}] has invalid role ${JSON.stringify(r)}`; + } + if (r === "system" && i === 0) { + return `roles: messages[0] must not be system (placement constraint)`; + } + } + return null; +} + +function checkContentPresent(body) { + for (let i = 0; i < body.messages.length; i++) { + const c = body.messages[i]?.content; + const ok = typeof c === "string" || (Array.isArray(c) && c.length > 0); + if (!ok) return `content: messages[${i}] has missing or empty content`; + } + return null; +} + +// Invariant 5 (BACKLOG.md, "suppression can strip a request's FINAL +// message", 2026-07-30): a message-REMOVING mutation shipped (duplicate +// suppression) without a tail-validity check, and three live requests +// ended assistant-role -> upstream "400 must end with a user message". +// The other four invariants are shape-level facts about ONE body; this +// one needs the body CC actually sent, so it takes it as a second +// argument rather than deriving anything from `body` alone. +// +// Conditioned on the INCOMING shape rather than an unconditional "never +// end assistant": if CC itself sent a request already ending in +// assistant role (a prefill-style continuation, however rare in observed +// traffic), that is the client's own intent and not this guard's business +// to overturn — the guard protects against OUR mutations, not against CC. +// `incomingBody` absent (e.g. the pre-mutation stash unavailable, or a +// direct unit-test call) means "cannot verify" for this one check, so it +// yields no violation rather than guessing. +function checkAssistantTerminal(body, incomingBody) { + if (!incomingBody || !Array.isArray(incomingBody.messages) || incomingBody.messages.length === 0) return null; + const incomingLast = incomingBody.messages[incomingBody.messages.length - 1]; + if (incomingLast?.role === "assistant") return null; + const forwardedLast = body.messages[body.messages.length - 1]; + if (forwardedLast?.role === "assistant") { + return "assistant-terminal: incoming request ended non-assistant but the forwarded body ends assistant — a mutation stripped the trailing message"; + } + return null; +} + +const VALIDATORS = [checkToolAdjacency, checkMarkerBudget, checkRoles, checkContentPresent, checkAssistantTerminal]; + +// Exported for tests: run all validators, return the first violation or +// null. `incomingBody` is optional — only checkAssistantTerminal reads it; +// every other validator is unaffected by its absence. +export function findViolation(body, incomingBody) { + for (const v of VALIDATORS) { + const violation = v(body, incomingBody); + if (violation) return violation; + } + return null; +} + +async function appendGuardEvent(dir, key, record) { + try { + await mkdir(dir, { recursive: true }); + await appendFile(join(dir, `${key}-guard-events.jsonl`), JSON.stringify(record) + "\n"); + } catch { + // Telemetry loss must not affect the request. + } +} + +export default { + name: "output-guard", + description: + "Validate hard invariants (tool adjacency, marker budget, roles, " + + "content presence) on the outgoing body; on violation forward the " + + "pre-mutation original instead", + enabled: false, // overridden by extensions.json + order: 690, + + async onRequest(ctx) { + if (!isGuardEnabled()) return; + if (!ctx || !ctx.body || !Array.isArray(ctx.body.messages)) return; + + ctx.meta = ctx.meta || {}; + let violation; + try { + violation = findViolation(ctx.body, ctx.meta._preMutationBody); + } catch (err) { + // Cannot verify -> pass the mutated body through (fail-open); a + // guard crash must never break the request or the pipeline's value. + ctx.meta.outputGuardStats = { verified: false, error: String(err?.message ?? err) }; + process.stderr.write( + `[output-guard] WARN: validator crashed (${err?.message ?? err}) — body passed through UNVERIFIED\n`, + ); + return; + } + + if (!violation) { + ctx.meta.outputGuardStats = { verified: true, fired: false }; + return; + } + + const stash = ctx.meta._preMutationBody; + const sid = ctx.headers ? resolveSessionId(ctx.headers) : null; + const key = sid ? `s-${sid.replace(/[^A-Za-z0-9_-]/g, "_")}` : "nokey"; + + if (stash) { + ctx.body = stash; + ctx.meta.outputGuardStats = { verified: true, fired: true, violation, restored: true }; + process.stderr.write( + `[output-guard] CRITICAL: ${violation} — pipeline output DISCARDED, original client body forwarded. An extension (or interaction) produced an invalid body; see ${key}-guard-events.jsonl\n`, + ); + } else { + // No stash (stash failed or guard enabled mid-request): nothing safe + // to restore to; forward the mutated body and say so loudly. + ctx.meta.outputGuardStats = { verified: true, fired: true, violation, restored: false }; + process.stderr.write( + `[output-guard] CRITICAL: ${violation} — NO pre-mutation stash available, mutated body forwarded as-is\n`, + ); + } + + await appendGuardEvent(getSnapshotDir(), key, { + ts: new Date().toISOString(), + sid, + violation, + restored: Boolean(stash), + messageCount: Array.isArray(ctx.body?.messages) ? ctx.body.messages.length : 0, + }); + + if (isDebug()) { + process.stderr.write(`[output-guard] DEBUG: stats=${JSON.stringify(ctx.meta.outputGuardStats)}\n`); + } + }, +}; diff --git a/test/fixtures/replay-classes/corpus-compaction.jsonl b/test/fixtures/replay-classes/corpus-compaction.jsonl new file mode 100644 index 00000000..2bb7b259 --- /dev/null +++ b/test/fixtures/replay-classes/corpus-compaction.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "x1"}]}, {"role": "user", "content": [{"type": "text", "text": "x2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "x3"}]}, {"role": "user", "content": [{"type": "text", "text": "x4"}]}, {"role": "assistant", "content": [{"type": "text", "text": "x5"}]}, {"role": "user", "content": [{"type": "text", "text": "x6"}]}, {"role": "assistant", "content": [{"type": "text", "text": "x7"}]}, {"role": "user", "content": [{"type": "text", "text": "x8"}]}, {"role": "assistant", "content": [{"type": "text", "text": "x9"}]}, {"role": "user", "content": [{"type": "text", "text": "x10"}]}, {"role": "assistant", "content": [{"type": "text", "text": "x11"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "user", "content": [{"type": "text", "text": "summary of everything"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-edit.jsonl b/test/fixtures/replay-classes/corpus-edit.jsonl new file mode 100644 index 00000000..be62578b --- /dev/null +++ b/test/fixtures/replay-classes/corpus-edit.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "original text"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "EDITED text"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-flip.jsonl b/test/fixtures/replay-classes/corpus-flip.jsonl new file mode 100644 index 00000000..a8aaaed7 --- /dev/null +++ b/test/fixtures/replay-classes/corpus-flip.jsonl @@ -0,0 +1,3 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "deep"}, {"type": "text", "text": "\nPreToolUse:Edit hook additional context: Spec-origin trace required\n"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "deep"}, {"type": "text", "text": "\nPreToolUse:Edit hook additional context: Spec-origin trace required\n"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}, {"role": "user", "content": [{"type": "text", "text": "q"}]}]}} +{"ts": "2026-07-28T02:02:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "deep"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}, {"role": "user", "content": [{"type": "text", "text": "next"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-flipback.jsonl b/test/fixtures/replay-classes/corpus-flipback.jsonl new file mode 100644 index 00000000..47d0d4c2 --- /dev/null +++ b/test/fixtures/replay-classes/corpus-flipback.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "target"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "target"}, {"type": "text", "text": "\nPreToolUse:Edit hook additional context: Spec-origin trace required\n"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}, {"role": "user", "content": [{"type": "text", "text": "go"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-prune.jsonl b/test/fixtures/replay-classes/corpus-prune.jsonl new file mode 100644 index 00000000..84ed1e10 --- /dev/null +++ b/test/fixtures/replay-classes/corpus-prune.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "m1"}]}, {"role": "user", "content": [{"type": "text", "text": "m2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m3"}]}, {"role": "user", "content": [{"type": "text", "text": "m4"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m5"}]}, {"role": "user", "content": [{"type": "text", "text": "m6"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m7"}]}, {"role": "user", "content": [{"type": "text", "text": "m8"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m9"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "m3"}]}, {"role": "user", "content": [{"type": "text", "text": "m4"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m5"}]}, {"role": "user", "content": [{"type": "text", "text": "m6"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m7"}]}, {"role": "user", "content": [{"type": "text", "text": "m8"}]}, {"role": "assistant", "content": [{"type": "text", "text": "m9"}]}, {"role": "user", "content": [{"type": "text", "text": "tail"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-sidecar.jsonl b/test/fixtures/replay-classes/corpus-sidecar.jsonl new file mode 100644 index 00000000..fa096b9c --- /dev/null +++ b/test/fixtures/replay-classes/corpus-sidecar.jsonl @@ -0,0 +1,3 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "main-0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "main-1"}]}]}} +{"ts": "2026-07-28T02:00:05Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "Generate a concise title", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "give this a title"}]}]}} +{"ts": "2026-07-28T02:00:10Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "main-0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "main-1"}]}, {"role": "user", "content": [{"type": "text", "text": "main-2"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-splice.jsonl b/test/fixtures/replay-classes/corpus-splice.jsonl new file mode 100644 index 00000000..af8265bf --- /dev/null +++ b/test/fixtures/replay-classes/corpus-splice.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "u2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "INJECTED"}]}, {"role": "user", "content": [{"type": "text", "text": "u2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-tooladd.jsonl b/test/fixtures/replay-classes/corpus-tooladd.jsonl new file mode 100644 index 00000000..b1ecc96a --- /dev/null +++ b/test/fixtures/replay-classes/corpus-tooladd.jsonl @@ -0,0 +1,4 @@ +{"ts": "2026-07-28T03:00:00Z", "sid": "tooladd-sid", "key": "s-tooladd-sid", "headers": {"anthropic-beta": "context-1m-2025-08-07", "session-id": "tooladd-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "tools": [{"name": "Read", "description": "Read tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "Bash", "description": "Bash tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0"}]}]}} +{"ts": "2026-07-28T03:00:10Z", "sid": "tooladd-sid", "key": "s-tooladd-sid", "headers": {"anthropic-beta": "context-1m-2025-08-07", "session-id": "tooladd-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "tools": [{"name": "Read", "description": "Read tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "Bash", "description": "Bash tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "u2"}]}]}} +{"ts": "2026-07-28T03:00:20Z", "sid": "tooladd-sid", "key": "s-tooladd-sid", "headers": {"anthropic-beta": "context-1m-2025-08-07", "session-id": "tooladd-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "tools": [{"name": "Read", "description": "Read tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "Bash", "description": "Bash tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "SendMessage", "description": "SendMessage tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "u2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}]}} +{"ts": "2026-07-28T03:00:30Z", "sid": "tooladd-sid", "key": "s-tooladd-sid", "headers": {"anthropic-beta": "context-1m-2025-08-07", "session-id": "tooladd-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "tools": [{"name": "Read", "description": "Read tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "Bash", "description": "Bash tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}, {"name": "SendMessage", "description": "SendMessage tool", "input_schema": {"type": "object", "properties": {"x": {"type": "string"}}}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, {"role": "user", "content": [{"type": "text", "text": "u2"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a3"}]}, {"role": "user", "content": [{"type": "text", "text": "u4"}]}]}} diff --git a/test/fixtures/replay-classes/corpus-toolpair.jsonl b/test/fixtures/replay-classes/corpus-toolpair.jsonl new file mode 100644 index 00000000..8e728e89 --- /dev/null +++ b/test/fixtures/replay-classes/corpus-toolpair.jsonl @@ -0,0 +1,2 @@ +{"ts": "2026-07-28T02:00:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "ls"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]}, {"role": "user", "content": [{"type": "text", "text": "after tools"}, {"type": "text", "text": "\nPreToolUse:Edit hook additional context: Spec-origin trace required\n"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a"}]}]}} +{"ts": "2026-07-28T02:01:00Z", "sid": "test-sid", "key": "s-test-sid", "headers": {"anthropic-beta": "context-management-2025-06-27", "session-id": "test-sid"}, "body": {"model": "claude-opus-5", "system": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}], "messages": [{"role": "user", "content": [{"type": "text", "text": "u0", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "ls"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]}, {"role": "user", "content": [{"type": "text", "text": "after tools"}]}, {"role": "assistant", "content": [{"type": "text", "text": "a"}]}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t2", "name": "Bash", "input": {"command": "ls"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "ok"}]}]}} diff --git a/test/output-guard.test.mjs b/test/output-guard.test.mjs new file mode 100644 index 00000000..04c78a51 --- /dev/null +++ b/test/output-guard.test.mjs @@ -0,0 +1,286 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createHash } from "node:crypto"; + +import guard, { findViolation } from "../proxy/extensions/output-guard.mjs"; +import stash from "../proxy/extensions/output-guard-stash.mjs"; +import { loadExtensions, runOnRequest } from "../proxy/pipeline.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const EXT_DIR = join(__dirname, "..", "proxy", "extensions"); +const EXT_CONFIG = join(__dirname, "..", "proxy", "extensions.json"); +const FIXTURES = join(__dirname, "fixtures", "replay-classes"); + +const cc = { cache_control: { type: "ephemeral" } }; + +function sha(v) { + return createHash("sha256").update(JSON.stringify(v)).digest("hex").slice(0, 12); +} + +function userMsg(text) { + return { role: "user", content: [{ type: "text", text }] }; +} + +function goodBody() { + return { + model: "m", + system: [{ type: "text", text: "sys", ...cc }], + messages: [ + userMsg("u0"), + { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Bash", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "t1", content: "ok" }] }, + ], + }; +} + +async function silenced(fn) { + const orig = process.stderr.write.bind(process.stderr); + process.stderr.write = () => true; + try { + return await fn(); + } finally { + process.stderr.write = orig; + } +} + +async function withGuardEnv(fn) { + const dir = await mkdtemp(join(tmpdir(), "output-guard-test-")); + const saved = { CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR, CACHE_FIX_OUTPUT_GUARD: process.env.CACHE_FIX_OUTPUT_GUARD }; + process.env.CLAUDE_CONFIG_DIR = dir; + process.env.CACHE_FIX_OUTPUT_GUARD = "1"; + try { + return await silenced(() => fn(dir)); + } finally { + for (const k of Object.keys(saved)) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + await rm(dir, { recursive: true, force: true }); + } +} + +// --- Validator unit coverage --- + +test("findViolation: healthy body -> null", () => { + assert.equal(findViolation(goodBody()), null); +}); + +test("findViolation: broken tool adjacency named", () => { + const b = goodBody(); + b.messages.splice(2, 0, userMsg("interloper")); + assert.match(findViolation(b), /tool-adjacency/); +}); + +test("findViolation: fifth marker named", () => { + const b = goodBody(); + for (let i = 0; i < 4; i++) b.messages.push({ role: "user", content: [{ type: "text", text: `m${i}`, ...cc }] }); + // system already carries 1 -> total 5. But adjacency must stay valid: + // appended AFTER the tool_result, fine. + assert.match(findViolation(b), /marker-budget: 5/); +}); + +test("findViolation: invalid role and empty content named", () => { + const b1 = goodBody(); + b1.messages.push({ role: "tool", content: [{ type: "text", text: "x" }] }); + assert.match(findViolation(b1), /roles: messages\[3\]/); + // system is legal mid-conversation (tool_addition injections, + // mid-conversation system messages) but never as messages[0] + const b2sys = goodBody(); + b2sys.messages.push({ role: "system", content: [{ type: "text", text: "ok" }] }); + assert.equal(findViolation(b2sys), null); + const b3sys = goodBody(); + b3sys.messages.unshift({ role: "system", content: [{ type: "text", text: "bad" }] }); + assert.match(findViolation(b3sys), /messages\[0\] must not be system/); + const b2 = goodBody(); + b2.messages.push({ role: "user", content: [] }); + assert.match(findViolation(b2), /content: messages\[3\]/); +}); + +// --- Invariant 5: assistant-terminal (BACKLOG.md, "suppression can strip +// a request's FINAL message", 2026-07-30) --- + +test("findViolation: healthy body, incoming also ends non-assistant -> null (no incomingBody = cannot verify, also null)", () => { + const b = goodBody(); // ends on a tool_result (role user) + assert.equal(findViolation(b, b), null); + assert.equal(findViolation(b), null, "no incomingBody -> this check cannot fire"); +}); + +test("findViolation: incoming ended non-assistant but forwarded ends assistant -> assistant-terminal named", () => { + const incoming = goodBody(); // last message role "user" + const forwarded = goodBody(); + forwarded.messages.pop(); // simulate a mutation stripping the trailing tool_result + assert.equal(forwarded.messages[forwarded.messages.length - 1].role, "assistant"); + assert.match(findViolation(forwarded, incoming), /assistant-terminal/); +}); + +test("findViolation: incoming ITSELF ended assistant (prefill-shaped) -> not this guard's business, no violation", () => { + const incoming = goodBody(); + incoming.messages.push({ role: "assistant", content: [{ type: "text", text: "partial" }] }); + const forwarded = structuredClone(incoming); // forwarded also ends assistant, matching CC's own intent + assert.equal(findViolation(forwarded, incoming), null); +}); + +test("findViolation: incoming and forwarded both end non-assistant -> null (healthy case)", () => { + const incoming = goodBody(); + const forwarded = structuredClone(incoming); + assert.equal(findViolation(forwarded, incoming), null); +}); + +test("gate 2 (assistant-terminal): a mutator that strips the trailing message is caught, forwards the original, telemetry names it", async () => { + await withGuardEnv(async (dir) => { + const body = goodBody(); + const originalHash = sha(body); + const ctx = { body, headers: { "x-session-id": "tail-strip-test" }, meta: { route: "messages" } }; + const stripTailMutator = { + name: "test-strip-tail-mutator", + order: 300, + async onRequest(c) { + c.body.messages.pop(); + }, + }; + await runOnRequest(ctx, [stash, stripTailMutator, guard]); + + assert.equal(ctx.meta.outputGuardStats.fired, true); + assert.equal(ctx.meta.outputGuardStats.restored, true); + assert.match(ctx.meta.outputGuardStats.violation, /assistant-terminal/); + assert.equal(sha(ctx.body), originalHash, "forwarded body is byte-identical to the pre-pipeline original"); + + const events = await readFile(join(dir, "cache-fix-snapshots", "s-tail-strip-test-guard-events.jsonl"), "utf-8"); + assert.match(events, /assistant-terminal/, "telemetry record names the violated invariant"); + }); +}); + +// --- Gate 1: zero fires on all healthy class corpora --- + +// The corpus COUNT is deliberately not pinned. It was (`=== 8`), and adding a +// ninth corpus in 3aeafef turned this into a hard failure — so a BLOCKING gate +// stopped validating anything the moment the regression set grew, which is the +// opposite of what a gate is for. Extending coverage must never break the +// check that consumes it. What matters is that the corpora are present and +// that every one of them replays without firing the guard; both are asserted. +test("gate 1: guard fires zero times across every class-matrix corpus", async () => { + await withGuardEnv(async () => { + const extensions = await loadExtensions(EXT_DIR, EXT_CONFIG); + const files = (await readdir(FIXTURES)).filter((f) => f.endsWith(".jsonl")); + assert.ok(files.length >= 8, `class-matrix corpora missing: found ${files.length}`); + let requests = 0; + for (const f of files) { + const lines = (await readFile(join(FIXTURES, f), "utf-8")).split("\n").filter((l) => l.trim()); + for (const line of lines) { + const rec = JSON.parse(line); + const ctx = { + body: structuredClone(rec.body), + headers: { "x-session-id": rec.headers?.["session-id"] ?? rec.sid }, + meta: { route: "messages" }, + }; + await runOnRequest(ctx, extensions); + requests++; + assert.notEqual(ctx.meta.outputGuardStats?.fired, true, `guard fired on healthy traffic in ${f}`); + } + } + assert.ok(requests >= 18, "corpora actually replayed"); + }); +}); + +// --- Gate 2: injection proof — a broken mutator is caught and undone --- + +function brokenAdjacencyMutator() { + return { + name: "test-broken-mutator", + order: 300, + async onRequest(ctx) { + // Splice a user message between tool_use and tool_result — the + // composition-defect shape the guard exists for. + const i = ctx.body.messages.findIndex( + (m) => m.role === "user" && Array.isArray(m.content) && m.content.some((b) => b?.type === "tool_result"), + ); + if (i > 0) ctx.body.messages.splice(i, 0, userMsg("BROKEN")); + }, + }; +} + +function fifthMarkerMutator() { + return { + name: "test-marker-mutator", + order: 300, + async onRequest(ctx) { + for (let i = 0; i < 5; i++) { + ctx.body.messages.push({ role: "user", content: [{ type: "text", text: `mk${i}`, ...cc }] }); + } + }, + }; +} + +for (const [label, mutator, pattern] of [ + ["adjacency break", brokenAdjacencyMutator, /tool-adjacency/], + ["marker overflow", fifthMarkerMutator, /marker-budget/], +]) { + test(`gate 2 (${label}): guard fires, forwards byte-identical original, telemetry names the invariant`, async () => { + await withGuardEnv(async (dir) => { + const body = goodBody(); + const originalHash = sha(body); + const ctx = { body, headers: { "x-session-id": "inject-test" }, meta: { route: "messages" } }; + await runOnRequest(ctx, [stash, mutator(), guard]); + + assert.equal(ctx.meta.outputGuardStats.fired, true); + assert.equal(ctx.meta.outputGuardStats.restored, true); + assert.match(ctx.meta.outputGuardStats.violation, pattern); + assert.equal(sha(ctx.body), originalHash, "forwarded body is byte-identical to the pre-pipeline original"); + + const events = await readFile( + join(dir, "cache-fix-snapshots", "s-inject-test-guard-events.jsonl"), + "utf-8", + ); + assert.match(events, pattern, "telemetry record names the violated invariant"); + }); + }); +} + +test("gate 2 addendum: no stash (guard enabled without stash ext) -> mutated body forwarded, restored=false", async () => { + await withGuardEnv(async () => { + const ctx = { body: goodBody(), headers: {}, meta: { route: "messages" } }; + await runOnRequest(ctx, [brokenAdjacencyMutator(), guard]); + assert.equal(ctx.meta.outputGuardStats.fired, true); + assert.equal(ctx.meta.outputGuardStats.restored, false); + }); +}); + +// --- Gate 3: guard crash -> fail-open, request unharmed --- + +test("gate 3: validator crash passes the mutated body through with verified=false", async () => { + await withGuardEnv(async () => { + // A body whose messages array is a Proxy that throws on access deep + // enough to pass the entry checks but crash a validator. + const body = goodBody(); + let reads = 0; + body.messages = new Proxy(body.messages, { + get(target, prop, receiver) { + if (prop === "length" && ++reads > 2) throw new Error("synthetic validator crash"); + return Reflect.get(target, prop, receiver); + }, + }); + const ctx = { body, headers: {}, meta: { route: "messages" } }; + await runOnRequest(ctx, [guard]); + assert.equal(ctx.meta.outputGuardStats.verified, false); + assert.match(ctx.meta.outputGuardStats.error, /synthetic validator crash/); + }); +}); + +// --- Flag off: complete no-op --- + +test("flag off: guard and stash are no-ops, no stats, no stash key", async () => { + const saved = process.env.CACHE_FIX_OUTPUT_GUARD; + delete process.env.CACHE_FIX_OUTPUT_GUARD; + try { + const ctx = { body: goodBody(), headers: {}, meta: { route: "messages" } }; + await runOnRequest(ctx, [stash, guard]); + assert.equal(ctx.meta.outputGuardStats, undefined); + assert.equal(ctx.meta._preMutationBody, undefined); + } finally { + if (saved !== undefined) process.env.CACHE_FIX_OUTPUT_GUARD = saved; + } +});