Preserve concrete specifics in Atlas distillation rewrites

src/__tests__/atlas-distillation-gate.test.ts

@@ -171,7 +171,47 @@ const FLIP_DISTILLED_TO_RESTATEMENT_CONTENT =
   `${FLIP_DISTILLED_TO_RESTATEMENT_MARKER}: the PR bumps the http client ` +
   "dependency to the latest patch release.";
 
+// PRESERVE-SPECIFICS gate-plumbing guard (companion to the real-LLM eval in
+// atlas-distillation-rewrite-eval.test.ts): an admin-ops-style candidate whose
+// MOCKED judge returns a `rewritten` verdict that RETAINS every concrete token.
+// This locks the CONTRACT that the gate flows the judge's rewrite content/title
+// through INTACT — it is a replay, so it stays green regardless of the prompt
+// (it is the plumbing guard, NOT the prompt proof). The prompt proof lives in
+// the opt-in real-LLM eval.
+const PRESERVE_MARKER = "PRESERVE-SPECIFICS-CASE";
+const PRESERVE_TITLE = `PR #412 (${PRESERVE_MARKER}): unify admin auth on ANALYTICS_TOKEN`;
+const PRESERVE_CONTENT =
+  `${PRESERVE_MARKER}: adds a POST /admin/:op endpoint, validates the ` +
+  "ANALYTICS_TOKEN header with timingSafeEqual, returns 202/400/401/503, and " +
+  "sets trust_proxy fail-closed. Removes the old PATHFINDER_ADMIN_TOKEN.";
+// The judge's rewrite: sharpened WHY prose that RETAINS every concrete token
+// (the post-fix behavior the real-LLM eval proves the prompt now produces).
+const PRESERVE_REWRITE_TITLE =
+  "POST /admin/:op unifies admin auth on ANALYTICS_TOKEN, timing-safe";
+const PRESERVE_REWRITE_CONTENT =
+  "The POST /admin/:op endpoint validates the ANALYTICS_TOKEN header with " +
+  "timingSafeEqual so a wrong token cannot be told apart by response timing; it " +
+  "returns 202 on success, 400 on a malformed body, 401 on a bad token, and 503 " +
+  "when overloaded. trust_proxy is fail-closed: an unresolved forwarded client " +
+  "IP is rejected. Collapsing PATHFINDER_ADMIN_TOKEN into ANALYTICS_TOKEN leaves " +
+  "operators one credential and one auth path to audit.";
+
 const fixtures: Fixture[] = [
+  // PRESERVE-SPECIFICS plumbing guard: rewritten verdict retaining the tokens.
+  {
+    match: {
+      systemMessage: DISTILL_SYSTEM_MARKER,
+      userMessage: PRESERVE_MARKER,
+    },
+    response: {
+      content: JSON.stringify({
+        verdict: "rewritten",
+        reason: "sharpened the WHY while keeping every endpoint/code/symbol",
+        title: PRESERVE_REWRITE_TITLE,
+        content: PRESERVE_REWRITE_CONTENT,
+      }),
+    },
+  },
   // Salvageable → rewritten (gated on the salvage marker in the user payload).
   {
     match: {
@@ -477,6 +517,9 @@ describe("enforceDistillation (aimock-backed real judge)", () => {
   });
 
   beforeEach(() => {
+    // Defensive/no-op for these fixtures: aimock only consults match counts for
+    // sequenceIndex fixtures, and every fixture here matches on message content
+    // (no sequenceIndex). Kept so adding a sequenced fixture later stays correct.
     mock.resetMatchCounts();
   });
 
@@ -624,6 +667,36 @@ describe("enforceDistillation (aimock-backed real judge)", () => {
     expect(validated.approvable).toBe(true);
   });
 
+  it("rewritten verdict flows the judge's specifics-preserving rewrite through INTACT (PRESERVE-SPECIFICS plumbing guard)", async () => {
+    // Companion to the opt-in real-LLM eval: that eval proves the PROMPT makes a
+    // real model retain concrete tokens on a rewrite; THIS aimock replay locks
+    // the GATE-PLUMBING contract — whatever content/title the judge returns on a
+    // `rewritten` verdict is what enforceDistillation swaps in, verbatim. So when
+    // the judge returns a specifics-preserving rewrite, the concrete tokens
+    // survive the gate.
+    const cand = makeCandidate({
+      title: PRESERVE_TITLE,
+      content: PRESERVE_CONTENT,
+      knowledge_type: "security",
+    });
+
+    const [gated] = await enforceDistillation([cand], { judge });
+
+    // Title/content are swapped for the judge's specifics-preserving rewrite.
+    expect(gated.title).toBe(PRESERVE_REWRITE_TITLE);
+    expect(gated.content).toBe(PRESERVE_REWRITE_CONTENT);
+    // The concrete verifiable detail flows through the gate intact.
+    expect(gated.content).toContain("POST /admin/:op");
+    expect(gated.content).toContain("timingSafeEqual");
+    expect(gated.content).toMatch(/\b401\b/);
+    expect(gated.content).toContain("trust_proxy");
+    expect(gated.title).toContain("POST /admin/:op");
+    // Salvage breadcrumb, not the restatement floor.
+    expect(gated.provenance.validated_against ?? "").toContain(
+      REWRITTEN_FROM_RESTATEMENT_MARKER,
+    );
+  });
+
   it("restatement→rewritten flip strips a PRIOR run's stale RESTATEMENT_MARKER so validate no longer floors the salvage", async () => {
     // A candidate a PRIOR run ruled a pure `restatement`: it carries the stale
     // floor marker on validated_against. THIS run's judge flips it to

src/__tests__/atlas-distillation-rewrite-eval.test.ts

@@ -0,0 +1,99 @@
+// Layer-2 REAL-LLM eval for the distillation judge's REWRITE branch (Theme A.1).
+//
+// ORG RULE: an aimock REPLAY test cannot prove a PROMPT change — the fixture is
+// canned, so it will pass regardless of what the prompt says. Proving that
+// DISTILLATION_SYSTEM_PROMPT actually stops the judge from paraphrasing away
+// concrete verifiable detail requires exercising the REAL failure surface: a
+// REAL OpenAI call through the REAL prompt. This file does exactly that.
+//
+// It is OPT-IN — gated on `OPENAI_API_KEY`. In normal CI (no key) the whole
+// suite is SKIPPED via `describe.skipIf`, so it never spends tokens or flakes on
+// missing credentials. It runs only when a real key is present (the red-green
+// proof for the prompt fix).
+//
+// The bug it guards: on a `rewritten` verdict the model used to paraphrase a
+// precise HOW/WHAT claim ("POST /admin/:op returns 401 via timingSafeEqual")
+// into generic WHY prose ("unified authentication enhances security"), dropping
+// every concrete identifier. The fix teaches the judge to RETAIN every
+// endpoint/status-code/symbol/config on a rewrite (or fall back to `distilled`
+// pass-through when it cannot). So the contract is: a concrete-mechanism claim
+// is acceptably handled EITHER as `rewritten` whose content still carries the
+// source's concrete tokens, OR as `distilled` pass-through (which by definition
+// keeps the original intact) — but NEVER as `restatement` (which would drop it).
+
+import { describe, expect, it } from "vitest";
+
+import { OpenAIDistiller } from "../atlas/llm.js";
+import type { DistillationJudgeInput } from "../atlas/llm.js";
+
+// An admin-ops-style fragment carrying dense concrete verifiable detail: an
+// endpoint route, four HTTP status codes, a named crypto symbol, a config key,
+// and a dropped env-var name. Crucially it is framed as a WHAT-restatement (a
+// "PR #N: unify …" title, terse "adds/validates/returns" body with a single
+// light "so operators manage one credential" why-hook) — that framing INVITES
+// the judge onto the `rewritten` branch instead of `distilled`, which is the
+// exact branch that pre-fix paraphrased every specific away into "unify
+// authentication … enhances security … simplify access control" (the
+// live-observed drop). Verified pre-fix: gpt-4o-mini rules this `rewritten` and
+// drops POST /admin/:op, 401, and trust_proxy on every run.
+const ADMIN_OPS_INPUT: DistillationJudgeInput = {
+  title: "PR #412: unify admin auth on ANALYTICS_TOKEN",
+  content:
+    "Adds a POST /admin/:op endpoint. Validates the ANALYTICS_TOKEN header " +
+    "with timingSafeEqual. Returns 202 on success, 400 on a malformed body, " +
+    "401 on a bad token, 503 when overloaded. Sets trust_proxy and rejects an " +
+    "unresolved forwarded client IP. Removes the old PATHFINDER_ADMIN_TOKEN " +
+    "so operators manage one credential instead of two.",
+  knowledge_type: "security",
+};
+
+// Assert on token PRESENCE, never exact strings — a rewrite is allowed to
+// rephrase the surrounding prose, it just must not DROP the concrete detail.
+function expectSpecificsRetained(content: string): void {
+  expect(content).toContain("POST /admin/:op");
+  expect(content).toContain("timingSafeEqual");
+  expect(content).toMatch(/\b401\b/);
+  expect(content).toContain("trust_proxy");
+}
+
+describe.skipIf(!process.env.OPENAI_API_KEY)(
+  "judgeDistillation REWRITE branch preserves concrete specifics (real LLM)",
+  () => {
+    it("a rewritten verdict RETAINS endpoints/status-codes/symbols/config (or falls back to distilled pass-through)", async () => {
+      // No baseURL → the REAL OpenAI API (honors OPENAI_API_KEY). No `model`
+      // option → the distiller resolves its own unexported DEFAULT_MODEL, exactly
+      // as production does. This deliberately AVOIDS pinning a duplicated model
+      // literal in the test: a duplicated pin could silently drift from the source
+      // default and make the eval exercise a different model than production. By
+      // deferring to the distiller's default we exercise whatever production runs,
+      // with zero drift surface. (temp 0 keeps it as reproducible as a real model
+      // allows.)
+      const distiller = new OpenAIDistiller();
+
+      const verdict = await distiller.judgeDistillation(ADMIN_OPS_INPUT);
+
+      // The semantic contract for a dense concrete-mechanism claim: it is
+      // acceptably handled EITHER as `rewritten` (whose content must retain every
+      // specific) OR as `distilled` pass-through (which by definition keeps the
+      // ORIGINAL content untouched). A `restatement` would DROP the fragment with
+      // no salvage — wrong for this input — so that outcome must fail loud.
+      //
+      // This assertion runs regardless of which verdict the model returns and is
+      // NOT implied by any enclosing guard: if a future regression routed this
+      // claim to `restatement`, it fails here. (Re-checking tokens on the static
+      // ADMIN_OPS_INPUT.content, or asserting `kind === "distilled"` inside a
+      // `kind === "distilled"` branch, would both be tautologies that verify
+      // nothing about the model's output.)
+      expect(["rewritten", "distilled"]).toContain(verdict.kind);
+
+      if (verdict.kind === "rewritten") {
+        // The failure surface: on a rewrite the concrete detail must survive in
+        // the model's OWN returned content (a distilled verdict carries no content
+        // of its own — it is a pure pass-through signal — so there is nothing to
+        // token-check there, and the pass-through preserves the original by
+        // construction).
+        expectSpecificsRetained(verdict.content);
+      }
+    });
+  },
+);

src/atlas/llm.ts

@@ -228,8 +228,8 @@ const DISTILLATION_SYSTEM_PROMPT = `You are a WHY-vs-WHAT judge for an engineeri
 You are given ONE candidate knowledge entry (title + content + knowledge_type). Institutional-memory knowledge must explain the WHY / HOW behind a decision, root cause, architecture choice, or operational reality — NOT merely RESTATE the WHAT that is already obvious from metadata (which PR merged, what a file is named, that a component was added).
 
 Classify the candidate into EXACTLY ONE verdict:
-- "distilled": already a why/how CLAIM (explains reasoning, tradeoffs, mechanism, or consequence). Keep as-is.
-- "rewritten": the SUBSTANCE is salvageable but the current title/content just restates WHAT happened; a why/how claim can be extracted. Provide the rewrite.
+- "distilled": already a why/how CLAIM (explains reasoning, tradeoffs, mechanism, or consequence). Keep as-is. A claim that already states a CONCRETE MECHANISM — specific endpoints/routes, HTTP status codes, error codes, named functions/methods/symbols, file paths, config keys, or specific numbers — is "distilled": keep it as-is; do NOT rewrite a concrete-mechanism claim up into higher-level rationale.
+- "rewritten": the SUBSTANCE is salvageable but the current title/content just restates WHAT happened; a why/how claim can be extracted. Provide the rewrite. The rewrite MUST RETAIN every concrete verifiable detail present in the source — API endpoints/routes, HTTP status codes, error codes, function/method/symbol names, file paths, config keys, and specific numbers. Sharpen the claim by adding the WHY/HOW AROUND those specifics; NEVER drop, generalize, or paraphrase them away. (Concretely: rewriting "POST /admin/:op returns 401 via timingSafeEqual" into "authentication enhances security" is WRONG — the endpoint, the code, and the symbol were all dropped.)
 - "restatement": a PURE WHAT restatement (e.g. "adds X/Y/Z components", "PR #N merged", a stack/component inventory) that carries NO new reasoning or verifiable engineering claim. Cannot be salvaged into a why/how claim from the given text.
 
 Return JSON with EXACTLY this structure:
@@ -243,6 +243,7 @@ Return JSON with EXACTLY this structure:
 Rules:
 - Be conservative about "distilled": if the content only names WHAT (files, components, PRs) with no reasoning, it is NOT distilled.
 - Only choose "rewritten" when the given text ACTUALLY contains extractable why/how substance — do NOT invent reasoning that is not present. If nothing is salvageable, choose "restatement".
+- PRESERVE-SPECIFICS is mandatory on "rewritten": if you cannot produce a rewrite that keeps EVERY identifier/endpoint/status-code/error-code/symbol/path/config-key/number from the source, return "distilled" instead (pass the original through unchanged). Losing a verifiable specific is worse than leaving the prose slightly WHAT-flavored.
 - title/content are REQUIRED for "rewritten" and ignored for the other verdicts.`;
 
 const DISTILL_DELTA_SYSTEM_PROMPT = `You are a knowledge-DELTA distiller for an engineering knowledge corpus.