feat(calibration): replay-derived provider track records adapter (#8278)

Maps the #8221 harness's cached per-fixture verdicts onto ProviderReviewSignal
(would_flag => fail, would_not_flag => pass, abstentions yield NO signal) and
aggregates with computeProviderTrackRecords — the identical function live
reviewer_vote rows feed. Offline report only: nothing persists, and the
rendered table carries the replay-derived disclaimer (#8278's segregation
rule). First same-seed two-provider table recorded on #8229.

Author: JSONbored

scripts/provider-replay-track-record.ts

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+// Replay-derived provider track records (#8278, seeding #8229 stage 1). Reads the counterfactual replay
+// harness's cached raw outputs (the #8221 artifacts dir) for one or more variants over the SAME seeded
+// fixture sample, maps each parsed verdict onto the consensus vote vocabulary, and aggregates with
+// `computeProviderTrackRecords` (#8228) — the identical function live votes will feed, zero new math.
+// REPLAY-DERIVED, NEVER LIVE: signals exist only in this offline report; nothing is written to any store
+// and nothing masquerades as a live reviewer_vote event (the #8278 segregation requirement).
+//
+//   tsx scripts/provider-replay-track-record.ts --fixtures corpus.json --artifacts dir \
+//     --variant <promptVersion@modelSpec> [--variant …] [--seed-suffix s] [--max-fixtures N]
+//
+// Verdict → vote mapping: would_flag ⇒ "fail" (the defect-flagging vote), would_not_flag ⇒ "pass",
+// abstained ⇒ NO signal (an abstention is not a vote — the same never-coerced discipline as scoring).
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  computeProviderTrackRecords,
+  COUNTERFACTUAL_SAMPLE_SEED_PREFIX,
+  type BacktestCase,
+  type CounterfactualVariant,
+  type ProviderReviewSignal,
+  type ProviderTrackRecord,
+} from "@loopover/engine";
+import { artifactKey, parseVariantVerdict, planReplay, type CounterfactualReplayPlan } from "./counterfactual-replay-core.js";
+
+/** PURE: map one variant's cached raw outputs onto provider signals over the shared plan. The provider id
+ *  is the variant's modelSpec (the reviewer identity live votes carry); abstentions and uncached fixtures
+ *  yield no signal, counted separately so the report can say how much of the sample actually voted. */
+export function artifactsToProviderSignals(
+  plan: CounterfactualReplayPlan,
+  variant: CounterfactualVariant,
+  readArtifact: (key: string) => string | null,
+): { signals: ProviderReviewSignal[]; abstained: number; uncached: number } {
+  const signals: ProviderReviewSignal[] = [];
+  let abstained = 0;
+  let uncached = 0;
+  for (const fixture of plan.fixtures) {
+    const raw = readArtifact(artifactKey(variant, fixture.fixtureId));
+    if (raw === null) {
+      uncached += 1;
+      continue;
+    }
+    const verdict = parseVariantVerdict(raw);
+    if (verdict === "abstained") {
+      abstained += 1;
+      continue;
+    }
+    signals.push({
+      provider: variant.modelSpec,
+      repoFullName: fixture.fixtureId.split("#")[0] ?? fixture.fixtureId,
+      targetKey: fixture.fixtureId,
+      vote: verdict === "would_flag" ? "fail" : "pass",
+    });
+  }
+  return { signals, abstained, uncached };
+}
+
+/** Render the overall-rollup rows (repoFullName null) as the markdown table #8229's stage-1 comment quotes. */
+export function renderProviderTable(records: readonly ProviderTrackRecord[]): string {
+  const overall = records.filter((record) => record.repoFullName === null);
+  const lines = [
+    "| Provider | Signals | Decided | Precision (fail⇒confirmed) | Agreement | Consensus |",
+    "| --- | --- | --- | --- | --- | --- |",
+  ];
+  const fmt = (value: number | null): string => (value === null ? "n/a" : value.toFixed(3));
+  for (const record of overall) {
+    lines.push(
+      `| ${record.provider} | ${record.signals} | ${record.decided} | ${fmt(record.precision)} | ${fmt(record.agreementRate)} | ${fmt(record.consensusRate)} |`,
+    );
+  }
+  return lines.join("\n");
+}
+
+function parseArgs(argv: string[]): { fixtures?: string; artifacts: string; variants: string[]; seedSuffix: string; maxFixtures: number } {
+  const args = { fixtures: undefined as string | undefined, artifacts: ".counterfactual-artifacts", variants: [] as string[], seedSuffix: "default", maxFixtures: 500 };
+  for (let i = 0; i < argv.length; i += 1) {
+    const flag = argv[i];
+    if (flag === "--fixtures") args.fixtures = argv[++i];
+    else if (flag === "--artifacts") args.artifacts = argv[++i] ?? args.artifacts;
+    else if (flag === "--variant") args.variants.push(argv[++i] ?? "");
+    else if (flag === "--seed-suffix") args.seedSuffix = argv[++i] ?? "default";
+    else if (flag === "--max-fixtures") args.maxFixtures = Number(argv[++i]);
+  }
+  return args;
+}
+
+async function main(): Promise<number> {
+  const args = parseArgs(process.argv.slice(2));
+  if (!args.fixtures || args.variants.length === 0 || args.variants.some((variant) => !variant.includes("@"))) {
+    console.error("Usage: tsx scripts/provider-replay-track-record.ts --fixtures <corpus.json> --artifacts <dir> --variant <promptVersion@modelSpec> [--variant …]");
+    return 1;
+  }
+  const manifest = JSON.parse(readFileSync(args.fixtures, "utf8")) as { cases?: BacktestCase[] };
+  if (!Array.isArray(manifest.cases)) {
+    console.error("--fixtures file has no cases[] — expected a backtest-corpus-export manifest.");
+    return 1;
+  }
+  const plan = planReplay(manifest.cases, { seed: `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:${args.seedSuffix}`, maxFixtures: args.maxFixtures });
+
+  const allSignals: ProviderReviewSignal[] = [];
+  for (const raw of args.variants) {
+    const [promptVersion, ...modelParts] = raw.split("@");
+    const variant: CounterfactualVariant = { promptVersion: promptVersion!, modelSpec: modelParts.join("@") };
+    const { signals, abstained, uncached } = artifactsToProviderSignals(plan, variant, (key) => {
+      const path = join(args.artifacts, `${key}.txt`);
+      return existsSync(path) ? readFileSync(path, "utf8") : null;
+    });
+    console.log(`${raw}: ${signals.length} signal(s), ${abstained} abstained, ${uncached} uncached (of ${plan.fixtures.length} planned)`);
+    allSignals.push(...signals);
+  }
+
+  console.log("");
+  console.log(renderProviderTable(computeProviderTrackRecords(allSignals, manifest.cases)));
+  console.log("\nREPLAY-DERIVED (offline #8221 artifacts) — not live reviewer votes; nothing was persisted.");
+  return 0;
+}
+
+main().then(
+  (code) => process.exit(code),
+  (error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  },
+);

test/unit/provider-replay-track-record.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import type { BacktestCase } from "@loopover/engine";
+import { computeProviderTrackRecords } from "@loopover/engine";
+import { artifactKey, planReplay } from "../../scripts/counterfactual-replay-core.js";
+import { artifactsToProviderSignals, renderProviderTable } from "../../scripts/provider-replay-track-record.js";
+
+// #8278: the replay→signals adapter. computeProviderTrackRecords has its own engine suite; these tests pin
+// the verdict→vote mapping, the abstention/uncached accounting, and the overall-rollup table rendering.
+
+function corpusCase(id: number, label: "confirmed" | "reversed"): BacktestCase {
+  return {
+    ruleId: "ai_consensus_defect",
+    targetKey: `acme/widgets#${id}`,
+    outcome: "close",
+    label,
+    firedAt: "2026-06-01T00:00:00.000Z",
+    decidedAt: "2026-06-02T00:00:00.000Z",
+    metadata: { diff: "diff --git a/x b/x" },
+  };
+}
+
+const SAMPLING = { seed: "counterfactual-replay-v1:test", maxFixtures: 100 };
+const VARIANT = { promptVersion: "minimal-judge", modelSpec: "qwen3:8b" };
+
+describe("artifactsToProviderSignals (#8278)", () => {
+  it("maps would_flag→fail, would_not_flag→pass; abstentions and uncached fixtures yield NO signal, counted separately", () => {
+    const cases = [corpusCase(1, "reversed"), corpusCase(2, "confirmed"), corpusCase(3, "confirmed"), corpusCase(4, "reversed")];
+    const plan = planReplay(cases, SAMPLING);
+    const byKey: Record<string, string> = {
+      [artifactKey(VARIANT, "acme/widgets#1")]: '{"blockers": ["real"]}',
+      [artifactKey(VARIANT, "acme/widgets#2")]: '{"blockers": []}',
+      [artifactKey(VARIANT, "acme/widgets#3")]: "no json at all", // abstains
+      // #4 uncached
+    };
+    const { signals, abstained, uncached } = artifactsToProviderSignals(plan, VARIANT, (key) => byKey[key] ?? null);
+    expect(abstained).toBe(1);
+    expect(uncached).toBe(1);
+    expect(signals).toEqual([
+      { provider: "qwen3:8b", repoFullName: "acme/widgets", targetKey: "acme/widgets#1", vote: "fail" },
+      { provider: "qwen3:8b", repoFullName: "acme/widgets", targetKey: "acme/widgets#2", vote: "pass" },
+    ]);
+    // The adapter's output feeds the engine aggregation directly: one decided fail on a reversed label.
+    const overall = computeProviderTrackRecords(signals, cases).find((record) => record.repoFullName === null)!;
+    expect(overall.decided).toBe(2);
+    expect(overall.precision).toBe(0); // the lone fail vote landed on a reversed... label "reversed" means the firing was wrong
+  });
+
+  it("renderProviderTable renders only the overall rollups with n/a for null rates", () => {
+    const table = renderProviderTable([
+      { provider: "a", repoFullName: null, signals: 2, decided: 2, confirmed: 1, reversed: 1, precision: 0.5, agreementRate: null, consensusRate: null, splitRate: null },
+      { provider: "a", repoFullName: "acme/widgets", signals: 2, decided: 2, confirmed: 1, reversed: 1, precision: 0.5, agreementRate: 0.5, consensusRate: null, splitRate: null },
+    ]);
+    expect(table).toContain("| a | 2 | 2 | 0.500 | n/a | n/a |");
+    expect(table.split("\n")).toHaveLength(3); // header + divider + ONE overall row (per-repo rows excluded)
+  });
+});