feat(calibration): pure counterfactual fixture assembler per the replay contract (#8220) (#8247)

Co-authored-by: RealDiligent <brave.challenge007@gmail.com>

Author: kai392

packages/loopover-engine/src/calibration/counterfactual-fixtures.ts

@@ -0,0 +1,84 @@
+// Counterfactual fixture assembler (#8220, sub-epic phase 2): from a labeled corpus, select and shape
+// exactly the cases that are replayable under the #8219 contract — pure selection, no AI calls, no IO.
+// Implements the contract's fixture/sampling shapes VERBATIM (counterfactual-contract.ts); every exclusion
+// is accounted for in the return shape (the #8139 skipped-case discipline), never silently dropped.
+//
+// PROVENANCE: the #8207/#8170 backfill patches a re-fetched row's metadata with a `rawContextProvenance`
+// tag (RAW_CONTEXT_REFETCH_PROVENANCE in scripts/backfill-calibration-corpus-phase2-core.ts); the live
+// #8129/#8130 capture writers never set that key. Presence of the key is therefore the era discriminator —
+// any tagged row is backfilled context, an untagged replayable row is live-captured.
+//
+// SAMPLING: when the eligible set exceeds the contract's budget, membership is decided by the same
+// content-hash discipline as splitBacktestCorpus — sha256(`${seed}:${targetKey}`), first 8 hex chars as the
+// rank, lowest ranks win — never "the first N", which would bias toward old cases. Selection keeps the
+// corpus's own case order (no shuffle); a rank tie (two firings of the SAME target share a hash) breaks
+// toward the earlier case, deterministically.
+
+import { createHash } from "node:crypto";
+import type { BacktestCase } from "./backtest-corpus.js";
+import {
+  isReplayableCase,
+  type CounterfactualFixture,
+  type CounterfactualSamplingContract,
+  type CounterfactualSkipReason,
+} from "./counterfactual-contract.js";
+
+export type CounterfactualFixtureAssembly = {
+  /** Replayable fixtures in corpus order, at most `contract.maxFixtures` of them. */
+  fixtures: CounterfactualFixture[];
+  /** Why every non-fixture case was excluded — `fixtures.length` plus these counts always sums to the
+   *  input corpus size (pinned by an invariant test). */
+  skipped: Record<CounterfactualSkipReason, number>;
+};
+
+function sampleRank(seed: string, targetKey: string): number {
+  return parseInt(createHash("sha256").update(`${seed}:${targetKey}`).digest("hex").slice(0, 8), 16);
+}
+
+function toFixture(backtestCase: BacktestCase): CounterfactualFixture {
+  // isReplayableCase already guaranteed a non-empty string diff for every case reaching here.
+  const metadata = backtestCase.metadata!;
+  return {
+    fixtureId: backtestCase.targetKey,
+    label: backtestCase.label,
+    boundedInputs: { diff: metadata.diff as string },
+    provenance: "rawContextProvenance" in metadata ? "raw_context_refetch" : "live_capture",
+  };
+}
+
+/**
+ * Assemble the replayable fixture set for one campaign per the #8219 contract: filter to cases carrying
+ * bounded raw context (via the contract's own {@link isReplayableCase}), apply the seeded deterministic
+ * sample when the eligible set exceeds `contract.maxFixtures`, and emit fixtures with era provenance —
+ * with full skip accounting for everything excluded. Deterministic: same corpus + contract ⇒ same fixture
+ * set, byte for byte.
+ */
+export function assembleCounterfactualFixtures(
+  cases: readonly BacktestCase[],
+  contract: CounterfactualSamplingContract,
+): CounterfactualFixtureAssembly {
+  const skipped: Record<CounterfactualSkipReason, number> = { no_raw_context: 0, sampled_out: 0 };
+  const eligible: BacktestCase[] = [];
+  for (const backtestCase of cases) {
+    if (!isReplayableCase(backtestCase)) {
+      skipped.no_raw_context += 1;
+      continue;
+    }
+    eligible.push(backtestCase);
+  }
+
+  let selected = eligible;
+  if (eligible.length > contract.maxFixtures) {
+    const kept = new Set(
+      eligible
+        .map((backtestCase, index) => ({ index, rank: sampleRank(contract.seed, backtestCase.targetKey) }))
+        .sort((a, b) => a.rank - b.rank || a.index - b.index)
+        .slice(0, contract.maxFixtures)
+        .map((entry) => entry.index),
+    );
+    selected = eligible.filter((_, index) => kept.has(index));
+    skipped.sampled_out = eligible.length - selected.length;
+  }
+
+  return { fixtures: selected.map(toFixture), skipped };
+}

packages/loopover-engine/src/index.ts

@@ -167,6 +167,7 @@ export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/repo-corpus-slice.js";
 export * from "./calibration/ams-prediction-corpus.js";
 export * from "./calibration/counterfactual-contract.js";
+export * from "./calibration/counterfactual-fixtures.js";
 export * from "./calibration/backtest-score.js";
 export * from "./calibration/backtest-compare.js";
 export * from "./calibration/backtest-report.js";

packages/loopover-engine/test/counterfactual-fixtures.test.ts

@@ -0,0 +1,49 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  assembleCounterfactualFixtures,
+  COUNTERFACTUAL_SAMPLE_SEED_PREFIX,
+  type BacktestCase,
+} from "../dist/index.js";
+
+const SEED = `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:workspace-suite`;
+
+function replayable(targetKey: string, label: BacktestCase["label"] = "confirmed", extraMetadata: Record<string, unknown> = {}): BacktestCase {
+  return {
+    ruleId: "ai_consensus_defect",
+    targetKey,
+    outcome: "close",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+    metadata: { diff: `@@ diff for ${targetKey}`, ...extraMetadata },
+  };
+}
+
+test("barrel: the public entrypoint re-exports the fixture assembler (#8220)", () => {
+  assert.equal(typeof assembleCounterfactualFixtures, "function");
+});
+
+test("assembler round-trip: eligibility, era provenance, and skip accounting per the #8219 contract", () => {
+  const { fixtures, skipped } = assembleCounterfactualFixtures(
+    [
+      replayable("acme/widgets#1", "reversed"),
+      { ...replayable("acme/widgets#2"), metadata: { confidence: 0.5 } },
+      replayable("acme/widgets#3", "confirmed", { rawContextProvenance: "github_raw_context_refetch" }),
+    ],
+    { seed: SEED, maxFixtures: 10 },
+  );
+  assert.equal(fixtures.length, 2);
+  assert.equal(fixtures[0]!.provenance, "live_capture");
+  assert.equal(fixtures[1]!.provenance, "raw_context_refetch");
+  assert.deepEqual(skipped, { no_raw_context: 1, sampled_out: 0 });
+});
+
+test("seeded sampling is deterministic and accounts every sampled-out case", () => {
+  const cases = Array.from({ length: 25 }, (_, i) => replayable(`acme/widgets#${i + 1}`));
+  const first = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 9 });
+  assert.equal(first.fixtures.length, 9);
+  assert.equal(first.skipped.sampled_out, 16);
+  assert.deepEqual(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 9 }), first);
+});

test/unit/counterfactual-fixtures-engine.test.ts

@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / repo-corpus-engine.test.ts.
+import { assembleCounterfactualFixtures } from "../../packages/loopover-engine/src/calibration/counterfactual-fixtures";
+import { COUNTERFACTUAL_SAMPLE_SEED_PREFIX } from "../../packages/loopover-engine/src/calibration/counterfactual-contract";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+
+const SEED = `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:test-campaign`;
+
+function replayable(targetKey: string, label: BacktestCase["label"] = "confirmed", extraMetadata: Record<string, unknown> = {}): BacktestCase {
+  return {
+    ruleId: "ai_consensus_defect",
+    targetKey,
+    outcome: "close",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+    metadata: { diff: `@@ diff for ${targetKey}`, ...extraMetadata },
+  };
+}
+
+describe("assembleCounterfactualFixtures (#8220)", () => {
+  it("shapes replayable cases into fixtures with era provenance, skipping context-less cases with accounting", () => {
+    const cases: BacktestCase[] = [
+      replayable("acme/widgets#1", "reversed"),
+      { ...replayable("acme/widgets#2"), metadata: { confidence: 0.9 } }, // no diff — not replayable
+      { ...replayable("acme/widgets#3"), metadata: { diff: "" } }, // empty diff — not replayable
+      replayable("acme/widgets#4", "confirmed", { rawContextProvenance: "github_raw_context_refetch" }),
+    ];
+    const { fixtures, skipped } = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 });
+    expect(fixtures).toEqual([
+      {
+        fixtureId: "acme/widgets#1",
+        label: "reversed",
+        boundedInputs: { diff: "@@ diff for acme/widgets#1" },
+        provenance: "live_capture",
+      },
+      {
+        fixtureId: "acme/widgets#4",
+        label: "confirmed",
+        boundedInputs: { diff: "@@ diff for acme/widgets#4" },
+        provenance: "raw_context_refetch",
+      },
+    ]);
+    expect(skipped).toEqual({ no_raw_context: 2, sampled_out: 0 });
+    // Sum invariant: every input case is a fixture or an accounted skip.
+    expect(fixtures.length + skipped.no_raw_context + skipped.sampled_out).toBe(cases.length);
+  });
+
+  it("applies the seeded sample only when the eligible set exceeds the budget, preserving corpus order", () => {
+    const cases = Array.from({ length: 20 }, (_, i) => replayable(`acme/widgets#${i + 1}`));
+    const under = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 20 });
+    expect(under.fixtures).toHaveLength(20);
+    expect(under.skipped.sampled_out).toBe(0);
+
+    const sampled = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 7 });
+    expect(sampled.fixtures).toHaveLength(7);
+    expect(sampled.skipped.sampled_out).toBe(13);
+    // Corpus order is preserved within the sample — fixture ids ascend by original position.
+    const positions = sampled.fixtures.map((fixture) => cases.findIndex((c) => c.targetKey === fixture.fixtureId));
+    expect(positions).toEqual([...positions].sort((a, b) => a - b));
+  });
+
+  it("is deterministic per seed, differs across seeds, and never selects 'the first N'", () => {
+    const cases = Array.from({ length: 30 }, (_, i) => replayable(`acme/widgets#${i + 1}`));
+    const first = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 });
+    expect(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 10 })).toEqual(first);
+
+    const otherSeed = assembleCounterfactualFixtures(cases, { seed: `${SEED}-b`, maxFixtures: 10 });
+    expect(otherSeed.fixtures.map((f) => f.fixtureId)).not.toEqual(first.fixtures.map((f) => f.fixtureId));
+    // Hash-ranked membership, not positional truncation.
+    expect(first.fixtures.map((f) => f.fixtureId)).not.toEqual(cases.slice(0, 10).map((c) => c.targetKey));
+  });
+
+  it("breaks a same-target rank tie toward the earlier case, deterministically", () => {
+    // Two firings of the SAME target share a sample hash — a two-element sort MUST compare exactly that
+    // tied pair, forcing the position tie-break: the earlier firing wins the single slot.
+    const cases = [replayable("acme/widgets#7", "confirmed"), replayable("acme/widgets#7", "reversed")];
+    const { fixtures, skipped } = assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 1 });
+    expect(fixtures).toHaveLength(1);
+    expect(fixtures[0]).toMatchObject({ fixtureId: "acme/widgets#7", label: "confirmed" });
+    expect(skipped.sampled_out).toBe(1);
+    // Reproducible byte-for-byte.
+    expect(assembleCounterfactualFixtures(cases, { seed: SEED, maxFixtures: 1 })).toEqual({ fixtures, skipped });
+  });
+
+  it("returns an empty assembly for an empty corpus", () => {
+    expect(assembleCounterfactualFixtures([], { seed: SEED, maxFixtures: 5 })).toEqual({
+      fixtures: [],
+      skipped: { no_raw_context: 0, sampled_out: 0 },
+    });
+  });
+});