feat(calibration): deterministic seeded held-out/visible split of the backtest corpus (#8087) (#8097)

Co-authored-by: RealDiligent <brave.challenge007@gmail.com>

Author: kai392

packages/loopover-engine/src/calibration/backtest-split.ts

@@ -0,0 +1,40 @@
+// Deterministic held-out/visible corpus split (#8087) -- the dual-target evaluation method: iterate a
+// candidate rule against the visible slice, score it against BOTH slices, so a fix can't be hand-tuned to
+// just the specific incidents already known about. Reuses the same content-hash approach as
+// stableProposalId in ../miner/deny-hook-synthesis.ts (sha256 over a composite key): a case's assignment
+// depends only on (seed, ruleId, targetKey) -- never on its position or on cases.length -- so a corpus
+// that grows over time never reshuffles which already-processed cases were previously held out.
+//
+// Same purity contract as the rest of this module family: no IO, no Math.random(), no wall-clock reads.
+
+import { createHash } from "node:crypto";
+import type { BacktestCase } from "./backtest-corpus.js";
+
+/**
+ * Partition `cases` into a visible slice and a held-out slice of roughly `heldOutFraction` of the corpus.
+ * Deterministic: sha256(`${seed}:${ruleId}:${targetKey}`), first 8 hex chars as a base-16 integer over
+ * 0xffffffff, held out when strictly below `heldOutFraction` -- identical inputs always produce
+ * byte-identical output. Each case keeps its original input-order position within its assigned bucket
+ * (no sorting, no shuffling). Throws when `heldOutFraction` is outside the inclusive [0, 1] range.
+ */
+export function splitBacktestCorpus(
+  cases: readonly BacktestCase[],
+  heldOutFraction: number,
+  seed: string,
+): { visible: BacktestCase[]; heldOut: BacktestCase[] } {
+  // Negated compound form so a NaN fraction also fails closed instead of silently splitting nothing out.
+  if (!(heldOutFraction >= 0 && heldOutFraction <= 1)) {
+    throw new Error(`invalid_held_out_fraction: ${heldOutFraction}`);
+  }
+  const visible: BacktestCase[] = [];
+  const heldOut: BacktestCase[] = [];
+  for (const backtestCase of cases) {
+    const digest = createHash("sha256")
+      .update(`${seed}:${backtestCase.ruleId}:${backtestCase.targetKey}`)
+      .digest("hex");
+    const value = parseInt(digest.slice(0, 8), 16) / 0xffffffff;
+    if (value < heldOutFraction) heldOut.push(backtestCase);
+    else visible.push(backtestCase);
+  }
+  return { visible, heldOut };
+}

packages/loopover-engine/test/backtest-split.test.ts

@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import type { BacktestCase } from "../dist/index.js";
+import { splitBacktestCorpus } from "../dist/calibration/backtest-split.js";
+
+function corpusCase(targetKey: string, overrides: Partial<BacktestCase> = {}): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label: "confirmed",
+    firedAt: "2026-07-22T00:00:00.000Z",
+    decidedAt: "2026-07-22T01:00:00.000Z",
+    ...overrides,
+  };
+}
+
+/** A dozen distinct targets -- enough that a 0.5 split reliably lands cases in BOTH buckets and that two
+ *  different seeds reliably disagree on at least one case, without depending on any specific hash value. */
+const corpus = Array.from({ length: 12 }, (_, index) => corpusCase(`acme/widgets#${index + 1}`));
+
+test("splitBacktestCorpus: heldOutFraction 0 keeps every case visible", () => {
+  const { visible, heldOut } = splitBacktestCorpus(corpus, 0, "seed-a");
+  assert.deepEqual(visible, corpus);
+  assert.deepEqual(heldOut, []);
+});
+
+test("splitBacktestCorpus: heldOutFraction 1 holds every case out", () => {
+  const { visible, heldOut } = splitBacktestCorpus(corpus, 1, "seed-a");
+  assert.deepEqual(heldOut, corpus);
+  assert.deepEqual(visible, []);
+});
+
+test("splitBacktestCorpus: identical inputs produce byte-identical output, including per-bucket order", () => {
+  const first = splitBacktestCorpus(corpus, 0.5, "seed-a");
+  const second = splitBacktestCorpus(corpus, 0.5, "seed-a");
+  assert.deepEqual(second, first);
+});
+
+test("splitBacktestCorpus: preserves each case's original input order within its bucket, with both buckets populated", () => {
+  const { visible, heldOut } = splitBacktestCorpus(corpus, 0.5, "seed-a");
+  assert.ok(visible.length > 0 && heldOut.length > 0, "0.5 over 12 distinct targets must populate both buckets");
+  const inputIndex = (backtestCase: BacktestCase) => corpus.indexOf(backtestCase);
+  for (const bucket of [visible, heldOut]) {
+    const order = bucket.map(inputIndex);
+    assert.deepEqual(order, [...order].sort((a, b) => a - b));
+  }
+});
+
+test("splitBacktestCorpus: a different seed produces a different split for at least one case", () => {
+  const first = splitBacktestCorpus(corpus, 0.5, "seed-a");
+  const second = splitBacktestCorpus(corpus, 0.5, "seed-b");
+  assert.notDeepEqual(
+    { visible: first.visible.map((c) => c.targetKey), heldOut: first.heldOut.map((c) => c.targetKey) },
+    { visible: second.visible.map((c) => c.targetKey), heldOut: second.heldOut.map((c) => c.targetKey) },
+  );
+});
+
+test("splitBacktestCorpus: throws on an out-of-range heldOutFraction in both directions, naming the value", () => {
+  assert.throws(() => splitBacktestCorpus(corpus, -0.1, "seed-a"), /invalid_held_out_fraction: -0\.1/);
+  assert.throws(() => splitBacktestCorpus(corpus, 1.5, "seed-a"), /invalid_held_out_fraction: 1\.5/);
+});

test/unit/backtest-split-engine.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built artifact for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts.
+import { splitBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-split";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+
+function corpusCase(targetKey: string): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label: "confirmed",
+    firedAt: "2026-07-22T00:00:00.000Z",
+    decidedAt: "2026-07-22T01:00:00.000Z",
+  };
+}
+
+// A dozen distinct targets -- enough that a 0.5 split reliably populates BOTH buckets and two different
+// seeds reliably disagree on at least one case, without depending on any specific hash value.
+const corpus = Array.from({ length: 12 }, (_, index) => corpusCase(`acme/widgets#${index + 1}`));
+
+describe("splitBacktestCorpus (#8087)", () => {
+  it("keeps every case visible at fraction 0 and holds every case out at fraction 1", () => {
+    expect(splitBacktestCorpus(corpus, 0, "seed-a")).toEqual({ visible: corpus, heldOut: [] });
+    expect(splitBacktestCorpus(corpus, 1, "seed-a")).toEqual({ visible: [], heldOut: corpus });
+  });
+
+  it("is deterministic: identical inputs produce byte-identical output, including per-bucket order", () => {
+    const first = splitBacktestCorpus(corpus, 0.5, "seed-a");
+    expect(splitBacktestCorpus(corpus, 0.5, "seed-a")).toEqual(first);
+  });
+
+  it("preserves original input order within each bucket, with both buckets populated at 0.5", () => {
+    const { visible, heldOut } = splitBacktestCorpus(corpus, 0.5, "seed-a");
+    expect(visible.length).toBeGreaterThan(0);
+    expect(heldOut.length).toBeGreaterThan(0);
+    for (const bucket of [visible, heldOut]) {
+      const order = bucket.map((backtestCase) => corpus.indexOf(backtestCase));
+      expect(order).toEqual([...order].sort((a, b) => a - b));
+    }
+  });
+
+  it("produces a different split for at least one case when only the seed changes", () => {
+    const withSeedA = splitBacktestCorpus(corpus, 0.5, "seed-a");
+    const withSeedB = splitBacktestCorpus(corpus, 0.5, "seed-b");
+    expect(withSeedB.heldOut.map((c) => c.targetKey)).not.toEqual(withSeedA.heldOut.map((c) => c.targetKey));
+  });
+
+  it("throws on an out-of-range heldOutFraction in both directions, naming the invalid value", () => {
+    expect(() => splitBacktestCorpus(corpus, -0.1, "seed-a")).toThrow("invalid_held_out_fraction: -0.1");
+    expect(() => splitBacktestCorpus(corpus, 1.5, "seed-a")).toThrow("invalid_held_out_fraction: 1.5");
+  });
+});