feat(calibration): pure confusion-matrix scorer for candidate rule classifiers (#8085)

Author: RealDiligent

packages/loopover-engine/src/calibration/backtest-score.ts

@@ -0,0 +1,65 @@
+// Backtest confusion-matrix scorer (#8085) -- replays a caller-supplied candidate classifier over a labeled
+// BacktestCase corpus (#8083) and scores it against the real human verdicts, answering "if THIS version of
+// the rule had been run against the same targets, would it have gotten more of them right?". Mirrors
+// src/review/auto-tune.ts's GateEvalRow confusion-matrix shape (wouldMerge/mergeConfirmed/mergeFalse/
+// decided/mergePrecision), but at a backtest-replay grain instead of a live-eval grain.
+//
+// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestCase } from "./backtest-corpus.js";
+
+// Convention: "reversed" is the positive class. A classifier that correctly predicts a case's real
+// label of "reversed" (i.e. correctly identifies that the rule's original firing was WRONG) is a true
+// positive. This is a deliberate, non-obvious choice — keep this comment attached to the type.
+export type BacktestScoreReport = {
+  ruleId: string;
+  caseCount: number;
+  truePositive: number;
+  falsePositive: number;
+  trueNegative: number;
+  falseNegative: number;
+  precision: number | null;
+  recall: number | null;
+};
+
+/**
+ * Score `classify` against every case in `cases` carrying this `ruleId`, accumulating the four
+ * confusion-matrix counts against the real human labels ("reversed" is the positive class -- see the
+ * report type's own convention comment). Cases for a different `ruleId` are excluded from every count,
+ * `caseCount` included -- mirrors computeRulePrecision's (signal-tracking.ts) defensive override filter.
+ * `precision`/`recall` are null when their denominator is 0, never coerced to 0 or 1 -- the same "unknown
+ * stays unknown" discipline as RulePrecisionReport.precision. `classify` is deliberately synchronous: every
+ * case must be scorable without I/O, so a caller can replay thousands of historical cases against a fast,
+ * in-memory candidate rule implementation.
+ */
+export function scoreBacktest(
+  ruleId: string,
+  cases: readonly BacktestCase[],
+  classify: (backtestCase: BacktestCase) => "reversed" | "confirmed",
+): BacktestScoreReport {
+  let caseCount = 0;
+  let truePositive = 0;
+  let falsePositive = 0;
+  let trueNegative = 0;
+  let falseNegative = 0;
+  for (const backtestCase of cases) {
+    if (backtestCase.ruleId !== ruleId) continue;
+    caseCount += 1;
+    const predicted = classify(backtestCase);
+    if (predicted === "reversed") {
+      if (backtestCase.label === "reversed") truePositive += 1;
+      else falsePositive += 1;
+    } else if (backtestCase.label === "confirmed") trueNegative += 1;
+    else falseNegative += 1;
+  }
+  return {
+    ruleId,
+    caseCount,
+    truePositive,
+    falsePositive,
+    trueNegative,
+    falseNegative,
+    precision: truePositive + falsePositive > 0 ? truePositive / (truePositive + falsePositive) : null,
+    recall: truePositive + falseNegative > 0 ? truePositive / (truePositive + falseNegative) : null,
+  };
+}

packages/loopover-engine/src/index.ts

@@ -164,6 +164,7 @@ export * from "./governor/action-mode.js";
 export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
+export * from "./calibration/backtest-score.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-score.test.ts

@@ -0,0 +1,123 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { scoreBacktest, type BacktestCase } from "../dist/index.js";
+
+function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial<BacktestCase> = {}): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label,
+    firedAt: "2026-07-22T00:00:00.000Z",
+    decidedAt: "2026-07-22T01:00:00.000Z",
+    ...overrides,
+  };
+}
+
+test("barrel: the public entrypoint re-exports the backtest scorer (#8085)", () => {
+  assert.equal(typeof scoreBacktest, "function");
+});
+
+test("scoreBacktest: an all-correct classifier scores precision 1 and recall 1", () => {
+  const cases = [
+    corpusCase("a#1", "reversed"),
+    corpusCase("a#2", "confirmed"),
+    corpusCase("a#3", "reversed"),
+  ];
+  const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label);
+  assert.deepEqual(report, {
+    ruleId: "missing_linked_issue",
+    caseCount: 3,
+    truePositive: 2,
+    falsePositive: 0,
+    trueNegative: 1,
+    falseNegative: 0,
+    precision: 1,
+    recall: 1,
+  });
+});
+
+test("scoreBacktest: an all-wrong classifier scores precision 0 and recall 0, with the misses in the right cells", () => {
+  const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")];
+  const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
+    backtestCase.label === "reversed" ? "confirmed" : "reversed",
+  );
+  assert.deepEqual(report, {
+    ruleId: "missing_linked_issue",
+    caseCount: 2,
+    truePositive: 0,
+    falsePositive: 1, // predicted reversed on the confirmed-labeled case
+    trueNegative: 0,
+    falseNegative: 1, // predicted confirmed on the reversed-labeled case
+    precision: 0,
+    recall: 0,
+  });
+});
+
+test("scoreBacktest: a mixed classifier accumulates all four confusion-matrix cells", () => {
+  const cases = [
+    corpusCase("a#1", "reversed"), // predicted reversed -> truePositive
+    corpusCase("a#2", "confirmed"), // predicted reversed -> falsePositive
+    corpusCase("a#3", "confirmed"), // predicted confirmed -> trueNegative
+    corpusCase("a#4", "reversed"), // predicted confirmed -> falseNegative
+  ];
+  const predictReversedFor = new Set(["a#1", "a#2"]);
+  const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
+    predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed",
+  );
+  assert.deepEqual(report, {
+    ruleId: "missing_linked_issue",
+    caseCount: 4,
+    truePositive: 1,
+    falsePositive: 1,
+    trueNegative: 1,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+  });
+});
+
+test("scoreBacktest: an empty corpus reports zero counts with precision AND recall null", () => {
+  const report = scoreBacktest("missing_linked_issue", [], () => "reversed");
+  assert.deepEqual(report, {
+    ruleId: "missing_linked_issue",
+    caseCount: 0,
+    truePositive: 0,
+    falsePositive: 0,
+    trueNegative: 0,
+    falseNegative: 0,
+    precision: null,
+    recall: null,
+  });
+});
+
+test("scoreBacktest: precision is null (not 0) when the classifier never predicts reversed, while recall stays real", () => {
+  const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed");
+  assert.equal(report.precision, null); // truePositive + falsePositive === 0
+  assert.equal(report.recall, 0); // truePositive / (0 + 1 falseNegative)
+});
+
+test("scoreBacktest: recall is null (not 0) when no case is labeled reversed, while precision stays real", () => {
+  const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed");
+  assert.equal(report.recall, null); // truePositive + falseNegative === 0
+  assert.equal(report.precision, 0); // truePositive / (0 + 1 falsePositive)
+});
+
+test("scoreBacktest: cases for a different ruleId are excluded from every count, caseCount included", () => {
+  const report = scoreBacktest(
+    "missing_linked_issue",
+    [corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")],
+    () => "reversed",
+  );
+  assert.deepEqual(report, {
+    ruleId: "missing_linked_issue",
+    caseCount: 1,
+    truePositive: 1,
+    falsePositive: 0,
+    trueNegative: 0,
+    falseNegative: 0,
+    precision: 1,
+    recall: 1,
+  });
+});

test/unit/backtest-score-engine.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts.
+import { scoreBacktest } from "../../packages/loopover-engine/src/calibration/backtest-score";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+
+function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial<BacktestCase> = {}): BacktestCase {
+  return {
+    ruleId: "missing_linked_issue",
+    targetKey,
+    outcome: "block",
+    label,
+    firedAt: "2026-07-22T00:00:00.000Z",
+    decidedAt: "2026-07-22T01:00:00.000Z",
+    ...overrides,
+  };
+}
+
+describe("scoreBacktest (#8085)", () => {
+  it("scores an all-correct classifier at precision 1 / recall 1", () => {
+    const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed"), corpusCase("a#3", "reversed")];
+    expect(scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label)).toEqual({
+      ruleId: "missing_linked_issue",
+      caseCount: 3,
+      truePositive: 2,
+      falsePositive: 0,
+      trueNegative: 1,
+      falseNegative: 0,
+      precision: 1,
+      recall: 1,
+    });
+  });
+
+  it("scores an all-wrong classifier at precision 0 / recall 0 with the misses in the right cells", () => {
+    const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")];
+    expect(
+      scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
+        backtestCase.label === "reversed" ? "confirmed" : "reversed",
+      ),
+    ).toEqual({
+      ruleId: "missing_linked_issue",
+      caseCount: 2,
+      truePositive: 0,
+      falsePositive: 1,
+      trueNegative: 0,
+      falseNegative: 1,
+      precision: 0,
+      recall: 0,
+    });
+  });
+
+  it("accumulates all four confusion-matrix cells for a mixed classifier", () => {
+    const cases = [
+      corpusCase("a#1", "reversed"),
+      corpusCase("a#2", "confirmed"),
+      corpusCase("a#3", "confirmed"),
+      corpusCase("a#4", "reversed"),
+    ];
+    const predictReversedFor = new Set(["a#1", "a#2"]);
+    expect(
+      scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
+        predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed",
+      ),
+    ).toEqual({
+      ruleId: "missing_linked_issue",
+      caseCount: 4,
+      truePositive: 1,
+      falsePositive: 1,
+      trueNegative: 1,
+      falseNegative: 1,
+      precision: 0.5,
+      recall: 0.5,
+    });
+  });
+
+  it("reports zero counts with precision AND recall null for an empty corpus", () => {
+    expect(scoreBacktest("missing_linked_issue", [], () => "reversed")).toEqual({
+      ruleId: "missing_linked_issue",
+      caseCount: 0,
+      truePositive: 0,
+      falsePositive: 0,
+      trueNegative: 0,
+      falseNegative: 0,
+      precision: null,
+      recall: null,
+    });
+  });
+
+  it("keeps precision null (not 0) when the classifier never predicts reversed, while recall stays real", () => {
+    const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed");
+    expect(report.precision).toBeNull();
+    expect(report.recall).toBe(0);
+  });
+
+  it("keeps recall null (not 0) when no case is labeled reversed, while precision stays real", () => {
+    const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed");
+    expect(report.recall).toBeNull();
+    expect(report.precision).toBe(0);
+  });
+
+  it("excludes cases for a different ruleId from every count, caseCount included", () => {
+    const report = scoreBacktest(
+      "missing_linked_issue",
+      [corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")],
+      () => "reversed",
+    );
+    expect(report.caseCount).toBe(1);
+    expect(report.truePositive).toBe(1);
+  });
+});