feat(calibration): Pareto-floor comparator between two BacktestScoreReports (#8086) (#8108)

Co-authored-by: RealDiligent <brave.challenge007@gmail.com>

Author: kai392

packages/loopover-engine/src/calibration/backtest-compare.ts

@@ -0,0 +1,52 @@
+// Pareto-floor comparator between two BacktestScoreReports (#8086) -- the dual-axis no-regression method:
+// a candidate rule change may not regress on ANY measured axis even while improving another; "trading one
+// axis for the other" is a regression, not a net win. This is deliberately NOT a weighted/averaged score --
+// a single regressed axis decides the verdict, which is the entire point of the floor.
+//
+// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestScoreReport } from "./backtest-score.js";
+
+/** The two comparable axes of a {@link BacktestScoreReport}. */
+type ComparisonAxis = "precision" | "recall";
+
+export type BacktestComparison = {
+  ruleId: string;
+  baseline: BacktestScoreReport;
+  candidate: BacktestScoreReport;
+  regressedAxes: Array<"precision" | "recall">;
+  improvedAxes: Array<"precision" | "recall">;
+  verdict: "improved" | "regressed" | "unchanged";
+};
+
+/**
+ * Compare a candidate rule change's backtest score against its baseline under the Pareto-floor rule: an
+ * axis regresses when the candidate's value is strictly below the baseline's, improves when strictly above,
+ * and is excluded from BOTH lists when either side is null (insufficient decided data is never treated as 0
+ * or as "no change" -- the same "unknown stays unknown" discipline the reports themselves use). The verdict
+ * is "regressed" whenever ANY axis regressed -- even if the other axis improved -- else "improved" when any
+ * axis improved, else "unchanged". Throws when the two reports describe different rules: that is a caller
+ * bug, not a valid comparison.
+ */
+export function compareBacktestScores(baseline: BacktestScoreReport, candidate: BacktestScoreReport): BacktestComparison {
+  if (baseline.ruleId !== candidate.ruleId) {
+    throw new Error(`cannot compare backtest scores for different rules: ${baseline.ruleId} vs ${candidate.ruleId}`);
+  }
+  const regressedAxes: ComparisonAxis[] = [];
+  const improvedAxes: ComparisonAxis[] = [];
+  for (const axis of ["precision", "recall"] as const) {
+    const baselineValue = baseline[axis];
+    const candidateValue = candidate[axis];
+    if (baselineValue === null || candidateValue === null) continue;
+    if (candidateValue < baselineValue) regressedAxes.push(axis);
+    else if (candidateValue > baselineValue) improvedAxes.push(axis);
+  }
+  return {
+    ruleId: baseline.ruleId,
+    baseline,
+    candidate,
+    regressedAxes,
+    improvedAxes,
+    verdict: regressedAxes.length > 0 ? "regressed" : improvedAxes.length > 0 ? "improved" : "unchanged",
+  };
+}

packages/loopover-engine/src/index.ts

@@ -165,6 +165,7 @@ export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
+export * from "./calibration/backtest-compare.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-compare.test.ts

@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { compareBacktestScores, type BacktestScoreReport } from "../dist/index.js";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 2,
+    trueNegative: 3,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+test("barrel: the public entrypoint re-exports the Pareto-floor comparator (#8086)", () => {
+  assert.equal(typeof compareBacktestScores, "function");
+});
+
+test("compareBacktestScores: both axes improving is an improved verdict with empty regressedAxes", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.7, recall: 0.6 }));
+  assert.deepEqual(comparison.improvedAxes, ["precision", "recall"]);
+  assert.deepEqual(comparison.regressedAxes, []);
+  assert.equal(comparison.verdict, "improved");
+});
+
+test("compareBacktestScores: PARETO FLOOR -- one axis improving while the other regresses is a regressed verdict", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.3 }));
+  assert.deepEqual(comparison.improvedAxes, ["precision"]);
+  assert.deepEqual(comparison.regressedAxes, ["recall"]);
+  assert.equal(comparison.verdict, "regressed");
+});
+
+test("compareBacktestScores: an axis with a null on either side is excluded from both lists", () => {
+  const nullBaseline = compareBacktestScores(report({ precision: null }), report({ precision: 0.9, recall: 0.6 }));
+  assert.deepEqual(nullBaseline.improvedAxes, ["recall"]);
+  assert.deepEqual(nullBaseline.regressedAxes, []);
+  assert.equal(nullBaseline.verdict, "improved");
+
+  const nullCandidate = compareBacktestScores(report(), report({ recall: null }));
+  assert.deepEqual(nullCandidate.improvedAxes, []);
+  assert.deepEqual(nullCandidate.regressedAxes, []);
+  assert.equal(nullCandidate.verdict, "unchanged");
+});
+
+test("compareBacktestScores: equal non-null axes land in neither list and yield an unchanged verdict", () => {
+  const comparison = compareBacktestScores(report(), report());
+  assert.deepEqual(comparison.improvedAxes, []);
+  assert.deepEqual(comparison.regressedAxes, []);
+  assert.equal(comparison.verdict, "unchanged");
+  assert.equal(comparison.ruleId, "missing_linked_issue");
+});
+
+test("compareBacktestScores: mismatched ruleIds throw, naming both rules", () => {
+  assert.throws(
+    () => compareBacktestScores(report(), report({ ruleId: "other_rule" })),
+    /cannot compare backtest scores for different rules: missing_linked_issue vs other_rule/,
+  );
+});

test/unit/backtest-compare-engine.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts.
+import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 2,
+    trueNegative: 3,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+describe("compareBacktestScores (#8086)", () => {
+  it("marks both-axes improvement as improved with empty regressedAxes", () => {
+    const comparison = compareBacktestScores(report(), report({ precision: 0.7, recall: 0.6 }));
+    expect(comparison.improvedAxes).toEqual(["precision", "recall"]);
+    expect(comparison.regressedAxes).toEqual([]);
+    expect(comparison.verdict).toBe("improved");
+    expect(comparison.baseline.precision).toBe(0.5);
+    expect(comparison.candidate.precision).toBe(0.7);
+  });
+
+  it("PARETO FLOOR: one axis improving while the other regresses is a regressed verdict", () => {
+    const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.3 }));
+    expect(comparison.improvedAxes).toEqual(["precision"]);
+    expect(comparison.regressedAxes).toEqual(["recall"]);
+    expect(comparison.verdict).toBe("regressed");
+  });
+
+  it("marks a regression on both axes as regressed with empty improvedAxes", () => {
+    const comparison = compareBacktestScores(report(), report({ precision: 0.1, recall: 0.2 }));
+    expect(comparison.regressedAxes).toEqual(["precision", "recall"]);
+    expect(comparison.improvedAxes).toEqual([]);
+    expect(comparison.verdict).toBe("regressed");
+  });
+
+  it("excludes an axis from both lists when either side is null -- null is never 0 and never 'no change'", () => {
+    const nullBaseline = compareBacktestScores(report({ precision: null }), report({ precision: 0.9, recall: 0.6 }));
+    expect(nullBaseline.improvedAxes).toEqual(["recall"]);
+    expect(nullBaseline.regressedAxes).toEqual([]);
+    expect(nullBaseline.verdict).toBe("improved");
+
+    const nullCandidate = compareBacktestScores(report(), report({ recall: null }));
+    expect(nullCandidate.improvedAxes).toEqual([]);
+    expect(nullCandidate.regressedAxes).toEqual([]);
+    expect(nullCandidate.verdict).toBe("unchanged");
+  });
+
+  it("yields unchanged when every comparable axis is equal", () => {
+    const comparison = compareBacktestScores(report(), report());
+    expect(comparison.improvedAxes).toEqual([]);
+    expect(comparison.regressedAxes).toEqual([]);
+    expect(comparison.verdict).toBe("unchanged");
+    expect(comparison.ruleId).toBe("missing_linked_issue");
+  });
+
+  it("throws on mismatched ruleIds, naming both rules in the message", () => {
+    expect(() => compareBacktestScores(report(), report({ ruleId: "other_rule" }))).toThrow(
+      "cannot compare backtest scores for different rules: missing_linked_issue vs other_rule",
+    );
+  });
+});