feat(engine): add the Pareto-floor comparator for backtest score reports (#8086)

Author: cleanjunc

packages/loopover-engine/src/calibration/backtest-compare.ts

@@ -0,0 +1,43 @@
+// Pareto-floor comparator for backtest score reports (#8086, parent epic #8082) -- applies the no-regression
+// discipline vanguarstew's score_pr_delta.py established (cited for the scoring METHOD only; nothing imported
+// or copied): a candidate that regresses on ANY measured axis is a regression, even while improving another.
+// "Trading one axis for the other" can never read as a net win, so a rule fix can't be gamed by sacrificing
+// recall for precision or vice versa.
+//
+// Pure, like everything in this module: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestScoreReport } from "./backtest-score.js";
+
+export type BacktestComparison = {
+  ruleId: string;
+  baseline: BacktestScoreReport;
+  candidate: BacktestScoreReport;
+  regressedAxes: Array<"precision" | "recall">;
+  improvedAxes: Array<"precision" | "recall">;
+  verdict: "improved" | "regressed" | "unchanged";
+};
+
+/**
+ * Compare a baseline and candidate score for the SAME rule. Per axis (precision, recall): when EITHER value
+ * is null the axis is excluded from both lists (insufficient decided data is not comparable -- null is never
+ * treated as 0 or as "no change"); otherwise strictly-less = regressed, strictly-greater = improved, equal =
+ * neither. The verdict is "regressed" whenever ANY axis regressed -- the Pareto-floor rule, never a
+ * weighted/averaged score -- else "improved" when any axis improved, else "unchanged". Throws when the two
+ * reports carry different ruleIds (a caller bug, not a valid comparison).
+ */
+export function compareBacktestScores(baseline: BacktestScoreReport, candidate: BacktestScoreReport): BacktestComparison {
+  if (baseline.ruleId !== candidate.ruleId) {
+    throw new Error(`cannot compare backtest scores for different rules: ${baseline.ruleId} vs ${candidate.ruleId}`);
+  }
+  const regressedAxes: Array<"precision" | "recall"> = [];
+  const improvedAxes: Array<"precision" | "recall"> = [];
+  for (const axis of ["precision", "recall"] as const) {
+    const baselineValue = baseline[axis];
+    const candidateValue = candidate[axis];
+    if (baselineValue === null || candidateValue === null) continue;
+    if (candidateValue < baselineValue) regressedAxes.push(axis);
+    else if (candidateValue > baselineValue) improvedAxes.push(axis);
+  }
+  const verdict = regressedAxes.length > 0 ? "regressed" : improvedAxes.length > 0 ? "improved" : "unchanged";
+  return { ruleId: baseline.ruleId, baseline, candidate, regressedAxes, improvedAxes, verdict };
+}

packages/loopover-engine/src/index.ts

@@ -165,6 +165,7 @@ export * from "./governor/chokepoint.js";
 export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
+export * from "./calibration/backtest-compare.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-compare.test.ts

@@ -0,0 +1,69 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { compareBacktestScores, type BacktestScoreReport } from "../dist/index.js";
+
+// #8086: the Pareto-floor comparator. The one non-negotiable case: an improvement on one axis NEVER cancels
+// a regression on the other — verdict is "regressed" the moment any axis regressed.
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "rule",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 1,
+    trueNegative: 4,
+    falseNegative: 1,
+    precision: 0.8,
+    recall: 0.8,
+    ...overrides,
+  };
+}
+
+test("both axes improve -> verdict improved with empty regressedAxes", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 }));
+  assert.deepEqual(comparison.regressedAxes, []);
+  assert.deepEqual(comparison.improvedAxes, ["precision", "recall"]);
+  assert.equal(comparison.verdict, "improved");
+});
+
+test("PARETO FLOOR: one axis improves while the other regresses -> verdict regressed", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 }));
+  assert.deepEqual(comparison.improvedAxes, ["precision"]);
+  assert.deepEqual(comparison.regressedAxes, ["recall"]);
+  assert.equal(comparison.verdict, "regressed");
+});
+
+test("a null on either side excludes that axis from both lists — null is never 0 and never 'no change'", () => {
+  const baselineNull = compareBacktestScores(report({ precision: null }), report({ precision: 0.99, recall: 0.9 }));
+  assert.deepEqual(baselineNull.regressedAxes, []);
+  assert.deepEqual(baselineNull.improvedAxes, ["recall"]);
+
+  const candidateNull = compareBacktestScores(report(), report({ recall: null, precision: 0.7 }));
+  assert.deepEqual(candidateNull.regressedAxes, ["precision"]);
+  assert.deepEqual(candidateNull.improvedAxes, []);
+  assert.equal(candidateNull.verdict, "regressed");
+});
+
+test("mismatched ruleId throws, and the message contains both rule IDs", () => {
+  assert.throws(
+    () => compareBacktestScores(report({ ruleId: "rule_a" }), report({ ruleId: "rule_b" })),
+    (error: Error) => error.message.includes("rule_a") && error.message.includes("rule_b"),
+  );
+});
+
+test("all comparable axes equal -> verdict unchanged with both lists empty", () => {
+  const comparison = compareBacktestScores(report(), report());
+  assert.deepEqual(comparison.regressedAxes, []);
+  assert.deepEqual(comparison.improvedAxes, []);
+  assert.equal(comparison.verdict, "unchanged");
+});
+
+test("the comparison carries ruleId and both full reports through", () => {
+  const baseline = report();
+  const candidate = report({ precision: 0.9 });
+  const comparison = compareBacktestScores(baseline, candidate);
+  assert.equal(comparison.ruleId, "rule");
+  assert.equal(comparison.baseline, baseline);
+  assert.equal(comparison.candidate, candidate);
+});

test/unit/backtest-compare.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+// Direct src-path import (not the package barrel, which resolves to dist and is outside vitest's
+// coverage.include) — the same coverage-twin pattern test/unit/backtest-corpus.test.ts established for this
+// module: the engine's own node:test suite runs against dist and is invisible to codecov/patch.
+import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "rule",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 1,
+    trueNegative: 4,
+    falseNegative: 1,
+    precision: 0.8,
+    recall: 0.8,
+    ...overrides,
+  };
+}
+
+describe("compareBacktestScores (#8086)", () => {
+  it("marks both axes improved when both rise, with an improved verdict", () => {
+    const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 }));
+    expect(comparison).toMatchObject({ regressedAxes: [], improvedAxes: ["precision", "recall"], verdict: "improved" });
+  });
+
+  it("PARETO FLOOR: a single regressed axis forces the regressed verdict even when the other axis improved", () => {
+    const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 }));
+    expect(comparison.improvedAxes).toEqual(["precision"]);
+    expect(comparison.regressedAxes).toEqual(["recall"]);
+    expect(comparison.verdict).toBe("regressed");
+  });
+
+  it("excludes an axis from both lists when either side is null — never treated as 0 or as no-change", () => {
+    const baselineNull = compareBacktestScores(report({ precision: null }), report({ precision: 0.99, recall: 0.9 }));
+    expect(baselineNull.regressedAxes).toEqual([]);
+    expect(baselineNull.improvedAxes).toEqual(["recall"]);
+
+    const candidateNull = compareBacktestScores(report(), report({ recall: null, precision: 0.7 }));
+    expect(candidateNull.regressedAxes).toEqual(["precision"]);
+    expect(candidateNull.verdict).toBe("regressed");
+
+    const bothNull = compareBacktestScores(report({ precision: null, recall: null }), report({ precision: null, recall: null }));
+    expect(bothNull.verdict).toBe("unchanged");
+  });
+
+  it("throws on mismatched ruleIds, naming both", () => {
+    expect(() => compareBacktestScores(report({ ruleId: "rule_a" }), report({ ruleId: "rule_b" }))).toThrow(
+      "cannot compare backtest scores for different rules: rule_a vs rule_b",
+    );
+  });
+
+  it("reports unchanged when every comparable axis is equal", () => {
+    expect(compareBacktestScores(report(), report()).verdict).toBe("unchanged");
+  });
+});