diff --git a/packages/loopover-engine/src/calibration/backtest-compare.ts b/packages/loopover-engine/src/calibration/backtest-compare.ts new file mode 100644 index 0000000000..d91c9cbd1c --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-compare.ts @@ -0,0 +1,43 @@ +// Pareto-floor comparator for backtest score reports (#8086, parent epic #8082) -- applies the no-regression +// discipline vanguarstew's score_pr_delta.py established (cited for the scoring METHOD only; nothing imported +// or copied): a candidate that regresses on ANY measured axis is a regression, even while improving another. +// "Trading one axis for the other" can never read as a net win, so a rule fix can't be gamed by sacrificing +// recall for precision or vice versa. +// +// Pure, like everything in this module: no IO, no randomness, no wall-clock reads. + +import type { BacktestScoreReport } from "./backtest-score.js"; + +export type BacktestComparison = { + ruleId: string; + baseline: BacktestScoreReport; + candidate: BacktestScoreReport; + regressedAxes: Array<"precision" | "recall">; + improvedAxes: Array<"precision" | "recall">; + verdict: "improved" | "regressed" | "unchanged"; +}; + +/** + * Compare a baseline and candidate score for the SAME rule. Per axis (precision, recall): when EITHER value + * is null the axis is excluded from both lists (insufficient decided data is not comparable -- null is never + * treated as 0 or as "no change"); otherwise strictly-less = regressed, strictly-greater = improved, equal = + * neither. The verdict is "regressed" whenever ANY axis regressed -- the Pareto-floor rule, never a + * weighted/averaged score -- else "improved" when any axis improved, else "unchanged". Throws when the two + * reports carry different ruleIds (a caller bug, not a valid comparison). + */ +export function compareBacktestScores(baseline: BacktestScoreReport, candidate: BacktestScoreReport): BacktestComparison { + if (baseline.ruleId !== candidate.ruleId) { + throw new Error(`cannot compare backtest scores for different rules: ${baseline.ruleId} vs ${candidate.ruleId}`); + } + const regressedAxes: Array<"precision" | "recall"> = []; + const improvedAxes: Array<"precision" | "recall"> = []; + for (const axis of ["precision", "recall"] as const) { + const baselineValue = baseline[axis]; + const candidateValue = candidate[axis]; + if (baselineValue === null || candidateValue === null) continue; + if (candidateValue < baselineValue) regressedAxes.push(axis); + else if (candidateValue > baselineValue) improvedAxes.push(axis); + } + const verdict = regressedAxes.length > 0 ? "regressed" : improvedAxes.length > 0 ? "improved" : "unchanged"; + return { ruleId: baseline.ruleId, baseline, candidate, regressedAxes, improvedAxes, verdict }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 6761786a77..51ab2d1714 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -165,6 +165,7 @@ export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; +export * from "./calibration/backtest-compare.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-compare.test.ts b/packages/loopover-engine/test/backtest-compare.test.ts new file mode 100644 index 0000000000..6e48f0df95 --- /dev/null +++ b/packages/loopover-engine/test/backtest-compare.test.ts @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { compareBacktestScores, type BacktestScoreReport } from "../dist/index.js"; + +// #8086: the Pareto-floor comparator. The one non-negotiable case: an improvement on one axis NEVER cancels +// a regression on the other — verdict is "regressed" the moment any axis regressed. + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "rule", + caseCount: 10, + truePositive: 4, + falsePositive: 1, + trueNegative: 4, + falseNegative: 1, + precision: 0.8, + recall: 0.8, + ...overrides, + }; +} + +test("both axes improve -> verdict improved with empty regressedAxes", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 })); + assert.deepEqual(comparison.regressedAxes, []); + assert.deepEqual(comparison.improvedAxes, ["precision", "recall"]); + assert.equal(comparison.verdict, "improved"); +}); + +test("PARETO FLOOR: one axis improves while the other regresses -> verdict regressed", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 })); + assert.deepEqual(comparison.improvedAxes, ["precision"]); + assert.deepEqual(comparison.regressedAxes, ["recall"]); + assert.equal(comparison.verdict, "regressed"); +}); + +test("a null on either side excludes that axis from both lists — null is never 0 and never 'no change'", () => { + const baselineNull = compareBacktestScores(report({ precision: null }), report({ precision: 0.99, recall: 0.9 })); + assert.deepEqual(baselineNull.regressedAxes, []); + assert.deepEqual(baselineNull.improvedAxes, ["recall"]); + + const candidateNull = compareBacktestScores(report(), report({ recall: null, precision: 0.7 })); + assert.deepEqual(candidateNull.regressedAxes, ["precision"]); + assert.deepEqual(candidateNull.improvedAxes, []); + assert.equal(candidateNull.verdict, "regressed"); +}); + +test("mismatched ruleId throws, and the message contains both rule IDs", () => { + assert.throws( + () => compareBacktestScores(report({ ruleId: "rule_a" }), report({ ruleId: "rule_b" })), + (error: Error) => error.message.includes("rule_a") && error.message.includes("rule_b"), + ); +}); + +test("all comparable axes equal -> verdict unchanged with both lists empty", () => { + const comparison = compareBacktestScores(report(), report()); + assert.deepEqual(comparison.regressedAxes, []); + assert.deepEqual(comparison.improvedAxes, []); + assert.equal(comparison.verdict, "unchanged"); +}); + +test("the comparison carries ruleId and both full reports through", () => { + const baseline = report(); + const candidate = report({ precision: 0.9 }); + const comparison = compareBacktestScores(baseline, candidate); + assert.equal(comparison.ruleId, "rule"); + assert.equal(comparison.baseline, baseline); + assert.equal(comparison.candidate, candidate); +}); diff --git a/test/unit/backtest-compare.test.ts b/test/unit/backtest-compare.test.ts new file mode 100644 index 0000000000..2834cb7b22 --- /dev/null +++ b/test/unit/backtest-compare.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import (not the package barrel, which resolves to dist and is outside vitest's +// coverage.include) — the same coverage-twin pattern test/unit/backtest-corpus.test.ts established for this +// module: the engine's own node:test suite runs against dist and is invisible to codecov/patch. +import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "rule", + caseCount: 10, + truePositive: 4, + falsePositive: 1, + trueNegative: 4, + falseNegative: 1, + precision: 0.8, + recall: 0.8, + ...overrides, + }; +} + +describe("compareBacktestScores (#8086)", () => { + it("marks both axes improved when both rise, with an improved verdict", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 })); + expect(comparison).toMatchObject({ regressedAxes: [], improvedAxes: ["precision", "recall"], verdict: "improved" }); + }); + + it("PARETO FLOOR: a single regressed axis forces the regressed verdict even when the other axis improved", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 })); + expect(comparison.improvedAxes).toEqual(["precision"]); + expect(comparison.regressedAxes).toEqual(["recall"]); + expect(comparison.verdict).toBe("regressed"); + }); + + it("excludes an axis from both lists when either side is null — never treated as 0 or as no-change", () => { + const baselineNull = compareBacktestScores(report({ precision: null }), report({ precision: 0.99, recall: 0.9 })); + expect(baselineNull.regressedAxes).toEqual([]); + expect(baselineNull.improvedAxes).toEqual(["recall"]); + + const candidateNull = compareBacktestScores(report(), report({ recall: null, precision: 0.7 })); + expect(candidateNull.regressedAxes).toEqual(["precision"]); + expect(candidateNull.verdict).toBe("regressed"); + + const bothNull = compareBacktestScores(report({ precision: null, recall: null }), report({ precision: null, recall: null })); + expect(bothNull.verdict).toBe("unchanged"); + }); + + it("throws on mismatched ruleIds, naming both", () => { + expect(() => compareBacktestScores(report({ ruleId: "rule_a" }), report({ ruleId: "rule_b" }))).toThrow( + "cannot compare backtest scores for different rules: rule_a vs rule_b", + ); + }); + + it("reports unchanged when every comparable axis is equal", () => { + expect(compareBacktestScores(report(), report()).verdict).toBe("unchanged"); + }); +});