Skip to content

Commit d07b057

Browse files
kai392RealDiligent
kai392
and
RealDiligent
authored
feat(calibration): pure confusion-matrix scorer for candidate rule classifiers (#8085) (#8098)
Co-authored-by: RealDiligent <brave.challenge007@gmail.com>
1 parent 43b02c9 commit d07b057

4 files changed

Lines changed: 302 additions & 0 deletions

File tree

‎packages/loopover-engine/src/calibration/backtest-score.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Backtest confusion-matrix scorer (#8085) -- replays a caller-supplied candidate classifier over a labeled
2+
// BacktestCase corpus (#8083) and scores it against the real human verdicts, answering "if THIS version of
3+
// the rule had been run against the same targets, would it have gotten more of them right?". Mirrors
4+
// src/review/auto-tune.ts's GateEvalRow confusion-matrix shape (wouldMerge/mergeConfirmed/mergeFalse/
5+
// decided/mergePrecision), but at a backtest-replay grain instead of a live-eval grain.
6+
//
7+
// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads.
8+
9+
import type { BacktestCase } from "./backtest-corpus.js";
10+
11+
// Convention: "reversed" is the positive class. A classifier that correctly predicts a case's real
12+
// label of "reversed" (i.e. correctly identifies that the rule's original firing was WRONG) is a true
13+
// positive. This is a deliberate, non-obvious choice — keep this comment attached to the type.
14+
export type BacktestScoreReport = {
15+
ruleId: string;
16+
caseCount: number;
17+
truePositive: number;
18+
falsePositive: number;
19+
trueNegative: number;
20+
falseNegative: number;
21+
precision: number | null;
22+
recall: number | null;
23+
};
24+
25+
/**
26+
* Score `classify` against every case in `cases` carrying this `ruleId`, accumulating the four
27+
* confusion-matrix counts against the real human labels ("reversed" is the positive class -- see the
28+
* report type's own convention comment). Cases for a different `ruleId` are excluded from every count,
29+
* `caseCount` included -- mirrors computeRulePrecision's (signal-tracking.ts) defensive override filter.
30+
* `precision`/`recall` are null when their denominator is 0, never coerced to 0 or 1 -- the same "unknown
31+
* stays unknown" discipline as RulePrecisionReport.precision. `classify` is deliberately synchronous: every
32+
* case must be scorable without I/O, so a caller can replay thousands of historical cases against a fast,
33+
* in-memory candidate rule implementation.
34+
*/
35+
export function scoreBacktest(
36+
ruleId: string,
37+
cases: readonly BacktestCase[],
38+
classify: (backtestCase: BacktestCase) => "reversed" | "confirmed",
39+
): BacktestScoreReport {
40+
let caseCount = 0;
41+
let truePositive = 0;
42+
let falsePositive = 0;
43+
let trueNegative = 0;
44+
let falseNegative = 0;
45+
for (const backtestCase of cases) {
46+
if (backtestCase.ruleId !== ruleId) continue;
47+
caseCount += 1;
48+
const predicted = classify(backtestCase);
49+
if (predicted === "reversed") {
50+
if (backtestCase.label === "reversed") truePositive += 1;
51+
else falsePositive += 1;
52+
} else if (backtestCase.label === "confirmed") trueNegative += 1;
53+
else falseNegative += 1;
54+
}
55+
return {
56+
ruleId,
57+
caseCount,
58+
truePositive,
59+
falsePositive,
60+
trueNegative,
61+
falseNegative,
62+
precision: truePositive + falsePositive > 0 ? truePositive / (truePositive + falsePositive) : null,
63+
recall: truePositive + falseNegative > 0 ? truePositive / (truePositive + falseNegative) : null,
64+
};
65+
}

‎packages/loopover-engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ export * from "./governor/action-mode.js";
164164
export * from "./governor/chokepoint.js";
165165
export * from "./calibration/signal-tracking.js";
166166
export * from "./calibration/backtest-corpus.js";
167+
export * from "./calibration/backtest-score.js";
167168
export {
168169
GOVERNOR_LEDGER_EVENT_TYPES,
169170
normalizeGovernorLedgerEvent,

‎packages/loopover-engine/test/backtest-score.test.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import assert from "node:assert/strict";
2+
import { test } from "node:test";
3+
4+
import { scoreBacktest, type BacktestCase } from "../dist/index.js";
5+
6+
function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial<BacktestCase> = {}): BacktestCase {
7+
return {
8+
ruleId: "missing_linked_issue",
9+
targetKey,
10+
outcome: "block",
11+
label,
12+
firedAt: "2026-07-22T00:00:00.000Z",
13+
decidedAt: "2026-07-22T01:00:00.000Z",
14+
...overrides,
15+
};
16+
}
17+
18+
test("barrel: the public entrypoint re-exports the backtest scorer (#8085)", () => {
19+
assert.equal(typeof scoreBacktest, "function");
20+
});
21+
22+
test("scoreBacktest: an all-correct classifier scores precision 1 and recall 1", () => {
23+
const cases = [
24+
corpusCase("a#1", "reversed"),
25+
corpusCase("a#2", "confirmed"),
26+
corpusCase("a#3", "reversed"),
27+
];
28+
const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label);
29+
assert.deepEqual(report, {
30+
ruleId: "missing_linked_issue",
31+
caseCount: 3,
32+
truePositive: 2,
33+
falsePositive: 0,
34+
trueNegative: 1,
35+
falseNegative: 0,
36+
precision: 1,
37+
recall: 1,
38+
});
39+
});
40+
41+
test("scoreBacktest: an all-wrong classifier scores precision 0 and recall 0, with the misses in the right cells", () => {
42+
const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")];
43+
const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
44+
backtestCase.label === "reversed" ? "confirmed" : "reversed",
45+
);
46+
assert.deepEqual(report, {
47+
ruleId: "missing_linked_issue",
48+
caseCount: 2,
49+
truePositive: 0,
50+
falsePositive: 1, // predicted reversed on the confirmed-labeled case
51+
trueNegative: 0,
52+
falseNegative: 1, // predicted confirmed on the reversed-labeled case
53+
precision: 0,
54+
recall: 0,
55+
});
56+
});
57+
58+
test("scoreBacktest: a mixed classifier accumulates all four confusion-matrix cells", () => {
59+
const cases = [
60+
corpusCase("a#1", "reversed"), // predicted reversed -> truePositive
61+
corpusCase("a#2", "confirmed"), // predicted reversed -> falsePositive
62+
corpusCase("a#3", "confirmed"), // predicted confirmed -> trueNegative
63+
corpusCase("a#4", "reversed"), // predicted confirmed -> falseNegative
64+
];
65+
const predictReversedFor = new Set(["a#1", "a#2"]);
66+
const report = scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
67+
predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed",
68+
);
69+
assert.deepEqual(report, {
70+
ruleId: "missing_linked_issue",
71+
caseCount: 4,
72+
truePositive: 1,
73+
falsePositive: 1,
74+
trueNegative: 1,
75+
falseNegative: 1,
76+
precision: 0.5,
77+
recall: 0.5,
78+
});
79+
});
80+
81+
test("scoreBacktest: an empty corpus reports zero counts with precision AND recall null", () => {
82+
const report = scoreBacktest("missing_linked_issue", [], () => "reversed");
83+
assert.deepEqual(report, {
84+
ruleId: "missing_linked_issue",
85+
caseCount: 0,
86+
truePositive: 0,
87+
falsePositive: 0,
88+
trueNegative: 0,
89+
falseNegative: 0,
90+
precision: null,
91+
recall: null,
92+
});
93+
});
94+
95+
test("scoreBacktest: precision is null (not 0) when the classifier never predicts reversed, while recall stays real", () => {
96+
const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed");
97+
assert.equal(report.precision, null); // truePositive + falsePositive === 0
98+
assert.equal(report.recall, 0); // truePositive / (0 + 1 falseNegative)
99+
});
100+
101+
test("scoreBacktest: recall is null (not 0) when no case is labeled reversed, while precision stays real", () => {
102+
const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed");
103+
assert.equal(report.recall, null); // truePositive + falseNegative === 0
104+
assert.equal(report.precision, 0); // truePositive / (0 + 1 falsePositive)
105+
});
106+
107+
test("scoreBacktest: cases for a different ruleId are excluded from every count, caseCount included", () => {
108+
const report = scoreBacktest(
109+
"missing_linked_issue",
110+
[corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")],
111+
() => "reversed",
112+
);
113+
assert.deepEqual(report, {
114+
ruleId: "missing_linked_issue",
115+
caseCount: 1,
116+
truePositive: 1,
117+
falsePositive: 0,
118+
trueNegative: 0,
119+
falseNegative: 0,
120+
precision: 1,
121+
recall: 1,
122+
});
123+
});

‎test/unit/backtest-score-engine.test.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import { describe, expect, it } from "vitest";
2+
3+
// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
4+
// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
5+
// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
6+
// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts.
7+
import { scoreBacktest } from "../../packages/loopover-engine/src/calibration/backtest-score";
8+
import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
9+
10+
function corpusCase(targetKey: string, label: BacktestCase["label"], overrides: Partial<BacktestCase> = {}): BacktestCase {
11+
return {
12+
ruleId: "missing_linked_issue",
13+
targetKey,
14+
outcome: "block",
15+
label,
16+
firedAt: "2026-07-22T00:00:00.000Z",
17+
decidedAt: "2026-07-22T01:00:00.000Z",
18+
...overrides,
19+
};
20+
}
21+
22+
describe("scoreBacktest (#8085)", () => {
23+
it("scores an all-correct classifier at precision 1 / recall 1", () => {
24+
const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed"), corpusCase("a#3", "reversed")];
25+
expect(scoreBacktest("missing_linked_issue", cases, (backtestCase) => backtestCase.label)).toEqual({
26+
ruleId: "missing_linked_issue",
27+
caseCount: 3,
28+
truePositive: 2,
29+
falsePositive: 0,
30+
trueNegative: 1,
31+
falseNegative: 0,
32+
precision: 1,
33+
recall: 1,
34+
});
35+
});
36+
37+
it("scores an all-wrong classifier at precision 0 / recall 0 with the misses in the right cells", () => {
38+
const cases = [corpusCase("a#1", "reversed"), corpusCase("a#2", "confirmed")];
39+
expect(
40+
scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
41+
backtestCase.label === "reversed" ? "confirmed" : "reversed",
42+
),
43+
).toEqual({
44+
ruleId: "missing_linked_issue",
45+
caseCount: 2,
46+
truePositive: 0,
47+
falsePositive: 1,
48+
trueNegative: 0,
49+
falseNegative: 1,
50+
precision: 0,
51+
recall: 0,
52+
});
53+
});
54+
55+
it("accumulates all four confusion-matrix cells for a mixed classifier", () => {
56+
const cases = [
57+
corpusCase("a#1", "reversed"),
58+
corpusCase("a#2", "confirmed"),
59+
corpusCase("a#3", "confirmed"),
60+
corpusCase("a#4", "reversed"),
61+
];
62+
const predictReversedFor = new Set(["a#1", "a#2"]);
63+
expect(
64+
scoreBacktest("missing_linked_issue", cases, (backtestCase) =>
65+
predictReversedFor.has(backtestCase.targetKey) ? "reversed" : "confirmed",
66+
),
67+
).toEqual({
68+
ruleId: "missing_linked_issue",
69+
caseCount: 4,
70+
truePositive: 1,
71+
falsePositive: 1,
72+
trueNegative: 1,
73+
falseNegative: 1,
74+
precision: 0.5,
75+
recall: 0.5,
76+
});
77+
});
78+
79+
it("reports zero counts with precision AND recall null for an empty corpus", () => {
80+
expect(scoreBacktest("missing_linked_issue", [], () => "reversed")).toEqual({
81+
ruleId: "missing_linked_issue",
82+
caseCount: 0,
83+
truePositive: 0,
84+
falsePositive: 0,
85+
trueNegative: 0,
86+
falseNegative: 0,
87+
precision: null,
88+
recall: null,
89+
});
90+
});
91+
92+
it("keeps precision null (not 0) when the classifier never predicts reversed, while recall stays real", () => {
93+
const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "reversed")], () => "confirmed");
94+
expect(report.precision).toBeNull();
95+
expect(report.recall).toBe(0);
96+
});
97+
98+
it("keeps recall null (not 0) when no case is labeled reversed, while precision stays real", () => {
99+
const report = scoreBacktest("missing_linked_issue", [corpusCase("a#1", "confirmed")], () => "reversed");
100+
expect(report.recall).toBeNull();
101+
expect(report.precision).toBe(0);
102+
});
103+
104+
it("excludes cases for a different ruleId from every count, caseCount included", () => {
105+
const report = scoreBacktest(
106+
"missing_linked_issue",
107+
[corpusCase("a#1", "reversed", { ruleId: "other_rule" }), corpusCase("a#2", "reversed")],
108+
() => "reversed",
109+
);
110+
expect(report.caseCount).toBe(1);
111+
expect(report.truePositive).toBe(1);
112+
});
113+
});

0 commit comments

Comments
 (0)