Skip to content

Commit 423a3d1

Browse files
michiot05michiot05
authored
feat(calibration): pure BacktestCase corpus builder from fired/override events (#8093)
Closes #8083. Adds packages/loopover-engine/src/calibration/backtest-corpus.ts with the BacktestCase type and buildBacktestCorpus(ruleId, fired, overrides): pairs each rule firing with the human verdict that decided it (nearest override strictly after the firing, else most recent), excludes undecided firings, mirrors overrideMatchesRule's ruleId filter and computeRulePrecision's doc-comment style. Pure: no IO/DB/env/clock. Adds the barrel export and both the node:test (engine gate) and a vitest src-path test (codecov coverage). Co-authored-by: michiot05 <281539540+michiot05@users.noreply.github.com>
1 parent 321c192 commit 423a3d1

4 files changed

Lines changed: 285 additions & 0 deletions

File tree

‎packages/loopover-engine/src/calibration/backtest-corpus.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// Labeled backtest corpus builder (#8083) -- turns the calibration module's raw fired/override event history
2+
// into a list of concrete "this rule fired against this target, and a human later said it was right/wrong"
3+
// cases, each replayable against a different candidate rule/classifier later (see the parent epic #8082).
4+
//
5+
// SELF-CONTAINED, PURE: no IO, no DB, no env, no wall-clock read, and no imports beyond the existing
6+
// RuleFiredEvent/HumanOverrideEvent types from signal-tracking.ts -- the same storage-agnostic discipline
7+
// that whole module follows. `Date.parse` on the events' own `occurredAt` strings is not a clock read; it is
8+
// pure parsing of caller-supplied data, so the function stays deterministic.
9+
10+
import type { HumanOverrideEvent, RuleFiredEvent } from "./signal-tracking.js";
11+
12+
/** One labeled backtest case: a single rule firing paired with the human verdict that later decided it.
13+
* `outcome` is the firing's own `RuleFiredEvent.outcome`; `label` is the paired `HumanOverrideEvent.verdict`
14+
* (`"reversed"` = the rule was wrong that time, `"confirmed"` = it was right); `firedAt`/`decidedAt` are the
15+
* two events' `occurredAt`. `metadata` carries the firing's own metadata, omitted entirely (never set to
16+
* `undefined`) when the firing has none -- the same optional-property discipline `RuleFiredEvent` uses. */
17+
export type BacktestCase = {
18+
ruleId: string;
19+
targetKey: string;
20+
outcome: string;
21+
label: "reversed" | "confirmed";
22+
firedAt: string;
23+
decidedAt: string;
24+
metadata?: Record<string, unknown>;
25+
};
26+
27+
/**
28+
* Build a labeled {@link BacktestCase} corpus for `ruleId` from its fired + override events. Only events whose
29+
* `ruleId` matches the argument are considered (mirrors `overrideMatchesRule` in signal-tracking.ts:
30+
* `event.ruleId === ruleId`); a caller MAY pass a mixed-rule list without filtering first.
31+
*
32+
* A firing with no matching override (same rule AND same `targetKey`) is EXCLUDED, not emitted as an
33+
* unlabeled case -- the same "only the decided ones count" discipline as {@link computeRulePrecision}.
34+
*
35+
* Pairing when a `targetKey` was fired + judged more than once: each firing takes the override whose
36+
* `occurredAt` is the nearest one STRICTLY AFTER that firing; if no override strictly follows it, the most
37+
* recent override by `occurredAt` is used. Each firing yields at most one case (no duplicates for one firing).
38+
*/
39+
export function buildBacktestCorpus(
40+
ruleId: string,
41+
fired: readonly RuleFiredEvent[],
42+
overrides: readonly HumanOverrideEvent[],
43+
): BacktestCase[] {
44+
// Mirrors overrideMatchesRule's one-line filter (event.ruleId === ruleId) in signal-tracking.ts.
45+
const ruleOverrides = overrides.filter((override) => override.ruleId === ruleId);
46+
const cases: BacktestCase[] = [];
47+
for (const firing of fired) {
48+
if (firing.ruleId !== ruleId) continue;
49+
const candidates = ruleOverrides.filter((override) => override.targetKey === firing.targetKey);
50+
if (candidates.length === 0) continue;
51+
const firedMs = Date.parse(firing.occurredAt);
52+
// candidates ascending by time: the first one strictly after the firing is the nearest-following match;
53+
// when none follows, sorted[last] is the most-recent override overall (the documented fallback).
54+
const sorted = [...candidates].sort((a, b) => Date.parse(a.occurredAt) - Date.parse(b.occurredAt));
55+
const decided = sorted.find((override) => Date.parse(override.occurredAt) > firedMs) ?? sorted[sorted.length - 1]!;
56+
const backtestCase: BacktestCase = {
57+
ruleId,
58+
targetKey: firing.targetKey,
59+
outcome: firing.outcome,
60+
label: decided.verdict,
61+
firedAt: firing.occurredAt,
62+
decidedAt: decided.occurredAt,
63+
};
64+
if (firing.metadata !== undefined) backtestCase.metadata = firing.metadata;
65+
cases.push(backtestCase);
66+
}
67+
return cases;
68+
}

‎packages/loopover-engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ export * from "./governor/kill-switch.js";
163163
export * from "./governor/action-mode.js";
164164
export * from "./governor/chokepoint.js";
165165
export * from "./calibration/signal-tracking.js";
166+
export * from "./calibration/backtest-corpus.js";
166167
export {
167168
GOVERNOR_LEDGER_EVENT_TYPES,
168169
normalizeGovernorLedgerEvent,

‎packages/loopover-engine/test/backtest-corpus.test.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import assert from "node:assert/strict";
2+
import { test } from "node:test";
3+
4+
import { buildBacktestCorpus, type BacktestCase, type HumanOverrideEvent, type RuleFiredEvent } from "../dist/index.js";
5+
6+
const RULE = "missing_linked_issue";
7+
8+
function fired(targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
9+
return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
10+
}
11+
12+
function override(
13+
targetKey: string,
14+
verdict: HumanOverrideEvent["verdict"],
15+
overrides: Partial<HumanOverrideEvent> = {},
16+
): HumanOverrideEvent {
17+
return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
18+
}
19+
20+
test("barrel: the public entrypoint re-exports buildBacktestCorpus (#8083)", () => {
21+
assert.equal(typeof buildBacktestCorpus, "function");
22+
});
23+
24+
test("buildBacktestCorpus: a fired event with no matching override is excluded (only decided cases count)", () => {
25+
const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]);
26+
assert.equal(corpus.length, 1);
27+
assert.equal(corpus[0]!.targetKey, "a#1");
28+
});
29+
30+
test("buildBacktestCorpus: a single fired+override pair produces one correctly-labeled case", () => {
31+
const corpus = buildBacktestCorpus(
32+
RULE,
33+
[fired("a#1", { outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })],
34+
[override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })],
35+
);
36+
assert.deepEqual(corpus, [
37+
{
38+
ruleId: RULE,
39+
targetKey: "a#1",
40+
outcome: "block",
41+
label: "reversed",
42+
firedAt: "2026-07-22T00:00:00.000Z",
43+
decidedAt: "2026-07-22T02:00:00.000Z",
44+
metadata: { pr: 1 },
45+
} satisfies BacktestCase,
46+
]);
47+
});
48+
49+
test("buildBacktestCorpus: metadata is omitted entirely (not undefined) when the fired event has none", () => {
50+
const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]);
51+
assert.equal("metadata" in corpus[0]!, false);
52+
});
53+
54+
test("buildBacktestCorpus: multiple overrides -> the firing pairs with the nearest override strictly after it", () => {
55+
const corpus = buildBacktestCorpus(
56+
RULE,
57+
[fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })],
58+
[
59+
override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
60+
override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
61+
override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }),
62+
],
63+
);
64+
// The 01:00 override is the nearest one strictly after the 00:00 firing -> label "reversed".
65+
assert.equal(corpus.length, 1);
66+
assert.equal(corpus[0]!.label, "reversed");
67+
assert.equal(corpus[0]!.decidedAt, "2026-07-22T01:00:00.000Z");
68+
});
69+
70+
test("buildBacktestCorpus: when no override strictly follows the firing, the most recent override is used", () => {
71+
const corpus = buildBacktestCorpus(
72+
RULE,
73+
[fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
74+
[
75+
override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
76+
override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }),
77+
],
78+
);
79+
// Both overrides precede the 05:00 firing -> fall back to the most recent (04:00, "confirmed").
80+
assert.equal(corpus.length, 1);
81+
assert.equal(corpus[0]!.label, "confirmed");
82+
assert.equal(corpus[0]!.decidedAt, "2026-07-22T04:00:00.000Z");
83+
});
84+
85+
test("buildBacktestCorpus: two firings for the same target each yield their own case (no duplicate for one firing)", () => {
86+
const corpus = buildBacktestCorpus(
87+
RULE,
88+
[fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })],
89+
[
90+
override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
91+
override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
92+
],
93+
);
94+
assert.equal(corpus.length, 2);
95+
assert.deepEqual(corpus.map((c) => c.decidedAt), ["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]);
96+
});
97+
98+
test("buildBacktestCorpus: fired and override events for a different ruleId are ignored", () => {
99+
const corpus = buildBacktestCorpus(
100+
RULE,
101+
[fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }],
102+
[override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }],
103+
);
104+
assert.equal(corpus.length, 1);
105+
assert.equal(corpus[0]!.ruleId, RULE);
106+
assert.equal(corpus[0]!.label, "confirmed");
107+
});
108+
109+
test("buildBacktestCorpus: empty input arrays produce an empty corpus", () => {
110+
assert.deepEqual(buildBacktestCorpus(RULE, [], []), []);
111+
});

‎test/unit/backtest-corpus.test.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import { describe, expect, it } from "vitest";
2+
// Direct src-path import (not the `@loopover/engine` package barrel, which resolves to dist and is NOT in
3+
// vitest's coverage.include): the engine's own node:test suite runs against dist and is invisible to Codecov
4+
// (only review-enrichment has a c8 dist-remap harvest step; the engine has none), so this vitest test is what
5+
// gives packages/loopover-engine/src/calibration/backtest-corpus.ts its codecov/patch coverage. The companion
6+
// packages/loopover-engine/test/backtest-corpus.test.ts is the issue-required node:test that gates the engine
7+
// workspace's own `npm run test`. Vite resolves the `.js` specifier to the sibling `.ts` on disk.
8+
import { buildBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js";
9+
import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js";
10+
import type { HumanOverrideEvent, RuleFiredEvent } from "../../packages/loopover-engine/src/calibration/signal-tracking.js";
11+
12+
const RULE = "missing_linked_issue";
13+
14+
function fired(targetKey: string, overrides: Partial<RuleFiredEvent> = {}): RuleFiredEvent {
15+
return { ruleId: RULE, targetKey, outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z", ...overrides };
16+
}
17+
18+
function override(targetKey: string, verdict: HumanOverrideEvent["verdict"], overrides: Partial<HumanOverrideEvent> = {}): HumanOverrideEvent {
19+
return { ruleId: RULE, targetKey, verdict, occurredAt: "2026-07-22T01:00:00.000Z", ...overrides };
20+
}
21+
22+
describe("buildBacktestCorpus (#8083)", () => {
23+
it("excludes a fired event with no matching override (only decided cases count)", () => {
24+
const corpus = buildBacktestCorpus(RULE, [fired("a#1"), fired("a#2")], [override("a#1", "confirmed")]);
25+
expect(corpus.map((c) => c.targetKey)).toEqual(["a#1"]);
26+
});
27+
28+
it("produces one correctly-labeled case for a single fired+override pair, carrying metadata through", () => {
29+
const corpus = buildBacktestCorpus(
30+
RULE,
31+
[fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z", metadata: { pr: 1 } })],
32+
[override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" })],
33+
);
34+
expect(corpus).toEqual([
35+
{
36+
ruleId: RULE,
37+
targetKey: "a#1",
38+
outcome: "block",
39+
label: "reversed",
40+
firedAt: "2026-07-22T00:00:00.000Z",
41+
decidedAt: "2026-07-22T02:00:00.000Z",
42+
metadata: { pr: 1 },
43+
} satisfies BacktestCase,
44+
]);
45+
});
46+
47+
it("omits metadata entirely (never sets it to undefined) when the fired event has none", () => {
48+
const corpus = buildBacktestCorpus(RULE, [fired("a#1")], [override("a#1", "confirmed")]);
49+
expect("metadata" in corpus[0]!).toBe(false);
50+
});
51+
52+
it("pairs a firing with the nearest override strictly after it when a target was judged multiple times", () => {
53+
const corpus = buildBacktestCorpus(
54+
RULE,
55+
[fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" })],
56+
[
57+
override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
58+
override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
59+
override("a#1", "confirmed", { occurredAt: "2026-07-21T23:00:00.000Z" }),
60+
],
61+
);
62+
expect(corpus).toHaveLength(1);
63+
expect(corpus[0]!.label).toBe("reversed");
64+
expect(corpus[0]!.decidedAt).toBe("2026-07-22T01:00:00.000Z");
65+
});
66+
67+
it("falls back to the most recent override when none strictly follows the firing", () => {
68+
const corpus = buildBacktestCorpus(
69+
RULE,
70+
[fired("a#1", { occurredAt: "2026-07-22T05:00:00.000Z" })],
71+
[
72+
override("a#1", "reversed", { occurredAt: "2026-07-22T02:00:00.000Z" }),
73+
override("a#1", "confirmed", { occurredAt: "2026-07-22T04:00:00.000Z" }),
74+
],
75+
);
76+
expect(corpus[0]!.label).toBe("confirmed");
77+
expect(corpus[0]!.decidedAt).toBe("2026-07-22T04:00:00.000Z");
78+
});
79+
80+
it("gives each of two firings for the same target its own case (no duplicate for one firing)", () => {
81+
const corpus = buildBacktestCorpus(
82+
RULE,
83+
[fired("a#1", { occurredAt: "2026-07-22T00:00:00.000Z" }), fired("a#1", { occurredAt: "2026-07-22T02:30:00.000Z" })],
84+
[
85+
override("a#1", "reversed", { occurredAt: "2026-07-22T01:00:00.000Z" }),
86+
override("a#1", "confirmed", { occurredAt: "2026-07-22T03:00:00.000Z" }),
87+
],
88+
);
89+
expect(corpus.map((c) => c.decidedAt)).toEqual(["2026-07-22T01:00:00.000Z", "2026-07-22T03:00:00.000Z"]);
90+
});
91+
92+
it("ignores fired and override events for a different ruleId", () => {
93+
const corpus = buildBacktestCorpus(
94+
RULE,
95+
[fired("a#1"), { ruleId: "other_rule", targetKey: "a#1", outcome: "block", occurredAt: "2026-07-22T00:00:00.000Z" }],
96+
[override("a#1", "confirmed"), { ruleId: "other_rule", targetKey: "a#1", verdict: "reversed", occurredAt: "2026-07-22T01:00:00.000Z" }],
97+
);
98+
expect(corpus).toHaveLength(1);
99+
expect(corpus[0]!).toMatchObject({ ruleId: RULE, label: "confirmed" });
100+
});
101+
102+
it("returns an empty corpus for empty input arrays", () => {
103+
expect(buildBacktestCorpus(RULE, [], [])).toEqual([]);
104+
});
105+
});

0 commit comments

Comments
 (0)