diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index f36bf38819..38bfe1f656 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -80,6 +80,26 @@ npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.jso Both CLIs are strictly read-only against the database. +## Counterfactual replay (design) + +Deterministic rule changes replay against history before they land; the same discipline is being +extended to the judgment layer — reviewer prompts and model choices — under the counterfactual-replay +contract (`@loopover/engine`'s `counterfactual-contract` module). Three decisions define it: + +- **Fixtures.** A historical case is replayable only when it carries the bounded raw context the live + capture recorded (the PR diff) and a human verdict. Fixtures keep their capture-era provenance + (live-captured vs backfilled), so a variant that only improves on one era is visible as such. When a + run cannot afford every eligible fixture, the subset is chosen by a seeded deterministic sample — + recorded in the run artifact so any re-run reproduces it exactly — never "the first N". +- **Spend.** Replay budgets use the same neuron-estimate units as live AI budgeting, with a hard per-run + cap defaulting well under one day's live-review spend. A run that exhausts its budget persists partial + scores and a resume cursor. Local models are the smoke path; paid providers sit behind explicit flags. + The harness is offline-only: it never posts to GitHub and never touches live reviews. +- **Scoring.** A replayed variant is just another classifier over the same labeled cases, so the standard + scorer and Pareto-floored comparator apply unchanged. Unparseable variant output is an *abstention* — + excluded from the confusion matrix and counted separately, never coerced to a verdict, so a degenerate + prompt cannot farm precision by failing to answer on hard cases. + ## Self-hosting On a self-host deployment the same calibration system runs against your Postgres instead of D1 — diff --git a/packages/loopover-engine/src/calibration/counterfactual-contract.ts b/packages/loopover-engine/src/calibration/counterfactual-contract.ts new file mode 100644 index 0000000000..769c872d1f --- /dev/null +++ b/packages/loopover-engine/src/calibration/counterfactual-contract.ts @@ -0,0 +1,91 @@ +// Counterfactual replay CONTRACT (#8219, sub-epic #8218, epic #8211 track C). This module is the design +// deliverable the later sub-issues implement against: the fixture shape, the sampling rule, the spend +// contract, and the scoring mapping — types + constants only, no logic (the pure assembler and the harness +// are their own issues and MUST cite these shapes rather than invent parallels). +// +// THE THREE CONTRACTS, decided here: +// +// 1) FIXTURES. A corpus case is replayable iff it carries bounded raw context (`metadata.diff`, the field +// the live #8130 capture and the #8170 re-fetch both write) AND a human label. Fixtures carry +// provenance (live-captured vs backfilled raw context) so results can be segmented by capture era — +// a prompt that only improves on backfilled-era cases is a red flag, not a win. When the eligible set +// exceeds a run's budget, sampling is SEEDED and deterministic (same seed + set ⇒ same sample; the +// splitBacktestCorpus hashing discipline) — never "the first N", which would bias toward old cases. +// +// 2) SPEND. Budgets are expressed in the same neuron-estimate terms the live AI budgeting uses +// (estimateNeurons in linked-issue-satisfaction-run.ts), with a hard per-run cap; a run that exhausts +// mid-set persists partial scores + a cursor and resumes (the #8170 budget/cursor discipline). Provider +// order: local ollama first for smoke runs, BYOK providers only behind explicit flags. The harness +// NEVER posts to GitHub and never touches live reviews. +// +// 3) SCORING. A replayed variant is just another classifier over BacktestCase inputs: its output maps to +// a binary would-flag verdict, so `scoreBacktest`/`compareBacktestScores` apply UNCHANGED. Unparseable +// output is an ABSTENTION: skipped and counted, never coerced to either verdict — coercion would let a +// degenerate prompt farm precision by being unparseable on hard cases. + +import type { BacktestCase } from "./backtest-corpus.js"; + +/** Deterministic sampling seed namespace — one seed string per evaluation campaign, recorded in every + * result artifact so a re-run reproduces the exact fixture subset. */ +export const COUNTERFACTUAL_SAMPLE_SEED_PREFIX = "counterfactual-replay-v1"; + +/** Default per-run spend cap in neuron-estimate units. Chosen to bound a full 460-fixture campaign on the + * current corpus to roughly the cost ceiling of ONE day's live-review budget — a replay campaign must + * never outspend the production feature it evaluates. The harness treats this as a default, not a limit: + * operators override per run, but never implicitly. */ +export const COUNTERFACTUAL_DEFAULT_NEURON_BUDGET = 250_000; + +/** One replayable historical judgment: the bounded inputs a judge variant sees, and the label it is + * scored against. `boundedInputs.diff` is already capped at capture time (RAW_CONTEXT_MAX_DIFF_CHARS). */ +export type CounterfactualFixture = { + /** Stable fixture id — the corpus targetKey. Stays in ARTIFACTS ONLY; never in public surfaces. */ + fixtureId: string; + label: "confirmed" | "reversed"; + boundedInputs: { + diff: string; + }; + /** Which capture era produced the raw context: the live writers or the #8170 re-fetch. */ + provenance: "live_capture" | "raw_context_refetch"; +}; + +/** The deterministic selection contract the assembler (#8220) implements. */ +export type CounterfactualSamplingContract = { + /** Campaign seed, prefixed with {@link COUNTERFACTUAL_SAMPLE_SEED_PREFIX}. */ + seed: string; + /** Hard cap on fixtures per run; the seeded sample applies only when the eligible set exceeds it. */ + maxFixtures: number; +}; + +/** Why a corpus case was excluded from the fixture set — skip accounting is part of the contract so a + * fixture set's composition is always explainable (mirrors the #8139 skipped-case discipline). */ +export type CounterfactualSkipReason = "no_raw_context" | "sampled_out"; + +/** A judge variant under evaluation. `promptVersion` and `modelSpec` are opaque identifiers recorded in + * artifacts; the harness maps them to real providers/config. */ +export type CounterfactualVariant = { + promptVersion: string; + modelSpec: string; +}; + +/** The scoring mapping: what a variant's raw output must reduce to per fixture. `abstained` fixtures are + * excluded from the confusion matrix and reported as their own count — never coerced. */ +export type CounterfactualVerdict = "would_flag" | "would_not_flag" | "abstained"; + +/** The per-run result envelope the harness persists (artifacts dir, never committed, never posted). + * `scored`/`abstained`/`skipped` must sum to the campaign's fixture universe for the run to be valid. */ +export type CounterfactualRunSummary = { + variant: CounterfactualVariant; + sampling: CounterfactualSamplingContract; + scored: number; + abstained: number; + skipped: Record; + neuronsSpent: number; + /** Present when the run exhausted its budget mid-set — the resume point (fixtureId ordering). */ + resumeFrom: string | null; +}; + +/** Narrowing helper the assembler and harness share: a case is replayable iff it carries the bounded + * diff AND a label (every BacktestCase has a label by construction; the diff is the variable part). */ +export function isReplayableCase(backtestCase: BacktestCase): boolean { + return typeof backtestCase.metadata?.diff === "string" && backtestCase.metadata.diff !== ""; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 5405f932e6..beef2399fc 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; +export * from "./calibration/counterfactual-contract.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; export * from "./calibration/backtest-report.js"; diff --git a/test/unit/counterfactual-contract.test.ts b/test/unit/counterfactual-contract.test.ts new file mode 100644 index 0000000000..2ac8a5d4b9 --- /dev/null +++ b/test/unit/counterfactual-contract.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import per the engine blind-spot rule (the barrel resolves to dist, outside Codecov). +import { + COUNTERFACTUAL_DEFAULT_NEURON_BUDGET, + COUNTERFACTUAL_SAMPLE_SEED_PREFIX, + isReplayableCase, +} from "../../packages/loopover-engine/src/calibration/counterfactual-contract.js"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus.js"; + +// #8219: the contract module is deliberately types + constants + one shared narrowing helper. These tests +// pin the constants the later sub-issues (#8220/#8221) cite and the replayability rule's every arm. + +function backtestCase(metadata?: Record): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey: "acme/widgets#7", + outcome: "unaddressed", + label: "confirmed", + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + ...(metadata !== undefined ? { metadata } : {}), + }; +} + +describe("counterfactual contract (#8219)", () => { + it("pins the campaign seed prefix and the default neuron budget the harness must treat as default-not-limit", () => { + expect(COUNTERFACTUAL_SAMPLE_SEED_PREFIX).toBe("counterfactual-replay-v1"); + expect(COUNTERFACTUAL_DEFAULT_NEURON_BUDGET).toBe(250_000); + }); + + it("isReplayableCase: bounded diff present and non-empty is the whole rule — every arm", () => { + expect(isReplayableCase(backtestCase({ diff: "diff --git a/x b/x" }))).toBe(true); + expect(isReplayableCase(backtestCase({ diff: "" }))).toBe(false); // empty diff never replays + expect(isReplayableCase(backtestCase({ rawSignal: "detail only" }))).toBe(false); // wrong context kind + expect(isReplayableCase(backtestCase({ diff: 42 }))).toBe(false); // non-string never replays + expect(isReplayableCase(backtestCase())).toBe(false); // no metadata at all + }); +});