Skip to content

Commit 1f15694

Browse files
JSONboredJSONbored
authored
feat(calibration): counterfactual prompt replay — offline harness + budget-gated advisory CI check (#8221, #8222) (#8271)
Closes the #8218 sub-epic's phases 3+4 against the #8219 contract verbatim: - scripts/counterfactual-replay-core.ts (pure, unit-tested): abstention-safe variant-output parsing (never coerced), scoreBacktest mapping over the #8220 fixture plan, budget arithmetic + resume cursor, run-accounting invariant, artifact cache keys, Pareto baseline comparison - scripts/counterfactual-replay.ts (thin IO): ollama smoke path only (BYOK refused loudly), raw outputs cached under a local artifacts dir so re-scoring is free, --baseline prints the shared renderer's comparison; exit reflects operational success only, never verdict - src/services/ai-review.ts: REVIEW_PROMPT_VERSION + buildCanonicalJudgePrompt (the pure, input-free judge prompt the dual-checkout replay diffs) - scripts/print-review-prompt.ts: the #8139 dual-checkout extractor - .github/workflows/counterfactual-replay-check.yml: path-filtered on the judge-prompt surface, hard-gated on an explicitly configured replay budget (visible skipped-notice otherwise), byte-identical prompts short-circuit, fail-open on every step, update-in-place marker comment, persisted calibration.counterfactual_backtest_run events feeding the SAME REGRESSED-verdict track record as the threshold/logic backtests - docs: worked ollama example + the prompt-change workflow
1 parent 211f32d commit 1f15694

8 files changed

Lines changed: 733 additions & 3 deletions

File tree

‎.github/workflows/counterfactual-replay-check.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
# Counterfactual prompt replay (#8222, sub-epic #8218, epic #8211 track C). When a PR touches the reviewer
2+
# judge-prompt surface, this job replays the CANONICAL judge prompt from BOTH the PR's head and base
3+
# checkouts (scripts/print-review-prompt.ts — the #8139 dual-checkout mechanism) against the recorded
4+
# raw-context fixture corpus, and posts the Pareto-floored comparison as its own advisory comment. Mirrors
5+
# backtest-logic-check.yml's posture end-to-end: separate workflow (untouched PRs pay nothing), advisory
6+
# only (#8105 — never a required check, never blocks merge), every step fails OPEN (notice + green; the
7+
# review engine auto-closes contributor PRs on ANY red check, so this job's own plumbing must never redden).
8+
#
9+
# SPEND GUARD (#8222's requirement): the replay costs real model inference, so the whole run is gated on
10+
# the deployment explicitly configuring a budget — the COUNTERFACTUAL_REPLAY_BUDGET repo variable (neuron
11+
# units, see COUNTERFACTUAL_DEFAULT_NEURON_BUDGET) plus the provider endpoint/model below. Unset (the
12+
# default), the job posts a visible "skipped: no replay budget configured" notice and does nothing else.
13+
name: counterfactual-replay
14+
15+
on:
16+
pull_request:
17+
types: [opened, synchronize, reopened, ready_for_review]
18+
# Exactly the judge-prompt surface: REVIEW_PROMPT_VERSION + buildSystemPrompt/parseModelReview live
19+
# here. Keep in sync with #8222's spec.
20+
paths:
21+
- "src/services/ai-review.ts"
22+
23+
permissions:
24+
contents: read
25+
pull-requests: write
26+
27+
concurrency:
28+
group: counterfactual-replay-${{ github.ref }}
29+
cancel-in-progress: true
30+
31+
jobs:
32+
replay:
33+
name: counterfactual replay (advisory)
34+
if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }}
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 20
37+
steps:
38+
# The budget gate comes FIRST so an unconfigured deployment pays one echo, not an npm ci.
39+
- name: Check replay budget configuration
40+
id: budget
41+
env:
42+
REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }}
43+
REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }}
44+
REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
45+
run: |
46+
if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ] && [ -n "$REPLAY_PROVIDER_URL" ]; then
47+
echo "configured=true" >> "$GITHUB_OUTPUT"
48+
else
49+
echo "configured=false" >> "$GITHUB_OUTPUT"
50+
echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables and the COUNTERFACTUAL_OLLAMA_URL secret to enable). Advisory only, never fails the PR."
51+
fi
52+
53+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
54+
if: ${{ steps.budget.outputs.configured == 'true' }}
55+
with:
56+
persist-credentials: false
57+
58+
# The PR's base commit inside the head workspace — base-side dynamic imports resolve bare specifiers
59+
# by walking up into the head checkout's node_modules, so one npm ci serves both sides (#8139).
60+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
61+
if: ${{ steps.budget.outputs.configured == 'true' }}
62+
with:
63+
ref: ${{ github.event.pull_request.base.sha }}
64+
path: .replay-base
65+
persist-credentials: false
66+
67+
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
68+
if: ${{ steps.budget.outputs.configured == 'true' }}
69+
with:
70+
node-version-file: .nvmrc
71+
cache: "npm"
72+
73+
- name: Install deps
74+
if: ${{ steps.budget.outputs.configured == 'true' }}
75+
run: npm ci --ignore-scripts
76+
77+
- name: Build engine package
78+
if: ${{ steps.budget.outputs.configured == 'true' }}
79+
run: npx turbo run build --filter=@loopover/engine
80+
81+
# Everything below fails OPEN — notice + green, never a red check (#8105 held against our own infra).
82+
- name: Extract base/head canonical prompts
83+
if: ${{ steps.budget.outputs.configured == 'true' }}
84+
id: prompts
85+
run: |
86+
if npx tsx scripts/print-review-prompt.ts --root . > head-prompt.txt \
87+
&& npx tsx scripts/print-review-prompt.ts --root .replay-base > base-prompt.txt \
88+
&& npx tsx scripts/print-review-prompt.ts --root . --version-only > head-version.txt \
89+
&& npx tsx scripts/print-review-prompt.ts --root .replay-base --version-only > base-version.txt; then
90+
if cmp -s head-prompt.txt base-prompt.txt; then
91+
echo "changed=false" >> "$GITHUB_OUTPUT"
92+
echo "::notice::Judge prompt surface touched but the canonical prompt text is byte-identical across base/head — nothing to replay."
93+
else
94+
echo "changed=true" >> "$GITHUB_OUTPUT"
95+
fi
96+
else
97+
echo "changed=false" >> "$GITHUB_OUTPUT"
98+
echo "::notice::Prompt extraction failed — replay skipped. Advisory only, never fails the PR."
99+
fi
100+
101+
- name: Export corpus from D1
102+
if: ${{ steps.budget.outputs.configured == 'true' && steps.prompts.outputs.changed == 'true' }}
103+
id: corpus
104+
env:
105+
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
106+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
107+
run: |
108+
if npx tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output replay-corpus.json --remote; then
109+
echo "available=true" >> "$GITHUB_OUTPUT"
110+
else
111+
echo "available=false" >> "$GITHUB_OUTPUT"
112+
echo "::notice::Corpus export from D1 failed — replay skipped. Advisory only, never fails the PR."
113+
fi
114+
115+
# Two runs over the IDENTICAL deterministic sample (seed = head SHA, per the design contract's
116+
# per-PR determinism requirement): base prompt first (the baseline artifact), then head with
117+
# --baseline (comparison + comment + persisted run event).
118+
- name: Replay base and head prompts
119+
if: ${{ steps.corpus.outputs.available == 'true' }}
120+
id: replay
121+
env:
122+
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
123+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
124+
REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }}
125+
REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }}
126+
REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
127+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
128+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
129+
PR_NUMBER: ${{ github.event.pull_request.number }}
130+
run: |
131+
BASE_VERSION=$(cat base-version.txt)
132+
HEAD_VERSION=$(cat head-version.txt)
133+
if npx tsx scripts/counterfactual-replay.ts \
134+
--fixtures replay-corpus.json \
135+
--variant "${BASE_VERSION}@${REPLAY_MODEL}" \
136+
--prompt-file base-prompt.txt \
137+
--budget "$REPLAY_BUDGET" \
138+
--seed-suffix "$HEAD_SHA" \
139+
--ollama-url "$REPLAY_PROVIDER_URL" \
140+
--out base-scores.json \
141+
&& npx tsx scripts/counterfactual-replay.ts \
142+
--fixtures replay-corpus.json \
143+
--variant "${HEAD_VERSION}@${REPLAY_MODEL}" \
144+
--prompt-file head-prompt.txt \
145+
--budget "$REPLAY_BUDGET" \
146+
--seed-suffix "$HEAD_SHA" \
147+
--ollama-url "$REPLAY_PROVIDER_URL" \
148+
--baseline base-scores.json \
149+
--base-variant-label "$BASE_VERSION" \
150+
--comment-out replay-comment.md \
151+
--head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \
152+
--persist --remote --db loopover \
153+
--repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER"; then
154+
echo "ready=true" >> "$GITHUB_OUTPUT"
155+
else
156+
echo "ready=false" >> "$GITHUB_OUTPUT"
157+
echo "::notice::Counterfactual replay failed — no comparison produced. Advisory only, never fails the PR."
158+
fi
159+
160+
- name: Post or update the PR comment
161+
if: ${{ steps.replay.outputs.ready == 'true' }}
162+
env:
163+
GH_TOKEN: ${{ github.token }}
164+
PR_NUMBER: ${{ github.event.pull_request.number }}
165+
run: |
166+
post_comment() {
167+
marker="<!-- loopover-counterfactual-backtest -->"
168+
comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \
169+
--jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1)
170+
if [ -n "$comment_id" ]; then
171+
gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@replay-comment.md
172+
else
173+
gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@replay-comment.md
174+
fi
175+
}
176+
if ! post_comment; then
177+
echo "::notice::PR comment post failed — replay computed and persisted but not posted. Advisory only, never fails the PR."
178+
fi
179+
180+
# Fork half of the paired convention: fork runs get no secrets, so the replay cannot run — say so.
181+
fork-notice:
182+
name: counterfactual replay (skipped for fork PRs)
183+
if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }}
184+
runs-on: ubuntu-latest
185+
timeout-minutes: 5
186+
steps:
187+
- name: Explain the skip
188+
run: echo "::notice::Fork PR — repo secrets are withheld, so the counterfactual replay is skipped. Advisory only; nothing blocks."

‎apps/loopover-ui/content/docs/backtest-calibration.mdx‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,41 @@ contract (`@loopover/engine`'s `counterfactual-contract` module). Three decision
162162
excluded from the confusion matrix and counted separately, never coerced to a verdict, so a degenerate
163163
prompt cannot farm precision by failing to answer on hard cases.
164164

165+
### The replay harness
166+
167+
The contract is implemented by `scripts/counterfactual-replay.ts` (pure core in
168+
`counterfactual-replay-core.ts`). A worked local example against an ollama model:
169+
170+
```bash
171+
# 1. Export the labeled corpus (raw-context diffs included) to a manifest:
172+
tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output corpus.json --remote
173+
174+
# 2. Score the current prompt as the baseline:
175+
tsx scripts/counterfactual-replay.ts --fixtures corpus.json \
176+
--variant review-prompt-v1@llama3.1:8b --out baseline.json
177+
178+
# 3. Score a candidate prompt against the SAME deterministic sample and print the Pareto comparison:
179+
tsx scripts/counterfactual-replay.ts --fixtures corpus.json \
180+
--variant candidate@llama3.1:8b --prompt-file candidate-prompt.txt \
181+
--baseline baseline.json
182+
```
183+
184+
Raw model outputs cache under `.counterfactual-artifacts/` (never committed), so re-scoring after a
185+
parser or mapping change costs nothing. Exit codes reflect operational success only — never a verdict.
186+
187+
### The prompt-change workflow
188+
189+
A PR that touches the reviewer judge-prompt surface (`src/services/ai-review.ts` — bump
190+
`REVIEW_PROMPT_VERSION` whenever a change shapes the judge's verdict) triggers the
191+
`counterfactual-replay` advisory workflow: it extracts the **canonical judge prompt** from both the
192+
base and head checkouts, replays both over the identical seeded sample (seed = head SHA, so re-runs
193+
per push are deterministic), posts the comparison as an update-in-place PR comment, and persists a
194+
`calibration.counterfactual_backtest_run` event feeding the same REGRESSED-verdict track record as the
195+
threshold and logic backtests. The check spends nothing unless the deployment explicitly configures a
196+
replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL` plus the
197+
provider endpoint secret) — otherwise it posts a visible "skipped: no replay budget configured" notice.
198+
Advisory forever until a separate, explicit authority decision, exactly like every other backtest gate.
199+
165200
## Self-hosting
166201

167202
On a self-host deployment the same calibration system runs against your Postgres instead of D1 —

‎scripts/backtest-track-record.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#!/usr/bin/env node
22
// Read-only D1 → REGRESSED-verdict track-record summary (#8140, epic #8082). Reads the BacktestComparison
3-
// results the advisory backtests persist — #8138's ORB-native threshold runs AND #8139's CI-side logic
4-
// runs, sibling event types with the same metadata.comparison shape — out of audit_events via `wrangler d1
3+
// results the advisory backtests persist — #8138's ORB-native threshold runs, #8139's CI-side logic
4+
// runs, and #8222's CI-side counterfactual prompt replays, sibling event types with the same
5+
// metadata.comparison shape — out of audit_events via `wrangler d1
56
// execute --json`, aggregates them with the pure computeRegressedVerdictTrackRecord (@loopover/engine), and
67
// prints the summary #8105's Phase-2 merge-gating decision needs. The aggregation lives in the engine
78
// (pure, unit-tested); this file is the thin IO wrapper — mirrors backtest-corpus-export.ts's identical split.
@@ -14,6 +15,7 @@ import { spawnSync } from "node:child_process";
1415
import { openPgDatabase, resolvePgConnection } from "./pg-cli.js";
1516
import { computeRegressedVerdictTrackRecord, type BacktestComparison } from "@loopover/engine";
1617
import { LOGIC_BACKTEST_EVENT_TYPE } from "./backtest-logic-check-core.js";
18+
import { COUNTERFACTUAL_BACKTEST_EVENT_TYPE } from "./counterfactual-replay-core.js";
1719

1820
// Mirrors THRESHOLD_BACKTEST_EVENT_TYPE in src/services/threshold-backtest-run.ts (#8138's writer) and must
1921
// be kept in sync with it by hand — that module is Worker-bound (D1 repositories import graph) and
@@ -59,7 +61,7 @@ async function main() {
5961
console.error("Usage: tsx scripts/backtest-track-record.ts --db <database> [--remote] | --pg <postgres://…>");
6062
process.exit(2);
6163
}
62-
const sql = `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`;
64+
const sql = `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}', '${COUNTERFACTUAL_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`;
6365
const rows = pgConnection ? await pgQuery(pgConnection, sql) : d1Query(args.db!, args.remote, sql);
6466
const comparisons: BacktestComparison[] = [];
6567
for (const row of rows) {

‎scripts/counterfactual-replay-core.ts‎

10.7 KB
Binary file not shown.

0 commit comments

Comments
 (0)