feat(ci): persist the corpus checksum with each logic-backtest run as its reproducibility freeze point (#8139)

The persisted run recorded the comparison and both shas but not WHICH corpus
snapshot it scored -- without that, a skeptical re-run can't prove it replayed
the same history. The manifest's own checksum (#8084) is already in hand at
run time; thread it into the persisted metadata and the PR comment, so
checksum + head/base shas + the public scoring code make every run
independently re-runnable end to end (the reproducible-backtest posture
#8136 is evaluating).

Author: JSONbored

scripts/backtest-logic-check-core.ts

@@ -131,22 +131,24 @@ export function runLogicBacktest(
 export const LOGIC_BACKTEST_COMMENT_MARKER = "<!-- loopover-logic-backtest -->";
 
 /**
- * Render the standalone advisory PR comment: marker, what was replayed against what, the engine's own
- * comparison Markdown (#8088), and the never-blocks-merge note. Deliberately its OWN comment, not a section
- * of ORB's unified review comment — see #8139's Boundaries (this CI job runs outside the Worker's review
- * flow, and joining that comment would need new Worker↔CI coupling).
+ * Render the standalone advisory PR comment: marker, what was replayed against what (including the corpus
+ * checksum — the freeze point that makes the run independently re-runnable, see #8084's manifest), the
+ * engine's own comparison Markdown (#8088), and the never-blocks-merge note. Deliberately its OWN comment,
+ * not a section of ORB's unified review comment — see #8139's Boundaries (this CI job runs outside the
+ * Worker's review flow, and joining that comment would need new Worker↔CI coupling).
  */
 export function renderLogicBacktestComment(
   comparison: BacktestComparison,
-  info: { replayableCount: number; skippedCount: number; headSha: string; baseSha: string },
+  info: { replayableCount: number; skippedCount: number; headSha: string; baseSha: string; corpusChecksum: string },
 ): string {
   const skippedNote = info.skippedCount > 0 ? ` ${info.skippedCount} historical case(s) lacked captured raw context and were skipped.` : "";
   return [
     LOGIC_BACKTEST_COMMENT_MARKER,
     "## Logic backtest",
     "",
     `Replayed ${info.replayableCount} historical case(s) for \`${comparison.ruleId}\` through the base` +
-      ` (\`${info.baseSha.slice(0, 7)}\`) and head (\`${info.headSha.slice(0, 7)}\`) versions of its detection logic.${skippedNote}`,
+      ` (\`${info.baseSha.slice(0, 7)}\`) and head (\`${info.headSha.slice(0, 7)}\`) versions of its detection logic` +
+      ` (corpus checksum \`${info.corpusChecksum.slice(0, 12)}\`).${skippedNote}`,
     "",
     renderBacktestComparison(comparison),
     "_Advisory only — this check never blocks merge (#8105)._",
@@ -164,14 +166,17 @@ export function sqlStringLiteral(value: string): string {
  * THRESHOLD_BACKTEST_EVENT_TYPE events, same audit_events columns recordAuditEvent writes (the CLI runs
  * outside the Worker, so it goes through `wrangler d1 execute` instead of the repositories module; the
  * caller supplies id/createdAt so this stays clock-free like the rest of this file). `metadata.comparison`
- * is the field backtest-track-record.ts's reader already looks for.
+ * is the field backtest-track-record.ts's reader already looks for; `corpusChecksum` + the two shas are
+ * the freeze point (#8136's reproducibility posture) — enough for a skeptic to re-export the corpus,
+ * verify the checksum, and re-run both sides of this exact comparison independently.
  */
 export function buildLogicBacktestAuditInsertSql(input: {
   id: string;
   targetKey: string;
   comparison: BacktestComparison;
   headSha: string;
   baseSha: string;
+  corpusChecksum: string;
   replayableCount: number;
   skippedCount: number;
   createdAt: string;
@@ -180,6 +185,7 @@ export function buildLogicBacktestAuditInsertSql(input: {
     comparison: input.comparison,
     headSha: input.headSha,
     baseSha: input.baseSha,
+    corpusChecksum: input.corpusChecksum,
     replayableCount: input.replayableCount,
     skippedCount: input.skippedCount,
   });

scripts/backtest-logic-check.ts

@@ -122,6 +122,7 @@ async function main() {
       skippedCount,
       headSha: args.headSha,
       baseSha: args.baseSha,
+      corpusChecksum: manifest.checksum,
     }),
   );
 
@@ -132,6 +133,7 @@ async function main() {
       comparison,
       headSha: args.headSha,
       baseSha: args.baseSha,
+      corpusChecksum: manifest.checksum,
       replayableCount: replayable.length,
       skippedCount,
       createdAt: new Date().toISOString(),

test/unit/backtest-logic-check-core.test.ts

@@ -194,10 +194,12 @@ describe("backtest-logic-check-core renderLogicBacktestComment (#8139)", () => {
       skippedCount: 0,
       headSha: "abcdef1234567890",
       baseSha: "1234567890abcdef",
+      corpusChecksum: "feedfacecafe0123456789",
     });
     expect(comment.startsWith(`${LOGIC_BACKTEST_COMMENT_MARKER}\n## Logic backtest`)).toBe(true);
     expect(comment).toContain("Replayed 12 historical case(s) for `linked_issue_scope_mismatch`");
     expect(comment).toContain("base (`1234567`) and head (`abcdef1`)");
+    expect(comment).toContain("corpus checksum `feedfacecafe`");
     expect(comment).toContain("### Backtest comparison: `linked_issue_scope_mismatch`");
     expect(comment).toContain("never blocks merge (#8105)");
     expect(comment).not.toContain("lacked captured raw context");
@@ -209,6 +211,7 @@ describe("backtest-logic-check-core renderLogicBacktestComment (#8139)", () => {
       skippedCount: 5,
       headSha: "abcdef1234567890",
       baseSha: "1234567890abcdef",
+      corpusChecksum: "feedfacecafe0123456789",
     });
     expect(comment).toContain("5 historical case(s) lacked captured raw context and were skipped.");
   });
@@ -229,6 +232,7 @@ describe("backtest-logic-check-core buildLogicBacktestAuditInsertSql (#8139)", (
       comparison,
       headSha: "abcdef1234567890",
       baseSha: "1234567890abcdef",
+      corpusChecksum: "feedfacecafe0123456789",
       replayableCount: 1,
       skippedCount: 2,
       createdAt: "2026-07-22T12:00:00.000Z",
@@ -246,6 +250,7 @@ describe("backtest-logic-check-core buildLogicBacktestAuditInsertSql (#8139)", (
     expect(metadata.comparison).toEqual(comparison);
     expect(metadata.headSha).toBe("abcdef1234567890");
     expect(metadata.baseSha).toBe("1234567890abcdef");
+    expect(metadata.corpusChecksum).toBe("feedfacecafe0123456789");
     expect(metadata.replayableCount).toBe(1);
     expect(metadata.skippedCount).toBe(2);
   });