feat(calibration): render backtest score/comparison reports as Markdown (#8088)

RealDiligent · RealDiligent · commit c774db95bb0b · 2026-07-23T07:11:05.000+08:00
diff --git a/packages/loopover-engine/src/calibration/backtest-report.ts b/packages/loopover-engine/src/calibration/backtest-report.ts
@@ -0,0 +1,68 @@
+// Markdown renderers for backtest results (#8088) -- the human-readable "receipt" a maintainer (and, per
+// the parent epic, eventually an advisory CI comment) reads for a BacktestScoreReport (#8085) or a
+// BacktestComparison (#8086). Deterministic pure functions producing stable Markdown, not ad-hoc console
+// logging: byte-identical input always renders byte-identical output.
+//
+// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestComparison } from "./backtest-compare.js";
+import type { BacktestScoreReport } from "./backtest-score.js";
+
+/** Render a nullable ratio for display: `null` is `N/A` -- never `0`, the word `null`, or an empty cell
+ *  (the same null-is-not-zero discipline BacktestScoreReport itself establishes). */
+function renderRatio(value: number | null): string {
+  return value === null ? "N/A" : String(value);
+}
+
+/**
+ * Render one {@link BacktestScoreReport} as a Markdown table: the rule ID as a heading, then every count
+ * and both (nullable) ratios. Pure string-in/string-out; the exact layout is pinned by a snapshot test.
+ */
+export function renderBacktestScoreReport(report: BacktestScoreReport): string {
+  return [
+    `### Backtest score — \`${report.ruleId}\``,
+    "",
+    "| Metric | Value |",
+    "| --- | --- |",
+    `| Cases scored | ${report.caseCount} |`,
+    `| True positives | ${report.truePositive} |`,
+    `| False positives | ${report.falsePositive} |`,
+    `| True negatives | ${report.trueNegative} |`,
+    `| False negatives | ${report.falseNegative} |`,
+    `| Precision | ${renderRatio(report.precision)} |`,
+    `| Recall | ${renderRatio(report.recall)} |`,
+  ].join("\n");
+}
+
+/**
+ * Render one {@link BacktestComparison} as Markdown: the rule ID as a heading, a "Regressed" section for
+ * every regressed axis, a visually separate "Improved" section for every improved axis (an axis can only
+ * ever appear under its own section -- the two lists are disjoint by construction upstream), and a closing
+ * verdict line. The `"regressed"` closing line contains the literal word `REGRESSED` and states the change
+ * should not be merged -- pinned wording, so a future automated consumer can detect the regressed case by
+ * string match without re-implementing the comparison logic. Sections with no axes render as "(none)"
+ * rather than listing anything, so an empty regression list can never read as if something regressed.
+ */
+export function renderBacktestComparison(comparison: BacktestComparison): string {
+  const axisLines = (axes: ReadonlyArray<"precision" | "recall">): string[] =>
+    axes.length === 0 ? ["- (none)"] : axes.map((axis) => `- ${axis}`);
+  const verdictLine =
+    comparison.verdict === "regressed"
+      ? "Verdict: REGRESSED — do not merge"
+      : comparison.verdict === "improved"
+        ? "Verdict: improved"
+        : "Verdict: unchanged";
+  return [
+    `### Backtest comparison — \`${comparison.ruleId}\``,
+    "",
+    "**Regressed**",
+    "",
+    ...axisLines(comparison.regressedAxes),
+    "",
+    "**Improved**",
+    "",
+    ...axisLines(comparison.improvedAxes),
+    "",
+    verdictLine,
+  ].join("\n");
+}
diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts
@@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
 export * from "./calibration/backtest-compare.js";
+export * from "./calibration/backtest-report.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,
diff --git a/packages/loopover-engine/test/backtest-report.test.ts b/packages/loopover-engine/test/backtest-report.test.ts
@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  renderBacktestComparison,
+  renderBacktestScoreReport,
+  type BacktestComparison,
+  type BacktestScoreReport,
+} from "../dist/index.js";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 4,
+    truePositive: 1,
+    falsePositive: 1,
+    trueNegative: 1,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+function comparison(overrides: Partial<BacktestComparison> = {}): BacktestComparison {
+  return {
+    ruleId: "missing_linked_issue",
+    baseline: report(),
+    candidate: report({ precision: 0.75 }),
+    regressedAxes: [],
+    improvedAxes: ["precision"],
+    verdict: "improved",
+    ...overrides,
+  };
+}
+
+test("renderBacktestScoreReport: snapshot -- a non-null report renders every count and both ratios", () => {
+  assert.equal(
+    renderBacktestScoreReport(report()),
+    [
+      "### Backtest score — `missing_linked_issue`",
+      "",
+      "| Metric | Value |",
+      "| --- | --- |",
+      "| Cases scored | 4 |",
+      "| True positives | 1 |",
+      "| False positives | 1 |",
+      "| True negatives | 1 |",
+      "| False negatives | 1 |",
+      "| Precision | 0.5 |",
+      "| Recall | 0.5 |",
+    ].join("\n"),
+  );
+});
+
+test("renderBacktestScoreReport: null precision/recall render as N/A, never 0, null, or an empty cell", () => {
+  const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+  assert.match(rendered, /\| Precision \| N\/A \|/);
+  assert.match(rendered, /\| Recall \| N\/A \|/);
+  assert.doesNotMatch(rendered, /\| Precision \| (0|null)? \|/);
+  assert.doesNotMatch(rendered, /\| Recall \| (0|null)? \|/);
+});
+
+test("renderBacktestComparison: a regressed comparison names the regressed axis under Regressed and closes with the literal REGRESSED wording", () => {
+  const rendered = renderBacktestComparison(
+    comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed" }),
+  );
+  assert.match(rendered, /\*\*Regressed\*\*\n\n- recall/);
+  assert.match(rendered, /\*\*Improved\*\*\n\n- precision/);
+  assert.match(rendered, /Verdict: REGRESSED — do not merge/);
+});
+
+test("renderBacktestComparison: an improved comparison claims no regressed axis", () => {
+  const rendered = renderBacktestComparison(comparison());
+  assert.match(rendered, /\*\*Regressed\*\*\n\n- \(none\)/);
+  assert.match(rendered, /\*\*Improved\*\*\n\n- precision/);
+  assert.match(rendered, /Verdict: improved/);
+  assert.doesNotMatch(rendered, /REGRESSED/);
+});
+
+test("renderBacktestComparison: an unchanged comparison lists no axis on either side", () => {
+  const rendered = renderBacktestComparison(
+    comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() }),
+  );
+  assert.match(rendered, /\*\*Regressed\*\*\n\n- \(none\)/);
+  assert.match(rendered, /\*\*Improved\*\*\n\n- \(none\)/);
+  assert.match(rendered, /Verdict: unchanged/);
+});
+
+test("both renderers are deterministic: identical input renders byte-identical output", () => {
+  assert.equal(renderBacktestScoreReport(report()), renderBacktestScoreReport(report()));
+  const regressed = comparison({ regressedAxes: ["recall"], verdict: "regressed" });
+  assert.equal(renderBacktestComparison(regressed), renderBacktestComparison(regressed));
+});
diff --git a/test/unit/backtest-report-engine.test.ts b/test/unit/backtest-report-engine.test.ts
@@ -0,0 +1,103 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts.
+import {
+  renderBacktestComparison,
+  renderBacktestScoreReport,
+} from "../../packages/loopover-engine/src/calibration/backtest-report";
+import type { BacktestComparison } from "../../packages/loopover-engine/src/calibration/backtest-compare";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "missing_linked_issue",
+    caseCount: 4,
+    truePositive: 1,
+    falsePositive: 1,
+    trueNegative: 1,
+    falseNegative: 1,
+    precision: 0.5,
+    recall: 0.5,
+    ...overrides,
+  };
+}
+
+function comparison(overrides: Partial<BacktestComparison> = {}): BacktestComparison {
+  return {
+    ruleId: "missing_linked_issue",
+    baseline: report(),
+    candidate: report({ precision: 0.75 }),
+    regressedAxes: [],
+    improvedAxes: ["precision"],
+    verdict: "improved",
+    ...overrides,
+  };
+}
+
+describe("renderBacktestScoreReport (#8088)", () => {
+  it("renders the exact snapshot for a non-null report", () => {
+    expect(renderBacktestScoreReport(report())).toBe(
+      [
+        "### Backtest score — `missing_linked_issue`",
+        "",
+        "| Metric | Value |",
+        "| --- | --- |",
+        "| Cases scored | 4 |",
+        "| True positives | 1 |",
+        "| False positives | 1 |",
+        "| True negatives | 1 |",
+        "| False negatives | 1 |",
+        "| Precision | 0.5 |",
+        "| Recall | 0.5 |",
+      ].join("\n"),
+    );
+  });
+
+  it("renders null precision/recall as N/A -- never 0, null, or an empty cell", () => {
+    const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+    expect(rendered).toContain("| Precision | N/A |");
+    expect(rendered).toContain("| Recall | N/A |");
+    expect(rendered).not.toContain("| Precision | 0 |");
+    expect(rendered).not.toContain("null");
+  });
+
+  it("is deterministic for identical input", () => {
+    expect(renderBacktestScoreReport(report())).toBe(renderBacktestScoreReport(report()));
+  });
+});
+
+describe("renderBacktestComparison (#8088)", () => {
+  it("puts each axis under its own section and pins the literal REGRESSED do-not-merge wording", () => {
+    const rendered = renderBacktestComparison(
+      comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed" }),
+    );
+    expect(rendered).toContain("**Regressed**\n\n- recall");
+    expect(rendered).toContain("**Improved**\n\n- precision");
+    expect(rendered).toContain("Verdict: REGRESSED — do not merge");
+  });
+
+  it("claims no regressed axis for an improved comparison", () => {
+    const rendered = renderBacktestComparison(comparison());
+    expect(rendered).toContain("**Regressed**\n\n- (none)");
+    expect(rendered).toContain("**Improved**\n\n- precision");
+    expect(rendered).toContain("Verdict: improved");
+    expect(rendered).not.toContain("REGRESSED");
+  });
+
+  it("lists no axis on either side for an unchanged comparison", () => {
+    const rendered = renderBacktestComparison(
+      comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() }),
+    );
+    expect(rendered).toContain("**Regressed**\n\n- (none)");
+    expect(rendered).toContain("**Improved**\n\n- (none)");
+    expect(rendered).toContain("Verdict: unchanged");
+  });
+
+  it("is deterministic for identical input", () => {
+    const regressed = comparison({ regressedAxes: ["recall"], verdict: "regressed" });
+    expect(renderBacktestComparison(regressed)).toBe(renderBacktestComparison(regressed));
+  });
+});
