feat(engine): render backtest score and comparison reports as deterministic Markdown (#8088)

Author: cleanjunc

packages/loopover-engine/src/calibration/backtest-report.ts

@@ -0,0 +1,67 @@
+// Markdown renderers for backtest results (#8088, parent epic #8082) -- the human-readable "receipt" a
+// maintainer (and, per the epic's proposal, a future advisory CI comment) reads directly. Deterministic pure
+// functions producing stable Markdown: byte-identical input -> byte-identical output, never ad-hoc logging.
+//
+// Pure, like everything in this module: string in, string out; no IO, no wall-clock reads.
+
+import type { BacktestComparison } from "./backtest-compare.js";
+import type { BacktestScoreReport } from "./backtest-score.js";
+
+/** Render a null-able rate as its number, or the literal `N/A` -- never `0`, `null`, or an empty cell (the
+ *  null-is-not-zero discipline BacktestScoreReport itself establishes). */
+function renderRate(value: number | null): string {
+  return value === null ? "N/A" : String(value);
+}
+
+/** Render one score report as a Markdown table: rule ID, case count, all four confusion-matrix counts, and
+ *  precision/recall (null rendered as `N/A`). */
+export function renderBacktestScoreReport(report: BacktestScoreReport): string {
+  return [
+    `### Backtest score: \`${report.ruleId}\``,
+    "",
+    "| Metric | Value |",
+    "| --- | --- |",
+    `| Cases scored | ${report.caseCount} |`,
+    `| True positives | ${report.truePositive} |`,
+    `| False positives | ${report.falsePositive} |`,
+    `| True negatives | ${report.trueNegative} |`,
+    `| False negatives | ${report.falseNegative} |`,
+    `| Precision | ${renderRate(report.precision)} |`,
+    `| Recall | ${renderRate(report.recall)} |`,
+    "",
+  ].join("\n");
+}
+
+/**
+ * Render a comparison with clearly separated Regressed / Improved sections (an axis only ever appears under
+ * its own heading) and a closing verdict line. The regressed closing line contains the literal word
+ * `REGRESSED` and states the change should not be merged -- exact wording a future automated consumer can
+ * string-match without re-implementing the comparison.
+ */
+export function renderBacktestComparison(comparison: BacktestComparison): string {
+  const lines: string[] = [`### Backtest comparison: \`${comparison.ruleId}\``, ""];
+  if (comparison.regressedAxes.length > 0) {
+    lines.push("**Regressed**", "");
+    for (const axis of comparison.regressedAxes) {
+      // A listed axis is non-null on BOTH sides by compareBacktestScores's own exclusion rule.
+      lines.push(`- ${axis}: ${comparison.baseline[axis]} → ${comparison.candidate[axis]}`);
+    }
+    lines.push("");
+  }
+  if (comparison.improvedAxes.length > 0) {
+    lines.push("**Improved**", "");
+    for (const axis of comparison.improvedAxes) {
+      lines.push(`- ${axis}: ${comparison.baseline[axis]} → ${comparison.candidate[axis]}`);
+    }
+    lines.push("");
+  }
+  if (comparison.verdict === "regressed") {
+    lines.push("Verdict: REGRESSED — do not merge (a regression on any axis outweighs improvement on another).");
+  } else if (comparison.verdict === "improved") {
+    lines.push("Verdict: improved — no axis regressed and at least one improved.");
+  } else {
+    lines.push("Verdict: unchanged — no comparable axis moved in either direction.");
+  }
+  lines.push("");
+  return lines.join("\n");
+}

packages/loopover-engine/src/index.ts

@@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js";
 export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
 export * from "./calibration/backtest-compare.js";
+export * from "./calibration/backtest-report.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/backtest-report.test.ts

@@ -0,0 +1,86 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  compareBacktestScores,
+  renderBacktestComparison,
+  renderBacktestScoreReport,
+  type BacktestScoreReport,
+} from "../dist/index.js";
+
+// #8088: the Markdown "receipt" renderers. Null rates render as the literal `N/A` (never 0/null/empty), a
+// regressed comparison's closing line contains the literal REGRESSED + do-not-merge wording, and both
+// functions are byte-identically deterministic.
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "rule",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 1,
+    trueNegative: 4,
+    falseNegative: 1,
+    precision: 0.8,
+    recall: 0.8,
+    ...overrides,
+  };
+}
+
+test("renderBacktestScoreReport: exact snapshot for a non-null fixture", () => {
+  assert.equal(
+    renderBacktestScoreReport(report()),
+    [
+      "### Backtest score: `rule`",
+      "",
+      "| Metric | Value |",
+      "| --- | --- |",
+      "| Cases scored | 10 |",
+      "| True positives | 4 |",
+      "| False positives | 1 |",
+      "| True negatives | 4 |",
+      "| False negatives | 1 |",
+      "| Precision | 0.8 |",
+      "| Recall | 0.8 |",
+      "",
+    ].join("\n"),
+  );
+});
+
+test("renderBacktestScoreReport: null precision/recall render as the literal N/A, never 0 or null", () => {
+  const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+  assert.ok(rendered.includes("| Precision | N/A |"));
+  assert.ok(rendered.includes("| Recall | N/A |"));
+  assert.ok(!rendered.includes("| Precision | 0 |"));
+  assert.ok(!rendered.includes("null"));
+});
+
+test("renderBacktestComparison: a regressed verdict's closing line contains REGRESSED and do-not-merge wording", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 }));
+  const rendered = renderBacktestComparison(comparison);
+  assert.ok(rendered.includes("REGRESSED"));
+  assert.ok(rendered.includes("do not merge"));
+  // The regressed axis appears only under Regressed; the improved axis only under Improved.
+  const regressedSection = rendered.slice(rendered.indexOf("**Regressed**"), rendered.indexOf("**Improved**"));
+  assert.ok(regressedSection.includes("recall"));
+  assert.ok(!regressedSection.includes("- precision"));
+});
+
+test("renderBacktestComparison: an improved comparison renders no regressed axis claim", () => {
+  const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 }));
+  const rendered = renderBacktestComparison(comparison);
+  assert.ok(rendered.includes("**Improved**"));
+  assert.ok(!rendered.includes("**Regressed**"));
+  assert.ok(rendered.includes("Verdict: improved"));
+});
+
+test("renderBacktestComparison: unchanged verdict states unchanged in words", () => {
+  const rendered = renderBacktestComparison(compareBacktestScores(report(), report()));
+  assert.ok(rendered.includes("Verdict: unchanged"));
+});
+
+test("both renderers are byte-identically deterministic for identical input", () => {
+  const scoreReport = report({ precision: null });
+  assert.equal(renderBacktestScoreReport(scoreReport), renderBacktestScoreReport(scoreReport));
+  const comparison = compareBacktestScores(report(), report({ recall: 0.9 }));
+  assert.equal(renderBacktestComparison(comparison), renderBacktestComparison(comparison));
+});

test/unit/backtest-report.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+// Direct src-path imports — the coverage-twin pattern test/unit/backtest-corpus.test.ts established for this
+// module (the engine's own node:test suite runs against dist, invisible to codecov/patch).
+import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js";
+import { renderBacktestComparison, renderBacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-report.js";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js";
+
+function report(overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return {
+    ruleId: "rule",
+    caseCount: 10,
+    truePositive: 4,
+    falsePositive: 1,
+    trueNegative: 4,
+    falseNegative: 1,
+    precision: 0.8,
+    recall: 0.8,
+    ...overrides,
+  };
+}
+
+describe("renderBacktestScoreReport (#8088)", () => {
+  it("renders the exact table for a non-null fixture (snapshot)", () => {
+    expect(renderBacktestScoreReport(report())).toBe(
+      [
+        "### Backtest score: `rule`",
+        "",
+        "| Metric | Value |",
+        "| --- | --- |",
+        "| Cases scored | 10 |",
+        "| True positives | 4 |",
+        "| False positives | 1 |",
+        "| True negatives | 4 |",
+        "| False negatives | 1 |",
+        "| Precision | 0.8 |",
+        "| Recall | 0.8 |",
+        "",
+      ].join("\n"),
+    );
+  });
+
+  it("renders null precision/recall as the literal N/A", () => {
+    const rendered = renderBacktestScoreReport(report({ precision: null, recall: null }));
+    expect(rendered).toContain("| Precision | N/A |");
+    expect(rendered).toContain("| Recall | N/A |");
+    expect(rendered).not.toContain("null");
+  });
+});
+
+describe("renderBacktestComparison (#8088)", () => {
+  it("keeps regressed and improved axes in visually separate sections and closes with REGRESSED — do not merge", () => {
+    const rendered = renderBacktestComparison(compareBacktestScores(report(), report({ precision: 0.95, recall: 0.6 })));
+    expect(rendered).toContain("REGRESSED");
+    expect(rendered).toContain("do not merge");
+    const regressedSection = rendered.slice(rendered.indexOf("**Regressed**"), rendered.indexOf("**Improved**"));
+    expect(regressedSection).toContain("recall");
+    expect(regressedSection).not.toContain("- precision");
+  });
+
+  it("renders an improved comparison with no Regressed section at all", () => {
+    const rendered = renderBacktestComparison(compareBacktestScores(report(), report({ precision: 0.9, recall: 0.85 })));
+    expect(rendered).not.toContain("**Regressed**");
+    expect(rendered).toContain("Verdict: improved");
+  });
+
+  it("omits a null-excluded axis from both sections entirely", () => {
+    const rendered = renderBacktestComparison(compareBacktestScores(report({ precision: null }), report({ precision: null, recall: 0.9 })));
+    expect(rendered).not.toContain("precision");
+    expect(rendered).toContain("- recall: 0.8 → 0.9");
+    expect(rendered).toContain("Verdict: improved");
+  });
+
+  it("states the unchanged verdict in words and is byte-identically deterministic", () => {
+    const comparison = compareBacktestScores(report(), report());
+    const first = renderBacktestComparison(comparison);
+    expect(first).toContain("Verdict: unchanged");
+    expect(renderBacktestComparison(comparison)).toBe(first);
+  });
+});