feat(calibration): per-provider reviewer track records from the consensus corpus (#8228)

Author: RealDiligent

packages/loopover-engine/src/calibration/provider-track-record.ts

@@ -0,0 +1,161 @@
+// Per-provider reviewer track records (#8228, epic #8211 track F). Dual-reviewer consensus events exist
+// (reviewer-consensus-calibration.ts) and reversal labels now say which calls were RIGHT; this module joins
+// the two: measured precision per reviewer identity, per repo and overall, over decided cases. Providers are
+// opaque ids — no provider names hardcoded, no config coupling. Mirrors the consensus module's ingestion
+// discipline (typed inputs, explicit vote vocabulary) and the #8085 scorer's null-below-the-sample-floor
+// rule: a slice that decided nothing reports null, never 0.
+//
+// JOIN SEMANTICS (documented once, tested as invariants):
+//   • A provider signal joins a labeled BacktestCase by exact `targetKey`. A signal whose target carries no
+//     decided label is counted (`signals`) but contributes to no rate — undecided is not evidence.
+//   • A provider "supported the firing" when it voted `fail` (the defect-flagging vote in the consensus
+//     vocabulary). `precision` = P(label "confirmed" | this provider voted fail) — the same
+//     correct-firing-as-numerator discipline as computeRulePrecision, at reviewer grain.
+//   • `agreementRate` = share of this provider's decided votes that MATCHED the human label (fail↔confirmed,
+//     pass/warn↔reversed) — a symmetric accuracy measure precision alone can't give a rarely-failing provider.
+//   • `consensusRate` = share of this provider's signals on targets that another provider ALSO reviewed
+//     where the two votes agreed (both-fail or both-non-fail); `splitRate` is its complement. Null when the
+//     provider shares no targets — one-provider corpora have no consensus to measure.
+//
+// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads.
+
+import type { BacktestCase } from "./backtest-corpus.js";
+import type { ReviewerConsensusVote } from "../reviewer-consensus-calibration.js";
+
+export type ProviderReviewSignal = {
+  /** Opaque reviewer identity — an id, never a hardcoded provider name. */
+  provider: string;
+  repoFullName: string;
+  /** Joins to {@link BacktestCase.targetKey} (`owner/repo#N`). */
+  targetKey: string;
+  vote: ReviewerConsensusVote;
+};
+
+export type ProviderTrackRecord = {
+  provider: string;
+  /** The repo this row aggregates, or null for the provider's overall rollup across every repo. */
+  repoFullName: string | null;
+  signals: number;
+  decided: number;
+  confirmed: number;
+  reversed: number;
+  precision: number | null;
+  agreementRate: number | null;
+  consensusRate: number | null;
+  splitRate: number | null;
+};
+
+type MutableStats = {
+  signals: number;
+  decided: number;
+  confirmed: number;
+  reversed: number;
+  failDecided: number;
+  failConfirmed: number;
+  agreed: number;
+  shared: number;
+  consensus: number;
+};
+
+function emptyStats(): MutableStats {
+  return { signals: 0, decided: 0, confirmed: 0, reversed: 0, failDecided: 0, failConfirmed: 0, agreed: 0, shared: 0, consensus: 0 };
+}
+
+function toRecord(provider: string, repoFullName: string | null, stats: MutableStats): ProviderTrackRecord {
+  return {
+    provider,
+    repoFullName,
+    signals: stats.signals,
+    decided: stats.decided,
+    confirmed: stats.confirmed,
+    reversed: stats.reversed,
+    precision: stats.failDecided > 0 ? stats.failConfirmed / stats.failDecided : null,
+    agreementRate: stats.decided > 0 ? stats.agreed / stats.decided : null,
+    consensusRate: stats.shared > 0 ? stats.consensus / stats.shared : null,
+    splitRate: stats.shared > 0 ? (stats.shared - stats.consensus) / stats.shared : null,
+  };
+}
+
+/**
+ * Compute per-(provider, repo) and per-provider-overall track records from reviewer signals joined against
+ * a labeled corpus, per the join semantics documented in this module's header. Deterministic ordering:
+ * providers ascending, and within each provider the overall rollup (repoFullName null) first, then repos
+ * ascending. Aggregates only — provider ids, repo names, and numbers; never target keys or vote payloads.
+ */
+export function computeProviderTrackRecords(
+  signals: readonly ProviderReviewSignal[],
+  cases: readonly BacktestCase[],
+): ProviderTrackRecord[] {
+  const labelByTarget = new Map<string, BacktestCase["label"]>();
+  for (const backtestCase of cases) labelByTarget.set(backtestCase.targetKey, backtestCase.label);
+
+  // Which providers reviewed each target, with their fail/non-fail stance — the consensus/split join.
+  const stancesByTarget = new Map<string, Map<string, boolean>>();
+  for (const signal of signals) {
+    let stances = stancesByTarget.get(signal.targetKey);
+    if (stances === undefined) {
+      stances = new Map();
+      stancesByTarget.set(signal.targetKey, stances);
+    }
+    stances.set(signal.provider, signal.vote === "fail");
+  }
+
+  const perRepo = new Map<string, Map<string, MutableStats>>(); // provider → repo → stats
+  const overall = new Map<string, MutableStats>();
+  for (const signal of signals) {
+    let repos = perRepo.get(signal.provider);
+    if (repos === undefined) {
+      repos = new Map();
+      perRepo.set(signal.provider, repos);
+    }
+    let repoStats = repos.get(signal.repoFullName);
+    if (repoStats === undefined) {
+      repoStats = emptyStats();
+      repos.set(signal.repoFullName, repoStats);
+    }
+    let overallStats = overall.get(signal.provider);
+    if (overallStats === undefined) {
+      overallStats = emptyStats();
+      overall.set(signal.provider, overallStats);
+    }
+
+    const label = labelByTarget.get(signal.targetKey);
+    const votedFail = signal.vote === "fail";
+    const stances = stancesByTarget.get(signal.targetKey)!;
+    for (const stats of [repoStats, overallStats]) {
+      stats.signals += 1;
+      if (label !== undefined) {
+        stats.decided += 1;
+        if (label === "confirmed") stats.confirmed += 1;
+        else stats.reversed += 1;
+        if (votedFail) {
+          stats.failDecided += 1;
+          if (label === "confirmed") stats.failConfirmed += 1;
+        }
+        // Matched the human: a fail vote on a confirmed firing, or a non-fail vote on a reversed one.
+        if (votedFail === (label === "confirmed")) stats.agreed += 1;
+      }
+      if (stances.size > 1) {
+        stats.shared += 1;
+        let agreeingOthers = 0;
+        let others = 0;
+        for (const [otherProvider, otherFail] of stances) {
+          if (otherProvider === signal.provider) continue;
+          others += 1;
+          if (otherFail === votedFail) agreeingOthers += 1;
+        }
+        if (agreeingOthers === others) stats.consensus += 1;
+      }
+    }
+  }
+
+  const records: ProviderTrackRecord[] = [];
+  for (const provider of [...overall.keys()].sort()) {
+    records.push(toRecord(provider, null, overall.get(provider)!));
+    const repos = perRepo.get(provider)!;
+    for (const repoFullName of [...repos.keys()].sort()) {
+      records.push(toRecord(provider, repoFullName, repos.get(repoFullName)!));
+    }
+  }
+  return records;
+}

packages/loopover-engine/src/index.ts

@@ -175,6 +175,7 @@ export * from "./calibration/backtest-track-record.js";
 // same way scripts/backtest-corpus-export.ts already imports BacktestCase.
 export * from "./calibration/backtest-split.js";
 export * from "./calibration/backtest-threshold.js";
+export * from "./calibration/provider-track-record.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,

packages/loopover-engine/test/provider-track-record.test.ts

@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { computeProviderTrackRecords, type BacktestCase, type ProviderReviewSignal } from "../dist/index.js";
+
+function labeled(targetKey: string, label: BacktestCase["label"]): BacktestCase {
+  return {
+    ruleId: "ai_consensus_defect",
+    targetKey,
+    outcome: "close",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+  };
+}
+
+function signal(provider: string, targetKey: string, vote: ProviderReviewSignal["vote"]): ProviderReviewSignal {
+  return { provider, repoFullName: "acme/widgets", targetKey, vote };
+}
+
+test("barrel: the public entrypoint re-exports the provider track-record computation (#8228)", () => {
+  assert.equal(typeof computeProviderTrackRecords, "function");
+});
+
+test("both-provider round-trip: precision + agreement + consensus rates land per provider", () => {
+  const records = computeProviderTrackRecords(
+    [
+      signal("a", "acme/widgets#1", "fail"),
+      signal("b", "acme/widgets#1", "fail"),
+      signal("a", "acme/widgets#2", "fail"),
+      signal("b", "acme/widgets#2", "pass"),
+    ],
+    [labeled("acme/widgets#1", "confirmed"), labeled("acme/widgets#2", "reversed")],
+  );
+  const aOverall = records.find((r) => r.provider === "a" && r.repoFullName === null)!;
+  assert.equal(aOverall.precision, 0.5);
+  assert.equal(aOverall.consensusRate, 0.5);
+  const bOverall = records.find((r) => r.provider === "b" && r.repoFullName === null)!;
+  assert.equal(bOverall.precision, 1);
+  assert.equal(bOverall.agreementRate, 1);
+});
+
+test("null discipline: no fail votes -> null precision; no shared targets -> null consensus/split", () => {
+  const records = computeProviderTrackRecords(
+    [signal("solo", "acme/widgets#1", "pass")],
+    [labeled("acme/widgets#1", "reversed")],
+  );
+  const overall = records.find((r) => r.repoFullName === null)!;
+  assert.equal(overall.precision, null);
+  assert.equal(overall.consensusRate, null);
+  assert.equal(overall.splitRate, null);
+  assert.equal(overall.agreementRate, 1);
+});

test/unit/provider-track-record-engine.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from "vitest";
+
+// Import the engine SOURCE directly (not the built dist) -- coverage.include lists
+// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in
+// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace
+// suite). Same pattern as backtest-corpus-engine.test.ts / repo-corpus-engine.test.ts.
+import {
+  computeProviderTrackRecords,
+  type ProviderReviewSignal,
+} from "../../packages/loopover-engine/src/calibration/provider-track-record";
+import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus";
+
+function labeled(targetKey: string, label: BacktestCase["label"]): BacktestCase {
+  return {
+    ruleId: "ai_consensus_defect",
+    targetKey,
+    outcome: "close",
+    label,
+    firedAt: "2026-07-01T00:00:00.000Z",
+    decidedAt: "2026-07-02T00:00:00.000Z",
+  };
+}
+
+function signal(provider: string, targetKey: string, vote: ProviderReviewSignal["vote"], repoFullName = "acme/widgets"): ProviderReviewSignal {
+  return { provider, repoFullName, targetKey, vote };
+}
+
+describe("computeProviderTrackRecords (#8228)", () => {
+  it("computes precision, agreement, and consensus/split rates for a both-provider corpus, per repo and overall", () => {
+    const cases = [
+      labeled("acme/widgets#1", "confirmed"),
+      labeled("acme/widgets#2", "reversed"),
+      labeled("acme/widgets#3", "confirmed"),
+    ];
+    const signals = [
+      // #1: both fail on a confirmed firing — consensus, both correct.
+      signal("provider-a", "acme/widgets#1", "fail"),
+      signal("provider-b", "acme/widgets#1", "fail"),
+      // #2: split — a fails (wrong: label reversed), b passes (right).
+      signal("provider-a", "acme/widgets#2", "fail"),
+      signal("provider-b", "acme/widgets#2", "pass"),
+      // #3: only a reviews it, warns (non-fail on a confirmed firing — disagreed with the human).
+      signal("provider-a", "acme/widgets#3", "warn"),
+    ];
+    const records = computeProviderTrackRecords(signals, cases);
+
+    const aOverall = records.find((r) => r.provider === "provider-a" && r.repoFullName === null)!;
+    expect(aOverall).toMatchObject({
+      signals: 3,
+      decided: 3,
+      confirmed: 2,
+      reversed: 1,
+      precision: 0.5, // of a's 2 fail votes, 1 hit a confirmed firing
+      agreementRate: 1 / 3, // matched the human only on #1
+      consensusRate: 0.5, // shared #1 (agreed) and #2 (split)
+      splitRate: 0.5,
+    });
+    const bOverall = records.find((r) => r.provider === "provider-b" && r.repoFullName === null)!;
+    expect(bOverall).toMatchObject({ signals: 2, decided: 2, precision: 1, agreementRate: 1, consensusRate: 0.5, splitRate: 0.5 });
+
+    // Single-repo corpus: each provider's per-repo row equals its overall rollup.
+    const aRepo = records.find((r) => r.provider === "provider-a" && r.repoFullName === "acme/widgets")!;
+    expect(aRepo).toMatchObject({ signals: aOverall.signals, decided: aOverall.decided, precision: aOverall.precision });
+  });
+
+  it("keeps a one-provider corpus's consensus/split rates null — no shared targets, no consensus to measure", () => {
+    const records = computeProviderTrackRecords(
+      [signal("solo", "acme/widgets#1", "fail"), signal("solo", "acme/widgets#2", "pass")],
+      [labeled("acme/widgets#1", "confirmed"), labeled("acme/widgets#2", "reversed")],
+    );
+    const overall = records.find((r) => r.repoFullName === null)!;
+    expect(overall.consensusRate).toBeNull();
+    expect(overall.splitRate).toBeNull();
+    expect(overall.precision).toBe(1);
+    expect(overall.agreementRate).toBe(1);
+  });
+
+  it("reports null (never 0) precision below the sample floor: undecided targets and providers that never voted fail", () => {
+    const records = computeProviderTrackRecords(
+      [
+        signal("quiet", "acme/widgets#9", "pass"), // undecided target — no label exists
+        signal("quiet", "acme/widgets#1", "warn"), // decided, but never a fail vote
+      ],
+      [labeled("acme/widgets#1", "reversed")],
+    );
+    const overall = records.find((r) => r.repoFullName === null)!;
+    expect(overall).toMatchObject({ signals: 2, decided: 1, precision: null, agreementRate: 1 });
+  });
+
+  it("rolls per-repo rows up into the overall row exactly, with deterministic provider→overall→repo ordering", () => {
+    const cases = [labeled("acme/widgets#1", "confirmed"), labeled("acme/gadgets#2", "confirmed")];
+    const signals = [
+      signal("zeta", "acme/widgets#1", "fail", "acme/widgets"),
+      signal("zeta", "acme/gadgets#2", "fail", "acme/gadgets"),
+      signal("alpha", "acme/widgets#1", "fail", "acme/widgets"),
+    ];
+    const records = computeProviderTrackRecords(signals, cases);
+    expect(records.map((r) => [r.provider, r.repoFullName])).toEqual([
+      ["alpha", null],
+      ["alpha", "acme/widgets"],
+      ["zeta", null],
+      ["zeta", "acme/gadgets"],
+      ["zeta", "acme/widgets"],
+    ]);
+    const zetaOverall = records.find((r) => r.provider === "zeta" && r.repoFullName === null)!;
+    const zetaRepos = records.filter((r) => r.provider === "zeta" && r.repoFullName !== null);
+    expect(zetaRepos.reduce((sum, r) => sum + r.decided, 0)).toBe(zetaOverall.decided);
+    expect(zetaRepos.reduce((sum, r) => sum + r.signals, 0)).toBe(zetaOverall.signals);
+    // Determinism: identical inputs yield the identical result.
+    expect(computeProviderTrackRecords(signals, cases)).toEqual(records);
+  });
+
+  it("never leaks target keys into any returned shape — provider ids, repo names, and numbers only", () => {
+    const records = computeProviderTrackRecords(
+      [signal("provider-a", "acme/widgets#42", "fail")],
+      [labeled("acme/widgets#42", "confirmed")],
+    );
+    expect(JSON.stringify(records)).not.toContain("#42");
+  });
+
+  it("reports null agreement (never 0) for a provider whose every signal is undecided", () => {
+    const records = computeProviderTrackRecords([signal("unjoined", "acme/widgets#404", "fail")], []);
+    const overall = records.find((r) => r.repoFullName === null)!;
+    expect(overall).toMatchObject({ signals: 1, decided: 0, precision: null, agreementRate: null });
+  });
+
+  it("returns an empty list for empty inputs", () => {
+    expect(computeProviderTrackRecords([], [])).toEqual([]);
+  });
+});