feat(benchmark): reversal-aware realized-history ground truth (#9261) (#9598)

The labels every candidate proposal is scored against: for each work unit
in a snapshot, what the maintainer ACTUALLY did within the prediction
horizon, emitted in #9260's exact action vocabulary so proposal and
outcome compare with no translation layer that could drift.

Three rules the surrounding system already believes, now enforced here:

- REVERSAL-AWARE SETTLEMENT. A merge later reverted is not a clean merge.
  The label is the settled state at horizon end with the reversal recorded
  alongside -- the treatment that makes public-rule-precision.ts's
  precision honest. The vocabulary is the established
  reversal_reopened/reverted/superseded set, never a benchmark-local
  re-invention, so benchmark and internal backtest ground truth cannot
  disagree about the same event. A reversal counts only inside the horizon
  AND strictly after the action it overturns; the earliest one wins.
- UNRESOLVED IS EXPLICIT AND EXCLUDED. No action inside the horizon is a
  first-class `unresolved`, never silently a correct abstention (which
  would reward declining exactly what nobody decided) nor an incorrect
  prediction (which would punish the horizon being short). It leaves the
  denominator, and the unresolved RATE is published so a climbing rate
  reads as "the horizon is wrong" rather than as a scoring artifact.
- STABILITY. Every decision is a pure function of events dated inside the
  window, so a later event -- including a later reversal -- cannot move a
  settled label. Asserted directly: the same input plus a year of
  post-horizon history yields byte-identical output.

Closes #9261

Author: JSONbored

packages/loopover-engine/src/calibration/benchmark-ground-truth.ts

@@ -0,0 +1,203 @@
+// Realized-history ground truth for the frozen-repo benchmark (#9261, harness #9216, epic #8534).
+//
+// For each work unit in a snapshot: what the maintainer ACTUALLY did within the prediction horizon. These
+// are the labels every candidate proposal (#9260) is scored against, so they are emitted in that module's
+// exact action vocabulary — proposal and outcome are directly comparable, with no translation layer that
+// could drift.
+//
+// Three rules this module exists to enforce, each of which the surrounding system already believes and which
+// a benchmark must not quietly contradict:
+//
+//   REVERSAL-AWARE SETTLEMENT. A merge later reverted is not a clean merge. The realized outcome is the
+//   SETTLED state at horizon end, with the reversal recorded alongside — the same treatment that makes
+//   `public-rule-precision.ts`'s precision honest (it subtracts reversals from confirmations rather than
+//   counting the original decision). The reversal vocabulary is the established one
+//   (`reversal_reopened` / `reversal_reverted` / `reversal_superseded`), not a new benchmark-local set, so
+//   benchmark ground truth and internal backtest ground truth can never disagree about the same event.
+//
+//   UNRESOLVED IS EXPLICIT AND EXCLUDED. A work unit with no maintainer action inside the horizon is
+//   `unresolved`. It is NOT silently counted as a correct abstention (which would reward an agent for
+//   declining exactly the cases nobody decided) nor as an incorrect prediction (which would punish an agent
+//   for the horizon being too short). It leaves the scoring denominator entirely, and the unresolved RATE is
+//   published so a climbing rate is legible as "the horizon is wrong" rather than as a scoring artifact.
+//
+//   STABILITY. Ground truth for a given (snapshot, horizon) must not depend on WHEN extraction runs, once
+//   the horizon has elapsed. Every decision here is a pure function of events dated inside the window, so a
+//   later event — including a later reversal — cannot retroactively change a settled label. That is what
+//   makes a leaderboard reproducible instead of drifting under its own scores.
+//
+// Pure core (this module); the IO wrapper that reads events from D1 lives engine-side of the harness, the
+// same split every sibling calibration module uses.
+
+import type { BenchmarkActionKind, BenchmarkCloseReasonClass } from "./benchmark-proposal.js";
+
+/** The established reversal vocabulary — deliberately imported as literals from the same set the audit-event
+ *  table and `public-rule-precision.ts` already use, never a benchmark-local re-invention. */
+export type RealizedReversalKind = "reversal_reopened" | "reversal_reverted" | "reversal_superseded";
+
+/** One realized maintainer event inside (or outside) a horizon, already normalized by the IO wrapper. */
+export type RealizedMaintainerEvent = {
+  workUnitId: string;
+  /** Same closed vocabulary the agent proposes in — comparison needs no translation. */
+  action: BenchmarkActionKind;
+  /** ISO-8601. Events at exactly the horizon end are INSIDE the window (inclusive), matching how the
+   *  corpus builder's own windowing treats a boundary event as belonging to the window it closes. */
+  occurredAt: string;
+  /** Present only for a `close`; the class realized history settled on. */
+  reasonClass?: BenchmarkCloseReasonClass | undefined;
+  /** Present only for a `label`. */
+  labels?: readonly string[] | undefined;
+};
+
+/** A reversal of a previously-realized action, in the established vocabulary. */
+export type RealizedReversalEvent = {
+  workUnitId: string;
+  kind: RealizedReversalKind;
+  occurredAt: string;
+};
+
+/** The settled label for ONE work unit. `unresolved` carries no action by construction — the type makes
+ *  "unresolved but somehow also a merge" unrepresentable rather than merely untested. */
+export type BenchmarkGroundTruth =
+  | { workUnitId: string; outcome: "unresolved" }
+  | {
+      workUnitId: string;
+      outcome: "settled";
+      action: BenchmarkActionKind;
+      reasonClass?: BenchmarkCloseReasonClass | undefined;
+      labels?: readonly string[] | undefined;
+      settledAt: string;
+      /** The reversal that overturned this action inside the horizon, if any. A reversed action is still
+       *  the realized action — it is simply not a CONFIRMED one, exactly as `public-rule-precision.ts`
+       *  treats a reversed decision: recorded, and subtracted from the confirmed count. */
+      reversal: { kind: RealizedReversalKind; occurredAt: string } | null;
+    };
+
+export type BenchmarkGroundTruthSet = {
+  schemaVersion: 1;
+  snapshotRef: string;
+  horizonDays: number;
+  frozenAt: string;
+  /** The horizon's inclusive end — `frozenAt + horizonDays`, computed once and published so a consumer
+   *  never has to re-derive (and possibly re-derive differently) the window a label was settled in. */
+  horizonEnd: string;
+  truths: BenchmarkGroundTruth[];
+  /** Denominator discipline, published rather than left implicit: `scoreable` is what a scorer divides by;
+   *  `unresolved` is excluded from it, and a climbing `unresolvedRate` is the signal the horizon is wrong. */
+  coverage: {
+    workUnits: number;
+    scoreable: number;
+    unresolved: number;
+    /** `unresolved / workUnits`, 3dp. `null` for an empty work-unit set — never 0, which would read as
+     *  "a fully resolved benchmark" (the same null-not-zero discipline as PublicRulePrecisionRow). */
+    unresolvedRate: number | null;
+  };
+};
+
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+/** The inclusive horizon end for a snapshot. Exported so the harness, the scorer, and any third-party
+ *  verifier compute the SAME instant rather than three slightly different ones. */
+export function benchmarkHorizonEnd(frozenAt: string, horizonDays: number): string {
+  return new Date(Date.parse(frozenAt) + horizonDays * MS_PER_DAY).toISOString();
+}
+
+function withinHorizon(occurredAt: string, frozenAtMs: number, horizonEndMs: number): boolean {
+  const at = Date.parse(occurredAt);
+  return Number.isFinite(at) && at >= frozenAtMs && at <= horizonEndMs;
+}
+
+/**
+ * Derive the settled, reversal-aware ground-truth set for one snapshot's work units.
+ *
+ * PURE: no clock, no IO, no randomness. Everything is decided from events dated inside the window, which is
+ * what makes the result stable regardless of when extraction runs (asserted as an invariant in the tests).
+ *
+ * `workUnitIds` is the authoritative roster from the snapshot — a work unit with no qualifying event is
+ * emitted as `unresolved` rather than omitted, so the unresolved rate has an honest denominator and a
+ * consumer can tell "nobody acted" apart from "the extractor forgot about it".
+ */
+export function deriveBenchmarkGroundTruth(input: {
+  snapshotRef: string;
+  frozenAt: string;
+  horizonDays: number;
+  workUnitIds: readonly string[];
+  events: readonly RealizedMaintainerEvent[];
+  reversals?: readonly RealizedReversalEvent[] | undefined;
+}): BenchmarkGroundTruthSet {
+  const frozenAtMs = Date.parse(input.frozenAt);
+  const horizonEnd = benchmarkHorizonEnd(input.frozenAt, input.horizonDays);
+  const horizonEndMs = Date.parse(horizonEnd);
+
+  // The SETTLED state is the LAST qualifying action in the window, not the first: a maintainer who labels,
+  // then requests changes, then merges has settled on the merge. Ties on identical timestamps keep the
+  // earlier-listed event, so a caller's stable input order yields a stable label.
+  const settled = new Map<string, RealizedMaintainerEvent>();
+  for (const event of input.events) {
+    if (!withinHorizon(event.occurredAt, frozenAtMs, horizonEndMs)) continue;
+    const current = settled.get(event.workUnitId);
+    if (!current || Date.parse(event.occurredAt) > Date.parse(current.occurredAt)) settled.set(event.workUnitId, event);
+  }
+
+  // A reversal counts only when it lands inside the same horizon AND strictly after the action it overturns
+  // — a reversal predating the settled action reversed something else, and counting it would mislabel an
+  // action nobody has yet overturned. The EARLIEST such reversal is recorded: the first overturning is the
+  // one that makes the action non-confirmed.
+  const reversalFor = (workUnitId: string, settledAt: string): { kind: RealizedReversalKind; occurredAt: string } | null => {
+    let best: RealizedReversalEvent | undefined;
+    for (const reversal of input.reversals ?? []) {
+      if (reversal.workUnitId !== workUnitId) continue;
+      if (!withinHorizon(reversal.occurredAt, frozenAtMs, horizonEndMs)) continue;
+      if (Date.parse(reversal.occurredAt) <= Date.parse(settledAt)) continue;
+      if (!best || Date.parse(reversal.occurredAt) < Date.parse(best.occurredAt)) best = reversal;
+    }
+    return best ? { kind: best.kind, occurredAt: best.occurredAt } : null;
+  };
+
+  const truths: BenchmarkGroundTruth[] = [];
+  let unresolved = 0;
+  for (const workUnitId of input.workUnitIds) {
+    const event = settled.get(workUnitId);
+    if (!event) {
+      unresolved += 1;
+      truths.push({ workUnitId, outcome: "unresolved" });
+      continue;
+    }
+    truths.push({
+      workUnitId,
+      outcome: "settled",
+      action: event.action,
+      // Spreads, not literal `undefined` values: an absent parameter must be OMITTED (the same
+      // exactOptionalPropertyTypes discipline every sibling module follows), so a `merge` truth cannot
+      // carry a present-but-undefined `reasonClass` that a strict comparison would trip over.
+      ...(event.action === "close" && event.reasonClass !== undefined ? { reasonClass: event.reasonClass } : {}),
+      ...(event.action === "label" && event.labels !== undefined ? { labels: [...event.labels] } : {}),
+      settledAt: event.occurredAt,
+      reversal: reversalFor(workUnitId, event.occurredAt),
+    });
+  }
+
+  const workUnits = input.workUnitIds.length;
+  return {
+    schemaVersion: 1,
+    snapshotRef: input.snapshotRef,
+    horizonDays: input.horizonDays,
+    frozenAt: input.frozenAt,
+    horizonEnd,
+    truths,
+    coverage: {
+      workUnits,
+      scoreable: workUnits - unresolved,
+      unresolved,
+      unresolvedRate: workUnits === 0 ? null : Math.round((unresolved / workUnits) * 1000) / 1000,
+    },
+  };
+}
+
+/** The work units a scorer may divide by — `unresolved` never enters the denominator (#9261 requirement 4).
+ *  Exported as the single definition of that rule so no scorer can re-implement it slightly differently. */
+export function scoreableGroundTruths(
+  set: BenchmarkGroundTruthSet,
+): Array<Extract<BenchmarkGroundTruth, { outcome: "settled" }>> {
+  return set.truths.filter((truth): truth is Extract<BenchmarkGroundTruth, { outcome: "settled" }> => truth.outcome === "settled");
+}

packages/loopover-engine/src/index.ts

@@ -184,6 +184,7 @@ export * from "./calibration/reliability-curve.js";
 export * from "./calibration/attestation-envelope.js";
 export * from "./calibration/attester.js";
 export * from "./calibration/benchmark-proposal.js";
+export * from "./calibration/benchmark-ground-truth.js";
 export {
   GOVERNOR_LEDGER_EVENT_TYPES,
   normalizeGovernorLedgerEvent,