feat(review): record MCP predicted-gate verdicts and measure predicted-vs-live gate agreement (#4558)

Records every gittensory_predict_gate/explain_gate_disposition call into a
new local-only, login-keyed predicted_gate_calls table (mirroring
contributor_gate_history's 0126 precedent -- review_audit stays
deliberately actor-login-free, per its own migration's privacy design).

A new sibling to computeGateEval/computeGateParity, computePredictedGateAgreement,
joins predicted_gate_calls against the already-existing contributor_gate_history
by (project, login, timestamp) -- the correlation key predict-time calls can
actually provide, since predictGateShape has no PR-number field pre-submission
-- and folds the result into a per-project agreement/disagreement confusion
matrix, surfaced via a new GET /v1/internal/predicted-agreement endpoint
alongside the existing /v1/internal/parity one.

Fixes #4516

Author: JSONbored

migrations/0134_predicted_gate_calls.sql

@@ -0,0 +1,43 @@
+-- #predicted-live-gate-agreement (maintainer review-stack x AMS integration audit, 2026-07-09): the data
+-- substrate for measuring how often the MCP `gittensory_predict_gate`/`gittensory_explain_gate_disposition`
+-- verdict agrees with the REAL gate decision the same contributor's PR later receives.
+--
+-- WHY A NEW TABLE, NOT A `review_audit` ROW: `predictGateShape` has no PR-number field (it is an explicit
+-- pre-PR-existence dry run), so a predicted call cannot be keyed `project#pr` the way recordNativeGateDecision
+-- keys a real gate_decision -- there is no PR yet to key against. The only reliable correlation key available
+-- at predict-time is (project, login, timestamp), which means CORRELATING a predicted call to its eventual
+-- real PR requires a login-keyed join. `review_audit` (migrations/0049) is DELIBERATELY actor-login-free
+-- ("No actor logins... ONLY") specifically because it feeds the anonymized cross-instance orb-collector export
+-- (src/selfhost/orb-collector.ts) -- exactly the same reason migrations/0126 (contributor_gate_history) is its
+-- own separate, local-only table rather than a review_audit column. This table follows that identical
+-- precedent: a SEPARATE, LOCAL-ONLY, login-keyed substrate, never wired into exportOrbBatch or any other
+-- cross-instance/public export path. See src/review/predicted-gate-calls.ts for the writer and
+-- src/review/predicted-gate-agreement.ts for the reader, which joins this table against the ALREADY-EXISTING
+-- contributor_gate_history (0126) -- the login-keyed real-decision data that table already records -- rather
+-- than duplicating the real side of the comparison into a second copy.
+--
+-- Privacy: this table is per-login by design (see migrations/0126's identical rationale for why login, not a
+-- hash, is fine for a LOCAL-ONLY table). Any output DERIVED from it (the agreement-rate metric) must remain
+-- aggregated -- never render which login contributed which paired row on any public/contributor-facing surface.
+CREATE TABLE IF NOT EXISTS predicted_gate_calls (
+  id TEXT PRIMARY KEY NOT NULL,
+  -- The GitHub login the prediction was requested for (the miner's own `login` input to predict_gate).
+  login TEXT NOT NULL,
+  -- Which repo the prediction is for.
+  project TEXT NOT NULL,
+  -- The predicted gate action: 'merge' | 'hold' (nativeGateActionFromConclusion's mapping -- the predicted-gate
+  -- engine never predicts 'close', mirroring the live gate: it is a CHECK that passes or blocks, never closes).
+  predicted_action TEXT NOT NULL,
+  -- The raw predicted verdict conclusion (success/failure/action_required/neutral), kept alongside the
+  -- collapsed predicted_action for observability -- e.g. distinguishing a hard blocker from an inconclusive hold.
+  conclusion TEXT NOT NULL,
+  -- Bounded reason-class code for the predicted verdict (mirrors review_audit.summary / neutralHoldReasonCode),
+  -- never a raw finding title/detail (which can embed contributor- or per-repo-controlled text).
+  reason_code TEXT,
+  created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- The read side (computePredictedGateAgreement) scans "this project's predicted calls in a recency window,
+-- grouped by login" to pair each against contributor_gate_history's real decisions.
+CREATE INDEX IF NOT EXISTS predicted_gate_calls_project_login_idx
+  ON predicted_gate_calls(project, login, created_at);

scripts/check-schema-drift.mjs

@@ -50,6 +50,7 @@ export const RAW_SQL_ONLY_TABLES = new Set([
   "orb_signals",
   "orb_webhook_events",
   "override_audit",
+  "predicted_gate_calls",
   "repo_chunks",
   "review_audit",
   "review_targets",

src/api/routes.ts

@@ -255,6 +255,7 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration";
 import { loadGatePrecisionReport } from "../services/gate-precision";
 import { computeOpsStats, isOpsEnabled } from "../review/ops-wire";
 import { computeParityReadiness, isParityAuditEnabled } from "../review/parity-wire";
+import { computePredictedGateAgreement } from "../review/predicted-gate-agreement";
 import { isRagEnabled } from "../review/rag-wire";
 import { getPublicStats, isPublicStatsEnabled } from "../review/public-stats";
 import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard";
@@ -3498,6 +3499,18 @@ export function createApp() {
     return c.json(await computeParityReadiness(c.env));
   });
 
+  // #predicted-live-gate-agreement (maintainer review-stack x AMS integration audit, 2026-07-09): how often the
+  // MCP predict_gate/explain_gate_disposition verdict agrees with the REAL gate decision a contributor's PR
+  // later receives -- a DIFFERENT question than /v1/internal/parity's reviewbot-vs-gittensory migration parity
+  // (see src/review/predicted-gate-agreement.ts's module header). Same gate/auth contract as /v1/internal/parity:
+  // bearer-gated by the `/v1/internal/*` middleware, 404 when GITTENSORY_REVIEW_PARITY_AUDIT is off so the
+  // endpoint does not exist on a deploy not running this telemetry family. Aggregate counts only — no PR
+  // content / actor logins (see that module's privacy note on why a per-login breakdown never belongs here).
+  app.get("/v1/internal/predicted-agreement", async (c) => {
+    if (!isParityAuditEnabled(c.env)) return c.json({ error: "not_found" }, 404);
+    return c.json(await computePredictedGateAgreement(c.env, { days: 90, nowMs: Date.now() }));
+  });
+
   app.post("/v1/internal/jobs/refresh-registry", async (c) => {
     const message: JobMessage = { type: "refresh-registry", requestedBy: "api" };
     await c.env.JOBS.send(message);

src/mcp/server.ts

@@ -162,6 +162,7 @@ import { loadUpstreamStatus } from "../upstream/ruleset";
 import { simulateOpenPrPressure, type OpenPrPressureInput } from "../services/open-pr-pressure-scenarios";
 import { buildFindingTaxonomyDocument, FINDING_TAXONOMY_URI } from "../review/finding-taxonomy";
 import { buildEnrichmentAnalyzersTaxonomyDocument, ENRICHMENT_ANALYZERS_URI } from "../review/enrichment-analyzers-taxonomy";
+import { recordPredictedGateCall } from "../review/predicted-gate-calls";
 
 type AppContext = Context<{ Bindings: Env }>;
 type ToolPayload = {
@@ -2957,6 +2958,13 @@ export class GittensoryMcp {
       confirmedContributor,
       ...(input.changedPaths === undefined ? {} : { changedPaths: input.changedPaths }),
     });
+    // #predicted-live-gate-agreement: record this call so a later real gate decision for the same
+    // (repo, login) can be paired against it (src/review/predicted-gate-agreement.ts). Shared by BOTH
+    // predictGate and explainGateDisposition (this function backs both tools) -- a caller that invokes both
+    // for what is really one logical check records two rows, a small, acceptable volume over-count rather
+    // than threading a request-scoped dedup key through a read-only prediction path. Best-effort; never
+    // blocks or fails the tool response.
+    await recordPredictedGateCall(this.env, { login: input.login, project: repoFullName, verdict });
     return { repoFullName, verdict };
   }
 

src/review/predicted-gate-agreement.ts

@@ -0,0 +1,75 @@
+// Predicted-gate call history (#predicted-live-gate-agreement, maintainer review-stack x AMS integration
+// audit 2026-07-09) -- records EVERY MCP `gittensory_predict_gate`/`gittensory_explain_gate_disposition` call,
+// so a later real gate decision for the same (project, login) can be paired against it (see
+// src/review/predicted-gate-agreement.ts for the read/join side). Structurally a sibling of
+// src/review/contributor-calibration.ts: `review_audit` (migrations/0049) is DELIBERATELY actor-login-free
+// (feeds the anonymized orb-collector export), so this is its own separate, LOCAL-ONLY table
+// (migrations/0132) -- never wired into exportOrbBatch or any other cross-instance/public export path.
+//
+// UNLIKE contributor-calibration.ts's per-commit dedup (a re-run at the same head_sha replaces its prior row),
+// every predict_gate call gets its OWN row here: there is no commit to dedup against pre-submission, and a
+// miner iterating on the same repo (tweaking a title, retrying after a blocker) makes a genuinely new inquiry
+// each time -- collapsing them would undercount how often the tool was actually consulted.
+
+import { isParityAuditEnabled, nativeGateActionFromConclusion } from "./parity-wire";
+import type { GateCheckConclusion } from "../rules/advisory";
+import { isSelfHostedReviewRuntime } from "../selfhost/review-runtime";
+import { errorMessage, nowIso } from "../utils/json";
+
+/** The minimal env shape the recorder needs -- mirrors parity-wire.ts's ParityRecorderEnv / contributor-
+ *  calibration.ts's ContributorCalibrationEnv exactly, since this records under the identical self-hosted/
+ *  parity-flag gate (one flag controls the whole gate-accuracy telemetry family). */
+type PredictedGateCallEnv = {
+  DB: D1Database;
+  GITTENSORY_REVIEW_PARITY_AUDIT?: string | undefined;
+  SELFHOST_TRANSIENT_CACHE?: NonNullable<Env["SELFHOST_TRANSIENT_CACHE"]>;
+};
+
+/** The minimal verdict shape this recorder needs -- structurally compatible with PredictedGateVerdict
+ *  (packages/gittensory-engine), whose `blockers` entries are the public-safe shape (no `severity`), unlike
+ *  the real gate's AdvisoryFinding -- so this reads only `.code`, never reusing neutralHoldReasonCode's
+ *  stricter AdvisoryFinding-typed signature (see the reasonCode comment below for why that's an acceptable,
+ *  deliberately coarser fallback on the predicted side). */
+type RecordablePredictedVerdict = {
+  conclusion: GateCheckConclusion;
+  blockers: Array<{ code: string }>;
+};
+
+/**
+ * Record one MCP predict_gate/explain_gate_disposition call into `predicted_gate_calls`, keyed by the
+ * requested contributor's login. Gated identically to {@link recordNativeGateDecision} in parity-wire.ts (same
+ * self-hosted-always-records / cloud-flag-gated contract) -- this is additive telemetry alongside the same
+ * gate-accuracy measurement family, not a separate feature with its own on/off knob.
+ *
+ * Best-effort: a write failure is swallowed (telemetry must never break the MCP tool response). A missing/
+ * empty login records nothing -- there is no meaningful per-actor row to write without one.
+ */
+export async function recordPredictedGateCall(
+  env: PredictedGateCallEnv,
+  input: { login: string | null | undefined; project: string; verdict: RecordablePredictedVerdict },
+): Promise<void> {
+  if (!isSelfHostedReviewRuntime(env) && !isParityAuditEnabled(env)) return;
+  const login = input.login?.trim();
+  if (!login) return;
+  const action = nativeGateActionFromConclusion(input.verdict.conclusion);
+  if (action === null) return; // "skipped" -- not a comparable prediction (mirrors recordNativeGateDecision)
+  const project = input.project.slice(0, 200);
+  // Coarser than the real gate_decision's summary (which recovers a specific neutral-hold sub-code via
+  // neutralHoldReasonCode): the predicted-gate engine's public verdict shape carries no `severity` on its
+  // findings, so it isn't AdvisoryFinding-shaped and can't reuse that stricter-typed helper. reason_code here
+  // is an observability aid only (not read by computePredictedGateAgreement's core comparison), so the bare
+  // conclusion string is an acceptable fallback for every non-failure case.
+  const reasonCode = input.verdict.conclusion === "failure" ? (input.verdict.blockers[0]?.code ?? input.verdict.conclusion) : input.verdict.conclusion;
+  try {
+    // Every call gets its own row (no dedup key) -- see the module header for why, unlike
+    // recordContributorGateDecision's per-commit replace.
+    await env.DB.prepare(
+      `INSERT INTO predicted_gate_calls (id, login, project, predicted_action, conclusion, reason_code, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`,
+    )
+      .bind(`predicted:${login}:${project}:${nowIso()}:${Math.random().toString(36).slice(2, 8)}`, login, project, action, input.verdict.conclusion, reasonCode.slice(0, 200), nowIso())
+      .run();
+  } catch (error) {
+    console.warn(JSON.stringify({ event: "predicted_gate_calls_record_error", project, message: errorMessage(error).slice(0, 200) }));
+  }
+}

src/review/predicted-gate-calls.ts

@@ -228,4 +228,50 @@ testExpectations:
     expect(result.isError).toBe(true);
     expect(JSON.stringify(result.content)).toContain("authenticated GitHub login");
   });
+
+  describe("records the call for predicted-vs-live agreement measurement (#predicted-live-gate-agreement)", () => {
+    async function rawAll(env: Env, sql: string): Promise<Record<string, unknown>[]> {
+      const res = await (env.DB as unknown as { prepare: (s: string) => { all: <T>() => Promise<{ results: T[] }> } }).prepare(sql).all<Record<string, unknown>>();
+      return res.results;
+    }
+
+    it("SELF-HOSTED instances record a predicted_gate_calls row on a successful predict_gate call", async () => {
+      const env = createTestEnv(); // SELFHOST_TRANSIENT_CACHE present by default → self-hosted
+      const client = await connect(env);
+
+      await client.callTool({
+        name: "gittensory_predict_gate",
+        arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" },
+      });
+
+      const rows = await rawAll(env, "SELECT * FROM predicted_gate_calls");
+      expect(rows).toHaveLength(1);
+      expect(rows[0]).toMatchObject({ login: "miner1", project: "acme/widgets" });
+    });
+
+    it("also records from gittensory_explain_gate_disposition (both tools share computePredictedGateVerdict)", async () => {
+      const env = createTestEnv();
+      const client = await connect(env);
+
+      await client.callTool({
+        name: "gittensory_explain_gate_disposition",
+        arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" },
+      });
+
+      expect(await rawAll(env, "SELECT * FROM predicted_gate_calls")).toHaveLength(1);
+    });
+
+    it("records NOTHING on the CLOUD WORKER when GITTENSORY_REVIEW_PARITY_AUDIT is unset (byte-identical default)", async () => {
+      const env = createTestEnv();
+      delete env.SELFHOST_TRANSIENT_CACHE; // simulate the cloud worker
+      const client = await connect(env);
+
+      await client.callTool({
+        name: "gittensory_predict_gate",
+        arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Add retry to upload client" },
+      });
+
+      expect(await rawAll(env, "SELECT * FROM predicted_gate_calls")).toHaveLength(0);
+    });
+  });
 });