feat(analytics): surface org-wide slop-band calibration + add the dashboard card (#2196) (#5126)

Wire the existing, unit-tested buildSlopOutcomeCalibration over
listAllPullRequests into the operator-dashboard payload as slopCalibration:
org-wide per-band merge/close rates over resolved PRs carrying a persisted
slop band, plus whether the deterministic slop score discriminates. The read
is wrapped in a buildOrgSlopCalibration() helper that fails safe to an empty
calibration on any read error (listAllPullRequests can throw, unlike the
sibling reads) so one DB hiccup never fails the whole dashboard — with both
the success and fail-safe branches unit-tested. Adds SlopBandCalibrationCard
(per-band merge-rate bars + predictive/inverted/insufficient verdict, bands
only, never raw scores) across all-bands, empty-band, inverted, no-data, and
absent-field branches.

Author: dhgoal

apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx

@@ -0,0 +1,76 @@
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+
+import { SlopBandCalibrationCard } from "@/components/site/app-panels/slop-band-calibration-card";
+
+describe("SlopBandCalibrationCard", () => {
+  it("renders a row per band with merge rate + counts and the discrimination verdict when all bands have data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 40,
+          overallMergeRate: 0.6,
+          discriminates: true,
+          bands: [
+            { band: "clean", sampleSize: 20, merged: 18, closed: 2, mergeRate: 0.9 },
+            { band: "low", sampleSize: 10, merged: 6, closed: 4, mergeRate: 0.6 },
+            { band: "elevated", sampleSize: 6, merged: 2, closed: 4, mergeRate: 0.333 },
+            { band: "high", sampleSize: 4, merged: 0, closed: 4, mergeRate: 0 },
+          ],
+        }}
+      />,
+    );
+    expect(screen.getByText("clean")).toBeTruthy();
+    expect(screen.getByText("high")).toBeTruthy();
+    expect(screen.getByText("90% merged")).toBeTruthy();
+    expect(screen.getByText("predictive")).toBeTruthy();
+    expect(screen.getByText(/40 resolved · 60% merged overall/)).toBeTruthy();
+  });
+
+  it("shows the '— no samples' state for an empty band and marks insufficient data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 3,
+          overallMergeRate: 1,
+          discriminates: null,
+          bands: [
+            { band: "clean", sampleSize: 3, merged: 3, closed: 0, mergeRate: 1 },
+            { band: "high", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 },
+          ],
+        }}
+      />,
+    );
+    expect(screen.getByText("— no samples")).toBeTruthy();
+    expect(screen.getByText("insufficient data")).toBeTruthy();
+  });
+
+  it("flags an inverted (non-predictive) score", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 20,
+          overallMergeRate: 0.5,
+          discriminates: false,
+          bands: [{ band: "clean", sampleSize: 20, merged: 10, closed: 10, mergeRate: 0.5 }],
+        }}
+      />,
+    );
+    expect(screen.getByText("inverted")).toBeTruthy();
+  });
+
+  it("shows the 'no resolved PRs' empty state when the calibration has no resolved data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{ totalResolved: 0, overallMergeRate: null, discriminates: null, bands: [] }}
+      />,
+    );
+    expect(screen.getByText("No resolved PRs with a slop band")).toBeTruthy();
+    expect(screen.queryByText("predictive")).toBeNull();
+  });
+
+  it("shows the 'not yet available' empty state when the calibration field is absent", () => {
+    render(<SlopBandCalibrationCard />);
+    expect(screen.getByText("Not yet available")).toBeTruthy();
+  });
+});

apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx

@@ -0,0 +1,106 @@
+import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell";
+import { StatusPill, type Status } from "@/components/site/control-primitives";
+import { cn } from "@/lib/utils";
+
+/** Slop-band calibration (#2196): per-band merge/close rates over resolved PRs that carry a persisted slop band —
+ *  is the deterministic slop score predictive (do higher-slop bands merge less)? Display slice over the
+ *  operator-dashboard's `slopCalibration` payload. Bands only, never raw scores (public/private boundary). */
+export type SlopBand = "clean" | "low" | "elevated" | "high";
+
+export type SlopBandCalibration = {
+  band: SlopBand;
+  sampleSize: number;
+  merged: number;
+  closed: number;
+  mergeRate: number;
+};
+
+export type SlopOutcomeCalibration = {
+  totalResolved: number;
+  bands: SlopBandCalibration[];
+  overallMergeRate: number | null;
+  discriminates: boolean | null;
+};
+
+/** clean → high reads low-to-high severity; the bar color tracks band severity. */
+const BAND_BAR: Record<SlopBand, string> = {
+  clean: "bg-success",
+  low: "bg-mint",
+  elevated: "bg-warning",
+  high: "bg-danger",
+};
+
+function discriminationPill(discriminates: boolean | null): { status: Status; label: string } {
+  if (discriminates === true) return { status: "ready", label: "predictive" };
+  if (discriminates === false) return { status: "degraded", label: "inverted" };
+  return { status: "info", label: "insufficient data" };
+}
+
+function BandRow({ row }: { row: SlopBandCalibration }) {
+  const hasSamples = row.sampleSize > 0;
+  const pct = hasSamples ? Math.round(row.mergeRate * 100) : null;
+  return (
+    <div className="space-y-1.5">
+      <div className="flex items-center justify-between gap-3 text-token-sm">
+        <span className="font-medium capitalize text-foreground">{row.band}</span>
+        <span className="font-mono text-token-xs text-muted-foreground">
+          {pct === null ? "— no samples" : `${pct}% merged`}
+        </span>
+      </div>
+      <div className="h-2 overflow-hidden rounded-full bg-border" aria-hidden>
+        {hasSamples ? (
+          <div className={cn("h-full", BAND_BAR[row.band])} style={{ width: `${pct}%` }} />
+        ) : null}
+      </div>
+      <div className="font-mono text-token-2xs text-muted-foreground">
+        {row.sampleSize} resolved · {row.merged} merged · {row.closed} closed
+      </div>
+    </div>
+  );
+}
+
+export function SlopBandCalibrationCard({ calibration }: { calibration?: SlopOutcomeCalibration }) {
+  const hasData =
+    calibration != null && calibration.totalResolved > 0 && calibration.bands.length > 0;
+
+  if (!hasData) {
+    return (
+      <AnalyticsCardShell
+        title="Slop-band calibration"
+        description="Predicted slop band vs realized merge/close outcome, across resolved PRs."
+        state="empty"
+        emptyTitle={calibration ? "No resolved PRs with a slop band" : "Not yet available"}
+        emptyHint={
+          calibration
+            ? "Calibration appears once resolved PRs carry a persisted slop band in the window."
+            : "Slop-band calibration appears once the payload includes resolved-PR slop bands."
+        }
+      />
+    );
+  }
+
+  const pill = discriminationPill(calibration.discriminates);
+  const overall =
+    calibration.overallMergeRate === null ? null : Math.round(calibration.overallMergeRate * 100);
+
+  return (
+    <AnalyticsCardShell
+      title="Slop-band calibration"
+      description="Predicted slop band vs realized merge/close outcome, across resolved PRs."
+      state="ready"
+    >
+      <div className="flex flex-wrap items-center justify-between gap-2">
+        <span className="font-mono text-token-2xs text-muted-foreground">
+          {calibration.totalResolved} resolved
+          {overall === null ? "" : ` · ${overall}% merged overall`}
+        </span>
+        <StatusPill status={pill.status}>{pill.label}</StatusPill>
+      </div>
+      <div className="mt-3 space-y-4">
+        {calibration.bands.map((row) => (
+          <BandRow key={row.band} row={row} />
+        ))}
+      </div>
+    </AnalyticsCardShell>
+  );
+}

apps/gittensory-ui/src/routes/app.analytics.tsx

@@ -28,6 +28,10 @@ import {
   FindingsBreakdownCard,
   type FindingsBreakdown,
 } from "@/components/site/app-panels/findings-breakdown-card";
+import {
+  SlopBandCalibrationCard,
+  type SlopOutcomeCalibration,
+} from "@/components/site/app-panels/slop-band-calibration-card";
 import { useApiResource } from "@/lib/api/use-api-resource";
 import { exportOperatorDashboardCsv } from "@/lib/csv-export";
 
@@ -124,6 +128,7 @@ type OperatorDashboard = {
   agentHealth?: ReversalHealth;
   acceptance?: FindingAcceptance;
   findingsBreakdown?: FindingsBreakdown;
+  slopCalibration?: SlopOutcomeCalibration;
 };
 
 function ProductAnalytics() {
@@ -232,6 +237,8 @@ function ProductAnalytics() {
 
           <FindingsBreakdownCard findings={data.findingsBreakdown} />
 
+          <SlopBandCalibrationCard calibration={data.slopCalibration} />
+
           {data.usageSummary ? (
             <ProductUsageBreakdownPanel
               byEvent={data.usageSummary.byEvent}

src/services/operator-dashboard.ts

@@ -4,6 +4,7 @@ import {
   getCommandUsefulnessSummary,
   getLatestScoringModelSnapshot,
   getProductUsageRollupStatus,
+  listAllPullRequests,
   listInstallationHealth,
   listInstallations,
   listLatestGitHubRateLimitObservations,
@@ -32,6 +33,7 @@ import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/st
 import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset";
 import { nowIso } from "../utils/json";
 import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report";
+import { buildSlopOutcomeCalibration, type SlopOutcomeCalibration } from "./outcome-calibration";
 import { buildWeeklyValueReport } from "./weekly-value-report";
 
 export type OperatorDashboardMetric = {
@@ -71,6 +73,9 @@ export type OperatorDashboardPayload = {
   calibration: Calibration;
   // Agent reversal health (#2193): how often humans reopened/reverted bot auto-actions (ops.ts AgentHealth).
   agentHealth: AgentHealth;
+  // Slop-band calibration (#2196): org-wide per-band merge/close rates over resolved PRs carrying a persisted
+  // slop band — is the deterministic slop score predictive? Bands only, never raw scores. Fails safe to empty.
+  slopCalibration: SlopOutcomeCalibration;
 };
 
 const USAGE_WINDOW_DAYS = 7;
@@ -98,6 +103,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     cycleTime,
     calibration,
     agentHealth,
+    slopCalibration,
   ] = await Promise.all([
     listRepositories(env),
     listInstallations(env),
@@ -121,6 +127,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     computeCycleTimeAggregate(env, { days: 90, nowMs: Date.now() }),
     computeCalibration(env, operatorAgentConfig(env)),
     computeAgentHealth(env, operatorAgentConfig(env)),
+    buildOrgSlopCalibration(env),
   ]);
   const weeklyValueReport = buildWeeklyValueReport({
     generatedAt: nowIso(),
@@ -224,9 +231,21 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     cycleTime,
     calibration,
     agentHealth,
+    slopCalibration,
   };
 }
 
+/** #2196: org-wide slop-band calibration from persisted slop bands on resolved PRs. `listAllPullRequests` can
+ *  throw (unlike the sibling reads, which fail safe internally), so this wraps it and degrades to an empty
+ *  calibration on any read error — one DB hiccup must never fail the whole dashboard build. */
+async function buildOrgSlopCalibration(env: Env): Promise<SlopOutcomeCalibration> {
+  try {
+    return buildSlopOutcomeCalibration(await listAllPullRequests(env));
+  } catch {
+    return buildSlopOutcomeCalibration([]);
+  }
+}
+
 function operatorAgentConfig(env: Env): { slug: string; secrets: Record<string, never> } {
   const slug =
     typeof env.GITHUB_APP_SLUG === "string" && env.GITHUB_APP_SLUG.trim()
@@ -235,7 +254,7 @@ function operatorAgentConfig(env: Env): { slug: string; secrets: Record<string,
   return { slug, secrets: {} };
 }
 
-export const __operatorDashboardInternals = { operatorAgentConfig };
+export const __operatorDashboardInternals = { operatorAgentConfig, buildOrgSlopCalibration };
 
 export function latestUsageRollup(rollups: ProductUsageDailyRollupRecord[]): ProductUsageDailyRollupRecord | null {
   if (rollups.length === 0) return null;

test/unit/operator-dashboard.test.ts

@@ -50,6 +50,12 @@ describe("operator dashboard payload", () => {
       recentAutoActions: 0,
       reversedTargets: [],
     });
+    // #2196: org-wide slop-band calibration fails safe to an empty calibration when no resolved PR carries a band.
+    expect(payload.slopCalibration).toMatchObject({
+      totalResolved: 0,
+      overallMergeRate: null,
+      discriminates: null,
+    });
     // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta.
     expect(payload.fleetMetrics.instanceCount).toBe(0);
     expect(payload.metrics).toEqual(
@@ -103,6 +109,26 @@ describe("operator dashboard payload", () => {
     });
   });
 
+  it("buildOrgSlopCalibration (#2196) degrades to an empty calibration when the PR read throws", async () => {
+    const { buildOrgSlopCalibration } = __operatorDashboardInternals;
+    // A DB whose every access throws forces listAllPullRequests to reject; the fail-safe must swallow it.
+    const brokenDb = new Proxy(
+      {},
+      {
+        get() {
+          throw new Error("DB unavailable");
+        },
+      },
+    );
+    const brokenEnv = { ...createTestEnv(), DB: brokenDb as unknown as D1Database };
+    const calibration = await buildOrgSlopCalibration(brokenEnv);
+    expect(calibration).toMatchObject({
+      totalResolved: 0,
+      overallMergeRate: null,
+      discriminates: null,
+    });
+  });
+
   it("surfaces populated fleet metrics + outliers from orb_signals", async () => {
     const env = createTestEnv();
     let n = 0;