feat(orb): add fleet-wide instance health aggregation

The existing readiness check (/health, /ready) is per-instance only and
never leaves the instance -- there was no way for an operator centrally
managing many self-hosted instances to see which ones are actually up.

Threads readiness() (the exact same check /ready already answers with)
into the hourly Orb telemetry export, riding the same request even in a
tick with nothing new to export -- no second, parallel health-check
mechanism. The central ingest handler accepts this as an optional health
field, relaxing the "empty events" rejection only when a health signal
accompanies it, and persists it to two new orb_instances columns
(healthy, health_reported_at) via a COALESCE-based upsert: an
outcome-only export from a build that hasn't upgraded yet never
overwrites a previously-reported status with null, and never looks
healthy just because the instance is otherwise active.

A new getFleetHealthSummary() aggregates healthy/unhealthy/unknown counts
across registered instances, with a staleness window (unresponsive reads
as unknown, not stuck on its last-known state) -- surfaced on the
operator dashboard as "Instance status", named distinctly from the
existing "Fleet health" card (gate-calibration quality, an unrelated
concept that happens to share the word "health").

Closes #4933

Author: github-actions[bot]

apps/loopover-ui/src/routes/app.operator.test.tsx

@@ -170,3 +170,61 @@ describe("OperatorDashboard storage by tenant (#4890)", () => {
     expect(screen.getByText("2 rows")).toBeTruthy();
   });
 });
+
+describe("OperatorDashboard instance status (#4933)", () => {
+  function mockDashboard(fleetHealth?: {
+    healthyCount: number;
+    unhealthyCount: number;
+    unknownCount: number;
+    totalCount: number;
+  }) {
+    useApiResource.mockImplementation((path: string) => {
+      if (path === "/v1/app/operator-dashboard") {
+        return {
+          status: "ready",
+          data: {
+            metrics: [{ label: "Installs", value: "12", delta: "+2" }],
+            noiseReduction: [],
+            weeklyReport: [],
+            fleetHealth,
+          },
+          error: null,
+          loadedAt: "2026-07-17T00:00:00.000Z",
+          reload: () => {},
+        };
+      }
+      return {
+        status: "error",
+        data: null,
+        error: "unavailable in this test",
+        errorKind: "unknown",
+        loadedAt: null,
+        reload: () => {},
+      };
+    });
+  }
+
+  it("renders no section at all when fleetHealth is absent (self-host, the common case)", () => {
+    mockDashboard(undefined);
+    render(<OperatorDashboard />);
+    expect(screen.queryByText("Instance status")).toBeNull();
+  });
+
+  it("renders no section when fleetHealth.totalCount is 0", () => {
+    mockDashboard({ healthyCount: 0, unhealthyCount: 0, unknownCount: 0, totalCount: 0 });
+    render(<OperatorDashboard />);
+    expect(screen.queryByText("Instance status")).toBeNull();
+  });
+
+  it("renders the healthy/unhealthy/unknown counts, distinct from the gate-calibration Fleet health card", () => {
+    mockDashboard({ healthyCount: 3, unhealthyCount: 1, unknownCount: 2, totalCount: 6 });
+    render(<OperatorDashboard />);
+    expect(screen.getByText("Instance status")).toBeTruthy();
+    expect(screen.getByText("Healthy")).toBeTruthy();
+    expect(screen.getByText("3")).toBeTruthy();
+    expect(screen.getByText("Unhealthy")).toBeTruthy();
+    expect(screen.getByText("1")).toBeTruthy();
+    expect(screen.getByText("Unknown")).toBeTruthy();
+    expect(screen.getByText("2")).toBeTruthy();
+  });
+});

apps/loopover-ui/src/routes/app.operator.tsx

@@ -36,6 +36,12 @@ type OperatorDashboardResponse = {
   upstreamDrift?: { status?: string } | null;
   aiCostByTenant?: Array<{ installationId: string; totalCostUsd: number }>;
   storageRowCountByTenant?: Array<{ installationId: string; rowCount: number }>;
+  fleetHealth?: {
+    healthyCount: number;
+    unhealthyCount: number;
+    unknownCount: number;
+    totalCount: number;
+  };
 };
 
 type FleetMetrics = {
@@ -254,6 +260,36 @@ export function OperatorDashboard() {
             </section>
           ) : null}
 
+          {data.fleetHealth && data.fleetHealth.totalCount > 0 ? (
+            <section className="rounded-token border border-border bg-transparent p-5">
+              <div className="flex items-center gap-2">
+                <BarChart3 className="size-4 text-mint" />
+                <h2 className="font-display text-token-lg font-semibold">Instance status</h2>
+              </div>
+              <p className="mt-1 max-w-2xl text-token-xs text-muted-foreground">
+                Fleet instance readiness across {data.fleetHealth.totalCount} registered self-hosted
+                instance(s) — separate from the gate-calibration numbers above.
+              </p>
+              <div className="mt-4 grid gap-3 sm:grid-cols-3">
+                <Stat
+                  label="Healthy"
+                  value={String(data.fleetHealth.healthyCount)}
+                  hint="reported ready"
+                />
+                <Stat
+                  label="Unhealthy"
+                  value={String(data.fleetHealth.unhealthyCount)}
+                  hint="reported not ready"
+                />
+                <Stat
+                  label="Unknown"
+                  value={String(data.fleetHealth.unknownCount)}
+                  hint="no recent report"
+                />
+              </div>
+            </section>
+          ) : null}
+
           {quality ? (
             <section className="rounded-token border border-border bg-transparent p-5">
               <div className="flex flex-wrap items-start justify-between gap-3">

migrations/0166_orb_instances_health.sql

@@ -0,0 +1,6 @@
+-- #4933: fleet-wide instance health aggregation. Distinct from orb_instances.last_seen_at (bumped on EVERY
+-- ingest call, including outcome-only exports from an older self-host build that doesn't send a health
+-- signal at all): healthy/health_reported_at are only set when a payload actually carries one, so an
+-- instance that hasn't upgraded yet stays NULL (unknown) rather than silently reading as healthy.
+ALTER TABLE orb_instances ADD COLUMN healthy INTEGER;
+ALTER TABLE orb_instances ADD COLUMN health_reported_at TEXT;

src/orb/analytics.ts

@@ -248,3 +248,41 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe
     gamingPatternFlags,
   };
 }
+
+/** #4933: fleet-wide instance READINESS, not gate-calibration quality -- deliberately separate from (and
+ *  named differently on the dashboard than) the "Fleet health" gate-precision card above, which this is
+ *  often confused with despite measuring something unrelated. */
+export interface FleetHealthSummary {
+  healthyCount: number;
+  unhealthyCount: number;
+  // Never reported a health status, or its last report is older than HEALTH_STALE_HOURS -- an
+  // unresponsive instance must read as "don't know," not silently keep counting as its last-known state.
+  unknownCount: number;
+  totalCount: number; // registered instances only, matching computeFleetAnalytics's own trust gate
+}
+
+// A bit over 2x the hourly export cron (server.ts's runOrbExport), so one missed tick doesn't immediately
+// flip an instance to "unknown."
+export const HEALTH_STALE_HOURS = 3;
+
+export async function getFleetHealthSummary(env: Env, now: Date = new Date()): Promise<FleetHealthSummary> {
+  const staleBefore = new Date(now.getTime() - HEALTH_STALE_HOURS * 60 * 60 * 1000).toISOString();
+  try {
+    const row = await env.DB.prepare(
+      `SELECT
+         SUM(CASE WHEN healthy = 1 AND health_reported_at IS NOT NULL AND health_reported_at > ? THEN 1 ELSE 0 END) AS healthy_count,
+         SUM(CASE WHEN healthy = 0 AND health_reported_at IS NOT NULL AND health_reported_at > ? THEN 1 ELSE 0 END) AS unhealthy_count,
+         COUNT(*) AS total_count
+       FROM orb_instances
+       WHERE registered = 1`,
+    )
+      .bind(staleBefore, staleBefore)
+      .first<{ healthy_count: number | null; unhealthy_count: number | null; total_count: number }>();
+    const healthyCount = Number(row?.healthy_count ?? 0);
+    const unhealthyCount = Number(row?.unhealthy_count ?? 0);
+    const totalCount = Number(row?.total_count ?? 0);
+    return { healthyCount, unhealthyCount, unknownCount: totalCount - healthyCount - unhealthyCount, totalCount };
+  } catch {
+    return { healthyCount: 0, unhealthyCount: 0, unknownCount: 0, totalCount: 0 };
+  }
+}

src/orb/ingest.ts

@@ -63,6 +63,9 @@ interface OrbIngestEvent {
 interface OrbIngestPayload {
   instance_id: string;
   events: OrbIngestEvent[];
+  // #4933: optional -- an older self-host build that hasn't upgraded yet simply omits this, and the
+  // instance's stored health stays whatever it last was (or NULL/unknown on first contact).
+  health?: { ok: boolean };
 }
 
 export type OrbIngestResult = { accepted: number } | { error: string };
@@ -89,19 +92,35 @@ export async function handleOrbIngest(body: string, db: D1Database): Promise<Orb
     return { error: "invalid_payload" };
   }
 
-  const { instance_id, events } = payload as OrbIngestPayload;
-  if (!instance_id || instance_id.length > MAX_INSTANCE_ID_CHARS || events.length === 0) {
+  const { instance_id, events, health } = payload as OrbIngestPayload;
+  // #4933: an empty batch is only valid when it's carrying a health-only ping (the hourly export still
+  // has to report health even in a tick with nothing new to export) -- a truly empty, health-less payload
+  // stays rejected exactly as before.
+  const healthy = typeof health === "object" && health !== null && typeof health.ok === "boolean" ? (health.ok ? 1 : 0) : null;
+  if (!instance_id || instance_id.length > MAX_INSTANCE_ID_CHARS || (events.length === 0 && healthy === null)) {
     return { error: "invalid_payload" };
   }
+  const healthReportedAt = healthy === null ? null : new Date().toISOString();
 
   // Record the instance on first contact (registered=0 by default) and bump last_seen. The registration
   // gate lives in computeFleetAnalytics: signals are stored for everyone, but only registered instances
   // count toward the fleet median — so open ingest can't be used to skew calibration (the das-github-mirror
   // model: every source is seen, trusted only once an operator opts it in).
+  //
+  // healthy/health_reported_at only move when THIS payload actually reported a health status (COALESCE
+  // falls back to whatever was already stored) -- an outcome-only ingest from a self-host build that
+  // hasn't upgraded to send health yet must never silently overwrite a real prior health reading with
+  // NULL, and must never look "healthy" just because the instance is otherwise active.
   try {
     await db
-      .prepare(`INSERT INTO orb_instances (instance_id) VALUES (?) ON CONFLICT(instance_id) DO UPDATE SET last_seen_at = CURRENT_TIMESTAMP`)
-      .bind(instance_id)
+      .prepare(
+        `INSERT INTO orb_instances (instance_id, healthy, health_reported_at) VALUES (?, ?, ?)
+         ON CONFLICT(instance_id) DO UPDATE SET
+           last_seen_at = CURRENT_TIMESTAMP,
+           healthy = COALESCE(excluded.healthy, orb_instances.healthy),
+           health_reported_at = COALESCE(excluded.health_reported_at, orb_instances.health_reported_at)`,
+      )
+      .bind(instance_id, healthy, healthReportedAt)
       .run();
   } catch {
     // best-effort: never fail ingest because the instance bookkeeping hiccupped

src/selfhost/orb-collector.ts

@@ -51,6 +51,7 @@ interface FleetEvent {
 interface OrbExportPayload {
   instance_id: string;
   events: FleetEvent[];
+  health?: { ok: boolean };
 }
 
 /** Stable instance identifier (hash of the Orb/App ID — no PII). A brokered instance holds no App id, so its
@@ -154,11 +155,14 @@ export function cycleTimeMs(decidedAt: string, outcomeAt: string): number | null
 }
 
 /**
- * Export newly-resolved PR outcomes (since this instance's watermark) to the central collector. Reads from
- * review_audit (de-noised, reversal-aware), anonymizes, signs, POSTs, then advances the cursor.
- * Returns the number of events exported (0 if air-gapped, the App isn't configured, or nothing new).
+ * Export newly-resolved PR outcomes (since this instance's watermark) to the central collector, and --
+ * when `healthOk` is supplied (#4933, threaded from server.ts's own readiness() result) -- a health ping
+ * riding the same request, even in a tick with nothing new to export. Reads from review_audit (de-noised,
+ * reversal-aware), anonymizes, signs, POSTs, then advances the cursor.
+ * Returns the number of events exported (0 if air-gapped, the App isn't configured, or nothing new to
+ * export -- a health-only ping with zero events still returns 0, matching "nothing new" today).
  */
-export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: typeof fetch = fetch): Promise<number> {
+export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: typeof fetch = fetch, healthOk?: boolean): Promise<number> {
   // Air-gapped/offline deployments explicitly suppress every outbound telemetry call, including brokered mode.
   if ((process.env.ORB_AIR_GAP ?? "").toLowerCase() === "true") return 0;
 
@@ -185,11 +189,13 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t
   const cursorTargetId = cursorRow?.last_exported_target_id ?? "";
 
   const { results } = await db.prepare(FLEET_QUERY).bind(cursorAt, cursorAt, cursorTargetId, batchSize).all<FleetRow>();
-  if (!results || results.length === 0) return 0;
+  // A health-only ping (healthOk !== undefined) still has something to send even with zero new events;
+  // otherwise, exactly as before, nothing new means nothing to do.
+  if ((!results || results.length === 0) && healthOk === undefined) return 0;
 
   const payload: OrbExportPayload = {
     instance_id: instance,
-    events: results.map((r) => ({
+    events: (results ?? []).map((r) => ({
       repo_hash: anonymize ? hmacAnonymize(r.project, secret) : r.project,
       pr_hash: anonymize ? hmacAnonymize(r.target_id, secret) : r.target_id,
       gate_verdict: r.verdict,
@@ -200,6 +206,7 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t
       decision_timestamp: r.decided_at,
       outcome_timestamp: r.outcome_at,
     })),
+    ...(healthOk !== undefined ? { health: { ok: healthOk } } : {}),
   };
 
   const body = JSON.stringify(payload);
@@ -235,14 +242,18 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t
   }
 
   // Advance the watermark to the newest event in this batch (rows are ordered by event_at, target_id).
-  const lastRow = results[results.length - 1]!;
-  await db
-    .prepare(
-      `INSERT OR REPLACE INTO orb_export_cursor (instance_hash, last_exported_at, last_exported_target_id, updated_at) VALUES (?, ?, ?, ?)`,
-    )
-    .bind(instance, lastRow.event_at, lastRow.target_id, new Date().toISOString())
-    .run();
+  // A health-only ping (no results) has no watermark to advance -- nothing new was read or sent as events.
+  if (results && results.length > 0) {
+    const lastRow = results[results.length - 1]!;
+    await db
+      .prepare(
+        `INSERT OR REPLACE INTO orb_export_cursor (instance_hash, last_exported_at, last_exported_target_id, updated_at) VALUES (?, ?, ?, ?)`,
+      )
+      .bind(instance, lastRow.event_at, lastRow.target_id, new Date().toISOString())
+      .run();
+  }
 
-  incr("loopover_orb_events_exported_total", {}, results.length);
-  return results.length;
+  const exportedCount = results?.length ?? 0;
+  if (exportedCount > 0) incr("loopover_orb_events_exported_total", {}, exportedCount);
+  return exportedCount;
 }

src/server.ts

@@ -1115,9 +1115,17 @@ async function main(): Promise<void> {
 
   // Orb fleet-telemetry export — ALWAYS ON (the fleet-calibration contract of self-hosting). Self-gates
   // inside exportOrbBatch: a no-op until the GitHub App is configured, or when ORB_AIR_GAP=true.
+  //
+  // #4933: rides the SAME readiness() this instance's own /ready endpoint uses (readinessProbes, built
+  // above) rather than inventing a second, parallel health check -- so "healthy" reported to the fleet
+  // always means exactly what /ready already means locally. A readiness() failure here degrades to "no
+  // health signal this tick" (healthOk stays undefined) rather than reporting a wrong status.
   /* v8 ignore start -- self-host entrypoint timers start a live server; monitor semantics are covered in selfhost tests. */
   const runOrbExport = () =>
-    runOrbExportWithMonitor(() => exportOrbBatch(backend.db)).catch((error) =>
+    runOrbExportWithMonitor(async () => {
+      const health = await readiness(backend.db, readinessProbes).catch(() => null);
+      return exportOrbBatch(backend.db, undefined, undefined, health?.ok);
+    }).catch((error) =>
       console.error(
         JSON.stringify({
           level: "error",

src/services/operator-dashboard.ts

@@ -30,7 +30,7 @@ import type {
   ScoringModelSnapshotRecord,
   WeeklyValueReport,
 } from "../types";
-import { computeFleetAnalytics, type FleetAnalytics } from "../orb/analytics";
+import { computeFleetAnalytics, getFleetHealthSummary, type FleetAnalytics, type FleetHealthSummary } from "../orb/analytics";
 import { computeAgentHealth, computeCalibration, type AgentHealth, type Calibration } from "../review/ops";
 import { computeGateEval, type GateEvalReport } from "../review/parity";
 import { computeCycleTimeAggregate, computeFindingAcceptance, type CycleTimeAggregate } from "../review/stats";
@@ -102,6 +102,9 @@ export type OperatorDashboardPayload = {
   // D1 storage cap already has alerting (src/selfhost/d1-size-probe.ts); this is the per-installation dimension
   // that alerting doesn't have. Same self-host-always-empty caveat as aiCostByTenant above.
   storageRowCountByTenant: RowCountByTenant[];
+  // #4933: fleet-wide instance READINESS (up/down/unknown), distinct from fleetMetrics above (gate-calibration
+  // quality). Always all-zero for a self-host operator (no registered peer instances).
+  fleetHealth: FleetHealthSummary;
 };
 
 const USAGE_WINDOW_DAYS = 7;
@@ -140,6 +143,7 @@ export async function buildOperatorDashboardPayload(
     findingAcceptance,
     aiCostByTenant,
     storageRowCountByTenant,
+    fleetHealth,
   ] = await Promise.all([
     listRepositories(env),
     listInstallations(env),
@@ -171,6 +175,8 @@ export async function buildOperatorDashboardPayload(
     listAiCostByTenantSince(env, usageSince),
     // #4890 (re-scoped): per-tenant row-count breakdown, same window as the rest of the usage metrics above.
     listRowCountByTenantSince(env, usageSince),
+    // #4933: fleet-wide instance readiness -- a point-in-time summary, no window needed.
+    getFleetHealthSummary(env),
   ]);
   const weeklyValueReport = buildWeeklyValueReport({
     generatedAt: nowIso(),
@@ -291,6 +297,7 @@ export async function buildOperatorDashboardPayload(
     acceptance,
     aiCostByTenant,
     storageRowCountByTenant,
+    fleetHealth,
   };
 }