obs(selfhost): add Cloudflare D1 size/row-count probe, metrics, alerts, panel (#4153)

Advances #3810 -- the dedup/retention fix (dedupeSignalSnapshots) already
shipped in #3857; this covers the remaining, deliberately-deferred half:
observability for the ~10GB D1 storage cap that incident hit.

Adds an opt-in Cloudflare Management API probe (src/selfhost/d1-size-probe.ts,
gated on CLOUDFLARE_D1_MONITOR_ACCOUNT_ID/DATABASE_ID/API_TOKEN -- absent on
almost every self-host install, which runs its own SQLite/Postgres backend)
that polls D1's file size and RETENTION_POLICY's monitored tables' row counts
every 15 minutes from the self-host process, publishing
gittensory_d1_database_size_bytes, gittensory_d1_table_row_count{table=...},
and gittensory_signal_snapshots_rows_per_key (scoped to the latest-only-dedup
signal types dedupeSignalSnapshots converges to ~1 row per key, so a climbing
ratio means that job has stopped running or its allowlist regressed). Size and
each table's row count are fetched independently and a failure in one never
blanks the other or the previous good reading, recorded via
gittensory_d1_probe_errors_total.

Adds four Prometheus alerts (D1 size warn ~70% / critical ~90% of the ~10GB
cap, the signal_snapshots dedup-regression ratio, and probe-failure) and a
"Cloudflare D1 (Central Cloud)" row on the self-host Grafana dashboard.

Author: JSONbored

apps/gittensory-ui/src/lib/selfhost-env-reference.ts

@@ -65,6 +65,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "CLAUDE_AI_TIMEOUT_MS",
     firstReference: "src/selfhost/ai.ts",
   },
+  {
+    name: "CLOUDFLARE_D1_MONITOR_ACCOUNT_ID",
+    firstReference: "src/selfhost/d1-size-probe.ts",
+  },
+  {
+    name: "CLOUDFLARE_D1_MONITOR_API_TOKEN",
+    firstReference: "src/selfhost/d1-size-probe.ts",
+  },
+  {
+    name: "CLOUDFLARE_D1_MONITOR_DATABASE_ID",
+    firstReference: "src/selfhost/d1-size-probe.ts",
+  },
   {
     name: "CODEX_AI_EFFORT",
     firstReference: "src/selfhost/ai.ts",
@@ -409,6 +421,9 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts` |",
   "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts` |",
   "| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts` |",
+  "| `CLOUDFLARE_D1_MONITOR_ACCOUNT_ID` | `src/selfhost/d1-size-probe.ts` |",
+  "| `CLOUDFLARE_D1_MONITOR_API_TOKEN` | `src/selfhost/d1-size-probe.ts` |",
+  "| `CLOUDFLARE_D1_MONITOR_DATABASE_ID` | `src/selfhost/d1-size-probe.ts` |",
   "| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts` |",
   "| `CODEX_AI_FIRST_OUTPUT_TIMEOUT_MS` | `src/selfhost/ai.ts` |",
   "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts` |",

grafana/dashboards/gittensory.json

@@ -3182,6 +3182,159 @@
           "refId": "A"
         }
       ]
+    },
+    {
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 222 },
+      "id": 200,
+      "title": "Cloudflare D1 (Central Cloud, #3810)",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 7000000000 },
+              { "color": "red", "value": 9000000000 }
+            ]
+          },
+          "unit": "decbytes"
+        }
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 223 },
+      "id": 201,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "textMode": "auto"
+      },
+      "title": "D1 Database Size (-1 = probe disabled/no sample yet)",
+      "description": "Cloudflare D1 file size for the monitored database, from the opt-in Management API probe (src/selfhost/d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars). -1 means the probe is disabled or has never completed a successful sample. D1's known per-database cap is ~10GB (#3810).",
+      "type": "stat",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_d1_database_size_bytes",
+          "legendFormat": "size",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 10 },
+              { "color": "red", "value": 50 }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 223 },
+      "id": 202,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "textMode": "auto"
+      },
+      "title": "signal_snapshots Rows per Dedup Key (-1 = unavailable)",
+      "description": "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots (src/db/retention.ts) converges to ~1 row per key. A climbing value means the daily dedup job has stopped running or its allowlist regressed -- the 2026-07-06 incident ratio was ~157 (342243 rows / 2183 keys).",
+      "type": "stat",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_signal_snapshots_rows_per_key",
+          "legendFormat": "rows/key",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 223 },
+      "id": 203,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "textMode": "auto"
+      },
+      "title": "D1 Probe Errors (total)",
+      "description": "Cloudflare Management API probe failures (src/selfhost/d1-size-probe.ts), labeled by part (database_info/table_row_count). Climbing while the size/row-count panels stay flat means the PROBE is broken (bad/expired token, wrong account or database id), not that the database stopped growing.",
+      "type": "stat",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum(gittensory_d1_probe_errors_total) or vector(0)",
+          "legendFormat": "errors",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 227 },
+      "id": 204,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "D1 Database Size & Monitored Table Row Counts",
+      "description": "Trend view of the same D1 size/row-count probe as the stat panels above -- database bytes on the left axis scale, monitored-table row counts (RETENTION_POLICY's tables, src/db/retention.ts) labeled by table.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_d1_database_size_bytes",
+          "legendFormat": "database bytes",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "gittensory_d1_table_row_count",
+          "legendFormat": "{{table}} rows",
+          "refId": "B"
+        }
+      ]
     }
   ],
   "refresh": "30s",
@@ -3206,5 +3359,5 @@
   "timezone": "browser",
   "title": "Gittensory Self-Host",
   "uid": "gittensory-selfhost",
-  "version": 9
+  "version": 10
 }

prometheus/rules/alerts.yml

@@ -552,3 +552,66 @@ groups:
           summary: "gittensory host clock skew is CRITICAL ({{ $value | printf \"%.0f\" }}s) -- GitHub App auth is likely failing"
           description: "Clock skew has exceeded 120s (sustained 2m), well past the point GitHub App JWT auth (\"Bad credentials\") is expected to start failing fleet-wide."
           runbook: "Same as GittensoryClockSkewWarning, but treat as urgent: fix NTP sync immediately (chronyc sources, chronyc makestep, restart chrony if every source stays at Reach: 0). Check for github_app_jwt_rejected logs to confirm auth impact."
+
+  # ── Cloudflare D1 (central cloud) size + signal_snapshots dedup regression (#3810) ────
+  # gittensory_d1_* metrics come from the OPT-IN Cloudflare Management API probe (src/selfhost/
+  # d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars) -- absent/disabled reads -1 on every gauge below,
+  # comfortably under every threshold here, so these rules never fire on an install that hasn't configured
+  # the probe (most self-host installs run their own SQLite/Postgres backend and have nothing to monitor).
+  - name: gittensory-d1-storage
+    rules:
+      - alert: GittensoryD1DatabaseSizeWarning
+        # 7e9 bytes is ~70% of D1's known ~10GB per-database cap that was hit on 2026-07-06 (#3810), which
+        # caused real D1 writes (including an Orb relay registration handshake) to start failing/timing out.
+        expr: gittensory_d1_database_size_bytes > 7000000000
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Cloudflare D1 database size is approaching its cap"
+          description: "The monitored D1 database is {{ $value | humanize }}B, over 70% of the ~10GB per-database cap (sustained 30m)."
+          runbook: "Check the D1 panel's per-table row counts (Cloudflare D1 (Central Cloud) row) and confirm the daily signal_snapshots dedup job (dedupeSignalSnapshots, wired into prune-retention) is actually running. See #3810 for the 2026-07-06 incident this guards against."
+
+      - alert: GittensoryD1DatabaseSizeCritical
+        # 9e9 bytes is ~90% of the ~10GB cap -- D1 writes started failing fleet-wide the last time the
+        # database actually hit the cap, so this is meant to fire well before that happens again.
+        expr: gittensory_d1_database_size_bytes > 9000000000
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Cloudflare D1 database size is CRITICAL -- writes may start failing soon"
+          description: "The monitored D1 database is {{ $value | humanize }}B, over 90% of the ~10GB per-database cap (sustained 10m). D1 writes failed fleet-wide the last time this cap was hit (2026-07-06)."
+          runbook: "Immediate: verify the signal_snapshots dedup job is running (POST /v1/internal/retention/preview, or check audit_events for its record) and identify + trim/archive any other unbounded table from the row-count panel. Contact Cloudflare to raise the account storage limit if cleanup alone doesn't recover enough headroom."
+
+      - alert: GittensorySignalSnapshotsDedupRegression
+        # gittensory_signal_snapshots_rows_per_key is rows-per-distinct-key scoped ONLY to the four
+        # latest-only-dedup signal types dedupeSignalSnapshots (src/db/retention.ts) actually converges to
+        # ~1 row per key -- NOT the whole signal_snapshots table, which intentionally keeps bounded
+        # multi-row history for other signal types (queue-health, contributor-decision-pack, ...). Healthy
+        # steady-state stays a small multiple of 1 (rows can accumulate for up to a day between the daily
+        # dedup run); 10 is a wide margin above that, while the actual 2026-07-06 incident ratio (342243
+        # rows / 2183 keys) was ~157 -- so a value anywhere near double digits means the dedup job has
+        # stopped running, started erroring, or its allowlist regressed.
+        expr: gittensory_signal_snapshots_rows_per_key > 10
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: "signal_snapshots dedup-by-key ratio is climbing"
+          description: "signal_snapshots has {{ $value | printf \"%.1f\" }} rows per distinct dedup key (sustained 30m) -- the daily dedupeSignalSnapshots job may not be running."
+          runbook: "Confirm the prune-retention cron (03:00 UTC daily) is completing (audit_events around that time) and that dedupeSignalSnapshots isn't throwing. See #3810 and src/db/retention.ts."
+
+      - alert: GittensoryD1ProbeFailing
+        # The probe itself (a Cloudflare Management API call) can fail independently of the database it
+        # monitors -- an expired/rotated API token, a wrong account/database id, or a Cloudflare API outage.
+        # Without this, an operator would see a flat/stale D1 panel and could mistake "the probe broke" for
+        # "the database stopped growing".
+        expr: increase(gittensory_d1_probe_errors_total[1h]) > 3
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "the Cloudflare D1 size/row-count probe is failing"
+          description: "{{ $value | printf \"%.0f\" }} D1 Management API probe failure(s) over the last 1h (sustained 15m, part={{ $labels.part }}). The size/row-count gauges below may be stale."
+          runbook: "Check CLOUDFLARE_D1_MONITOR_API_TOKEN is still valid and CLOUDFLARE_D1_MONITOR_ACCOUNT_ID/DATABASE_ID are correct. Tail logs for level=error event=d1_size_probe_error."

src/db/retention.ts

@@ -94,7 +94,12 @@ export async function pruneExpiredRecords(
 
 export type SignalSnapshotDedupeResult = { signalType: string; deleted: number };
 
-const LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES = [
+/** Exported so the D1 size/row-count observability probe (#3810, src/selfhost/d1-size-probe.ts) can scope its
+ *  signal_snapshots "rows per dedup key" ratio to exactly the population this dedup job converges to ~1 row
+ *  per key -- NOT the whole table, which intentionally keeps bounded multi-row history for other signal
+ *  types (queue-health, contributor-decision-pack, ...). Single source of truth: if this list changes, the
+ *  probe's ratio scope changes with it automatically. */
+export const LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES = [
   "repo-culture-profile",
   "repo-doc-refresh-attempt",
   "repo-focus-manifest",

src/env.d.ts

@@ -379,6 +379,19 @@ declare global {
      *  flagged (see the same-author guard in buildCollisionReport). Default OFF — unset/false leaves every
      *  PullRequestRecord's changedFiles unset, byte-identical to today. See src/signals/engine.ts prItem. */
     GITTENSORY_OPEN_PR_FILE_COLLISION?: string;
+    /** D1 size/row-count observability probe (#3810): the Cloudflare account id that owns the D1 database to
+     *  monitor. Presence of this AND the two vars below IS the enablement switch (see isD1SizeProbeEnabled,
+     *  src/selfhost/d1-size-probe.ts) -- unset/blank ⇒ the probe never runs, byte-identical to today. Most
+     *  self-host operators run their own SQLite/Postgres backend and have no Cloudflare D1 to watch; this is
+     *  for whichever deployment owns a real D1 worth monitoring (including gittensory's own central cloud
+     *  database, the one that hit its ~10GB cap on 2026-07-06). */
+    CLOUDFLARE_D1_MONITOR_ACCOUNT_ID?: string;
+    /** The D1 database id (uuid) to monitor. See CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */
+    CLOUDFLARE_D1_MONITOR_DATABASE_ID?: string;
+    /** A Cloudflare API token with read access to D1 for the account above (a scoped, read-only custom
+     *  token is sufficient — this probe never writes). A secret — never commit a real value. See
+     *  CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */
+    CLOUDFLARE_D1_MONITOR_API_TOKEN?: string;
   }
 }