fix(selfhost): close the dead-letter alerting blind spot and require alerting setup

alertmanager/alertmanager.yml

@@ -6,6 +6,13 @@
 # "null" receiver that discards notifications. Nothing pages you until you opt in by
 # uncommenting one of the example receivers below and pointing the route at it. This
 # means `docker compose --profile observability up -d` always comes up green.
+#
+# THIS IS A REQUIRED STEP for any deployment you expect to run unattended, not an
+# optional one — see the self-hosting Operations doc's "Alerting" section for the
+# fastest verified path (a Discord webhook, uncomment the block below) and for the
+# Grafana "Active Alerts" panel you can check manually until you do. Until a real
+# receiver is wired up, every rule in prometheus/rules/alerts.yml can fire and nobody
+# will be notified — only an operator who opens Grafana will see it.
 
 global:
   # How long to wait before declaring a firing alert "resolved" once it stops arriving.

apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx

@@ -95,6 +95,32 @@ review_context_fetch_failed`}
 docker compose --profile postgres --profile observability --profile backup up -d`}
       />
 
+      <h2>Alerting — required for a 24/7 deployment</h2>
+      <p>
+        Alertmanager ships with a valid but <strong>silent</strong> default: every alert routes to a
+        name-only receiver that discards it, so{" "}
+        <code>docker compose --profile observability up -d</code> always starts clean even before
+        you've configured anywhere to send notifications. This is intentional — the shipped config
+        can&apos;t bake in a Slack/Discord/email destination that works for everyone — but it means
+        nothing pages anyone until you edit <code>alertmanager/alertmanager.yml</code> yourself.
+        Treat this as a required step, not an optional one, for any deployment you expect to run
+        unattended.
+      </p>
+      <p>
+        The fastest verified path: create a Discord channel webhook (channel settings → Integrations
+        → Webhooks → New Webhook), then uncomment the <code>discord</code> receiver block in{" "}
+        <code>alertmanager/alertmanager.yml</code> and point the root route at it. Slack, email, and
+        a generic webhook receiver (for PagerDuty or a custom handler) are also ready to uncomment
+        in the same file.
+      </p>
+      <p>
+        Until you do, alerts are still visible without any extra setup: open Grafana and check the{" "}
+        <strong>Alerts</strong> row on the main dashboard, which lists every currently-firing alert
+        directly from Prometheus, independent of Alertmanager routing. Use this as your fallback
+        check if you haven&apos;t wired up push notifications yet — it&apos;s exactly what the{" "}
+        <code>Dead jobs stay at zero</code> routine check below is watching for.
+      </p>
+
       <h2>Sentry tracing</h2>
       <p>
         Leave <code>SENTRY_TRACES_SAMPLE_RATE</code> unset or blank to disable trace export, or set

prometheus/rules/alerts.yml

@@ -83,6 +83,23 @@ groups:
           description: "The dead-letter queue holds {{ $value | printf \"%.0f\" }} jobs (sustained 10m). Work is accumulating with no automatic recovery."
           runbook: "Drain or triage the dead-letter queue. If it only ever grows, fix the root cause before requeuing or you'll just re-fill it."
 
+      - alert: GittensoryDeadLetterBacklogPersisting
+        # The two rules above have a coverage gap: the rate alert (increase() over 15m)
+        # only fires in a narrow window around a job's transition INTO dead, and the
+        # backlog alert only trips above 50. A single dead job — the common case, e.g. one
+        # job hit a since-fixed bug during a deploy — sits in the table forever after that
+        # 15m window closes, invisible to both. This is a LEVEL check with no count floor:
+        # any non-zero dead-letter count sustained for an hour pages, regardless of how it
+        # got there or how large it is.
+        expr: gittensory_queue_dead > 0
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "gittensory dead-letter queue not empty for over an hour"
+          description: "{{ $value | printf \"%.0f\" }} job(s) have sat in the dead-letter queue for over an hour with no automatic recovery."
+          runbook: "Check whether the underlying bug is already fixed and redeployed — if so the job(s) can be requeued; otherwise triage before requeuing."
+
   # ── Queue backlog (live processing pressure) ───────────────────────────────
   - name: gittensory-queue
     rules: