alerts.yml

# Prometheus alerting rules for the loopover self-host stack (#980 observability).
#
# Mounted into the prometheus container at /etc/prometheus/rules/alerts.yml and loaded
# via the `rule_files: ["/etc/prometheus/rules/*.yml"]` glob in prometheus.yml.
#
# Every rule below is grounded ONLY in metrics the loopover app actually exports at
# GET /metrics, plus the synthetic `up` metric Prometheus emits per scrape target. The
# exceptions are the final three groups -- loopover-miner-prediction (#5188),
# loopover-miner-portfolio-queue (#5186), and loopover-miner-governor (#5187) -- which
# target the loopover-miner's OWN separate scrape surface (see each group's own comment
# for why it still ships here and stays dormant until a miner scrape target exists).
#
# Thresholds are sane defaults for a SMALL single-host self-host. Tune the numbers in
# `expr` / `for` to your traffic — each is commented with what it means and how to adjust.
#
# Severity convention (consumed by Alertmanager routing/inhibition):
#   severity: critical  → page / wake someone up; the service is down or losing data.
#   severity: warning   → look soon; degraded but not (yet) an outage.
#
# Annotation convention:
#   summary     → one-line headline (templated with the offending series' labels).
#   description → what tripped + the live value, for the notification body.
#   runbook     → a short, actionable first-response hint (no external URL needed).

groups:
  # ── Scrape target liveness ────────────────────────────────────────────────
  - name: loopover-availability
    rules:
      - alert: LoopoverTargetDown
        # `up` is 0 when Prometheus cannot scrape /metrics (process down, crash-loop,
        # network partition, or wrong port). Scoped to our job so node/other targets
        # don't trip this. 2m absorbs a single missed scrape + a restart.
        expr: up{job="loopover"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "loopover target {{ $labels.instance }} is down"
          description: "Prometheus has failed to scrape {{ $labels.instance }} (job {{ $labels.job }}) for over 2m. The app is unreachable or not serving /metrics."
          runbook: "Check `docker compose ps` and `docker compose logs loopover`. Look for a missing selfhost_listening log line or a crash-loop (restart count climbing)."

  # ── Job queue / worker health ─────────────────────────────────────────────
  - name: loopover-jobs
    rules:
      - alert: LoopoverHighJobFailureRatio
        # Fraction of jobs ATTEMPTED (failed + succeeded) that failed over the last 10m.
        # loopover_jobs_processed_total only increments on success, so the denominator
        # must be failed+processed, not processed alone (#3892) -- dividing by processed
        # alone computes failed:success and can read as high as 100% at a true 50% failure
        # rate. The `> 0` guard on the ratio avoids a 0/0 = NaN (which never fires but is
        # noise in /rules) when no jobs ran at all. 0.10 = 10% of jobs failing. Raise if
        # your workload has expected transient failures; lower if any failure is a real
        # problem. Mirrors the Grafana "Job Failure Rate" panel's formula.
        expr: |
          (
            sum(rate(loopover_jobs_failed_total[10m]))
            /
            (
              sum(rate(loopover_jobs_failed_total[10m]))
              +
              sum(rate(loopover_jobs_processed_total[10m]))
            ) > 0
          ) > 0.10
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "loopover job failure ratio above 10%"
          description: "{{ $value | humanizePercentage }} of jobs attempted in the last 10m failed (sustained 15m). Expected: well under 10%."
          runbook: "Tail logs for level=error job events (e.g. selfhost_cron_error). A spike usually means a bad upstream (GitHub API / AI provider / DB) or a poison payload — check what changed."

      - alert: LoopoverDeadLetterJobsGrowing
        # Jobs exhausted all retries and landed in the dead-letter queue. ANY new dead
        # job in 15m is worth a look — these are lost work that won't self-heal.
        # `increase(...) > 0` fires on the first dead job in the window.
        expr: increase(loopover_jobs_dead_total[15m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover dead-letter jobs increasing"
          description: "{{ $value | printf \"%.0f\" }} job(s) moved to the dead-letter queue in the last 15m. These exhausted all retries and need manual attention."
          runbook: "Inspect loopover_queue_dead gauge and the dead-letter records. Decide whether to fix-and-requeue or discard. A steady climb points at a systematic failure, not a blip."

      - alert: LoopoverDeadLetterBacklogHigh
        # Standing size of the dead-letter queue. Distinct from the rate alert above:
        # this catches a large backlog that built up before alerting was watching.
        # 50 dead jobs is a generous default for a small host — tune down if you expect ~0.
        expr: loopover_queue_dead > 50
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover dead-letter backlog above 50"
          description: "The dead-letter queue holds {{ $value | printf \"%.0f\" }} jobs (sustained 10m). Work is accumulating with no automatic recovery."
          runbook: "Drain or triage the dead-letter queue. If it only ever grows, fix the root cause before requeuing or you'll just re-fill it."

      - alert: LoopoverDeadLetterBacklogPersisting
        # The two rules above have a coverage gap: the rate alert (increase() over 15m)
        # only fires in a narrow window around a job's transition INTO dead, and the
        # backlog alert only trips above 50. A single dead job — the common case, e.g. one
        # job hit a since-fixed bug during a deploy — sits in the table forever after that
        # 15m window closes, invisible to both. This is a LEVEL check with no count floor:
        # any non-zero dead-letter count sustained for an hour pages, regardless of how it
        # got there or how large it is.
        expr: loopover_queue_dead > 0
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "loopover dead-letter queue not empty for over an hour"
          description: "{{ $value | printf \"%.0f\" }} job(s) have sat in the dead-letter queue for over an hour with no automatic recovery."
          runbook: "Check whether the underlying bug is already fixed and redeployed — if so the job(s) can be requeued; otherwise triage before requeuing."

  # ── Queue backlog (live processing pressure) ───────────────────────────────
  - name: loopover-queue
    rules:
      - alert: LoopoverQueueBacklogHigh
        # Pending (not-yet-processed) jobs. A sustained high pending count means workers
        # can't keep up with enqueue rate. 100 is a starting line for a single-node host;
        # raise it if your normal steady-state pending depth is higher.
        expr: loopover_queue_pending > 100
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover queue backlog above 100"
          description: "{{ $value | printf \"%.0f\" }} jobs pending for over 10m — the worker is falling behind the enqueue rate."
          runbook: "Compare rate(loopover_jobs_enqueued_total) vs rate(loopover_jobs_processed_total). If enqueue > processed persistently, the worker is the bottleneck (slow AI/DB, or it's stuck). Check worker logs."

  # ── Maintenance-job backpressure (#selfhost-runtime-pressure) ──────────────
  - name: loopover-maintenance-pressure
    rules:
      - alert: LoopoverLiveQueueStarved
        # Live/foreground work (webhooks, per-PR re-gates) is the whole point of the review stack -- it
        # should clear in seconds to low minutes. A sustained old live job means something OTHER than
        # maintenance admission is the bottleneck (AI latency, GitHub, DB, or genuine host CPU pressure),
        # since maintenance-admission.ts already yields to live work at claim time.
        expr: loopover_queue_oldest_live_pending_age_seconds > 300
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover live/webhook work is stuck behind something other than maintenance"
          description: "The oldest live-priority queue job has been pending for {{ $value | printf \"%.0f\" }}s. Maintenance admission already yields to live work, so check AI latency, GitHub rate limits, DB pressure, or host CPU (loopover_host_load_avg1_per_core)."
          runbook: "Open the Runtime Pressure & Maintenance row. If loopover_host_load_avg1_per_core is elevated, a co-located CI runner or other host process is starving the app -- see docker-compose.yml's runner isolation guidance."

      - alert: LoopoverLiveQueueNoRunnableWork
        # #selfhost-queue-liveness: the precise "is the foreground lane actually stuck" signal, distinct from
        # LoopoverLiveQueueStarved's age-based check -- a large live-pending count on its own is NORMAL (a
        # legitimate burst, or work intentionally staggered/rate-deferred); this only fires when there is
        # PENDING live work AND NOTHING runnable right now, sustained. That combination is exactly the
        # production incident this invariant exists to make impossible: hundreds of pending contributor-PR-
        # review jobs, zero processing, zero runnable, previously requiring manual intervention to unstick.
        expr: loopover_queue_live_pending > 0 and loopover_queue_live_runnable_now == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "loopover foreground/live queue has pending work but nothing runnable"
          description: "{{ $value | printf \"%.0f\" }} live-priority job(s) are pending but loopover_queue_live_runnable_now has read 0 for over 3m -- contributor PR review work is not making progress."
          runbook: "Open the Foreground Liveness row. Check loopover_jobs_rate_limit_admission_deferred_total / loopover_jobs_rate_limit_budget_deferred_total for a stuck GitHub rate-limit bucket, and loopover_jobs_foreground_liveness_released_total for whether the liveness sweep is already recovering it (releases should show up within FOREGROUND_LIVENESS_CHECK_INTERVAL_MS, default 1m). If releases are firing repeatedly with no lasting effect, the underlying rate-limit exhaustion is the real bottleneck, not the queue."

      - alert: LoopoverForegroundLivenessReleasing
        # Informational: the liveness sweep IS working (foreground work would otherwise be stuck), but a
        # sustained rate means something keeps re-deferring the SAME class of work past the trickle ceiling --
        # worth investigating even though the invariant is holding.
        expr: rate(loopover_jobs_foreground_liveness_released_total[15m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "loopover foreground-liveness sweep is repeatedly force-releasing deferred work"
          description: "The foreground-liveness sweep has released stale-deferred jobs for over 15m — the invariant is holding, but something is repeatedly pushing foreground work past FOREGROUND_LIVENESS_MAX_DEFER_MS."
          runbook: "Check loopover_jobs_rate_limit_admission_deferred_total / loopover_jobs_rate_limit_budget_deferred_total by key_scope for a chronically exhausted GitHub REST budget (often installation-scoped after a burst)."

      - alert: LoopoverMaintenanceStarved
        # Maintenance admission (maintenance-admission.ts) has TWO age escapes (#selfhost-maintenance-self-pin):
        # a short maintenanceDrainAgeMs trickle that lets old jobs through even while `maintenance_pending_high`
        # is breached (bounded further by the queue's own background concurrency cap), and the ultimate
        # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS (default 4h) force-admit that applies regardless of pressure --
        # so under correct operation this should never sit much past that outer ceiling, and in practice the
        # drain escape should keep it well under it. A value well beyond 4h means either an escape isn't
        # triggering (a bug) or the host is so overloaded even a force-admitted job can't be claimed/processed.
        expr: loopover_queue_oldest_maintenance_pending_age_seconds > 21600
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "loopover maintenance work has not run in over 6h"
          description: "The oldest maintenance-lane queue job has been pending for {{ $value | printf \"%.0f\" }}s, past the default trickle ceiling. Contributor evidence, RAG indexing, drift scans, and similar sweeps are stale."
          runbook: "Check loopover_jobs_maintenance_admission_deferred_by_reason_total for the dominant defer reason and loopover_jobs_maintenance_admission_granted_under_pressure_total to confirm the escapes are actually firing, and confirm MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS / MAINTENANCE_ADMISSION_DRAIN_AGE_MS weren't raised. Sustained host_load_high suggests the box itself (not just this app) is overloaded -- host_load_high also blocks the drain escape specifically, see maintenance-admission.ts."

  # ── GitHub API budget / queue admission pressure ──────────────────────────
  - name: loopover-github-rate-limits
    rules:
      - alert: LoopoverGitHubRateLimitResponses
        # Live 403/429 GitHub rate-limit responses mean admission/caching did not avoid
        # a depleted bucket. This should be rare; sustained values explain hard queue stalls.
        expr: sum by (status, retry, key_scope) (rate(loopover_github_rest_rate_limit_responses_total[5m])) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "loopover is receiving GitHub REST rate-limit responses"
          description: "GitHub REST rate-limit responses are occurring for over 2m. Check labels status/retry/key_scope on loopover_github_rest_rate_limit_responses_total."
          runbook: "Open the GitHub REST Rate Limits panel and compare with Queue Rate-Limit Deferrals. If responses are installation-scoped, lower concurrent GitHub work for that installation; if key_scope=none/other, identify the unaffiliated caller and route it through an admission key."

      - alert: LoopoverQueueRateLimitDeferralsHigh
        # Jobs are being intentionally parked to preserve GitHub budget. That is better
        # than burning through the bucket, but a sustained storm means throughput is bound
        # by GitHub admission rather than worker capacity.
        expr: |
          sum by (kind, key_scope, job_type) (rate(loopover_jobs_rate_limit_admission_deferred_total[5m])) > 0.05
          or
          sum by (kind, key_scope, job_type) (rate(loopover_jobs_rate_limit_budget_deferred_total[5m])) > 0.05
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover queue is rate-limit deferring jobs"
          description: "GitHub rate-limit deferrals exceeded 0.05/s for 10m. The queue is protecting the bucket but throughput is GitHub-budget bound."
          runbook: "Use loopover_jobs_rate_limit_* labels (kind/key_scope/job_type) to identify the stuck class. Prefer reducing duplicate webhook/regate work or adding cache coverage before raising worker concurrency."

  # ── Postgres database + backup freshness ───────────────────────────────────
  - name: loopover-postgres
    rules:
      - alert: LoopoverPostgresConnectionPressure
        # Postgres exporter is present only for Postgres-backed installs. Fire when the app DB
        # consumes more than 80% of max_connections for 10m; use PgBouncer or lower app concurrency.
        expr: |
          (
            sum(pg_stat_activity_count{datname="loopover"})
            /
            max(pg_settings_max_connections)
          ) > 0.80
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Postgres connection usage is above 80%"
          description: "The loopover database is using {{ $value | humanizePercentage }} of max_connections for over 10m."
          runbook: "Enable the pgbouncer profile or reduce self-host worker/app concurrency. Check the Postgres Connections panel by state before increasing max_connections."

      - alert: LoopoverPostgresLockWaits
        # Wait-event labels are emitted by the postgres_exporter stat_activity collector. Any sustained
        # lock wait means a write path is blocked and queue processing can appear stuck.
        expr: sum(pg_stat_activity_count{datname="loopover", wait_event_type="Lock"}) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Postgres sessions are waiting on locks"
          description: "{{ $value | printf \"%.0f\" }} loopover database session(s) have waited on Postgres locks for over 5m."
          runbook: "Inspect the Postgres Locks and Slow Transactions panels. Look for long transactions, migration jobs, or stuck writers before restarting workers."

      - alert: LoopoverPostgresSlowTransaction
        # Long active transactions are the self-host proxy for slow/problem queries without requiring
        # pg_stat_statements or superuser-only setup.
        expr: max(pg_stat_activity_max_tx_duration{datname="loopover"}) > 120
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Postgres transaction has been active for over 2m"
          description: "The longest active loopover database transaction is {{ $value | printf \"%.0f\" }}s old."
          runbook: "Check app logs around DB-heavy jobs and review whether queue workers are saturating Postgres. A single long transaction can block cleanup or migrations."

      - alert: LoopoverPostgresDeadlocks
        # Deadlocks should be effectively zero; even a low rate points at competing write paths.
        expr: rate(pg_stat_database_deadlocks{datname="loopover"}[5m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Postgres deadlocks detected"
          description: "Postgres deadlocks are occurring for the loopover database."
          runbook: "Correlate with deploys and queue write paths. Repeated deadlocks usually mean a transaction ordering bug or too much parallel write pressure."

      - alert: LoopoverPostgresDatabaseGrowingFast
        # Growth over a 6h window smooths normal small-table churn while catching runaway queue/audit growth.
        # 262144 bytes/s is roughly 5.4 GiB over 6h.
        expr: deriv(pg_database_size_bytes{datname="loopover"}[6h]) > 262144
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Postgres database is growing quickly"
          description: "The loopover database is growing at about {{ $value | humanize }} bytes/s over a 6h trend window."
          runbook: "Open the Postgres Table Growth panel and inspect queue/audit/event tables first. Confirm retention and backup volume capacity before raising traffic."

      - alert: LoopoverPostgresDeadTuplesHigh
        # Autovacuum pressure: high dead tuple ratio and enough absolute dead rows to avoid tiny-table noise.
        expr: |
          (
            sum(pg_stat_user_tables_n_dead_tup{datname="loopover"})
            /
            clamp_min(
              sum(pg_stat_user_tables_n_live_tup{datname="loopover"} + pg_stat_user_tables_n_dead_tup{datname="loopover"}),
              1
            )
          ) > 0.20
          and
          sum(pg_stat_user_tables_n_dead_tup{datname="loopover"}) > 10000
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Postgres dead tuples are high"
          description: "More than 20% of table tuples are dead and at least 10k dead tuples remain for over 30m."
          runbook: "Check the Dead Tuples / Autovacuum panel. If autovacuum counts are flat while dead tuples rise, reduce write pressure or tune autovacuum for the largest tables."

      - alert: LoopoverBackupMissing
        # Fires only when the backup-exporter service is running (backup profile enabled) and no retained
        # DB backup exists yet. Qdrant backups are best-effort and not included in this DB backup alert.
        expr: loopover_backup_files{target=~"postgres|sqlite"} == 0
        for: 2h
        labels:
          severity: warning
        annotations:
          summary: "No retained loopover database backup found"
          description: "The backup profile is exposing metrics, but {{ $labels.target }} has no retained backup file after 2h."
          runbook: "Run `docker compose --profile backup run --rm backup sh /backup.sh` and inspect the loopover-backups volume. For Postgres, confirm DATABASE_URL is available to the backup service."

      - alert: LoopoverBackupStale
        # Default backup loop is daily. 26h allows one missed scrape/restart window before warning.
        expr: |
          (time() - loopover_backup_latest_timestamp_seconds{target=~"postgres|sqlite"} > 93600)
          and
          loopover_backup_latest_timestamp_seconds{target=~"postgres|sqlite"} > 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "loopover database backup is stale"
          description: "The newest retained {{ $labels.target }} backup is older than 26h."
          runbook: "Check `docker compose --profile backup logs backup` and run the backup command manually. Pair this with the restore verification drill before deleting old backups."

  # ── Qdrant vector backend ──────────────────────────────────────────────────
  - name: loopover-qdrant
    rules:
      - alert: LoopoverQdrantErrorRateHigh
        # Qdrant errors (across upsert/query/delete) relative to total Qdrant traffic.
        # `... or vector(0)` keeps the denominator non-empty so the ratio is defined even
        # when query/upsert counters haven't been created yet. > 0.05 = 5% error rate.
        expr: |
          (
            sum(rate(loopover_qdrant_errors_total[10m]))
            /
            (
              sum(rate(loopover_qdrant_queries_total[10m]))
              + sum(rate(loopover_qdrant_upserts_total[10m]))
              + sum(rate(loopover_qdrant_errors_total[10m]))
              > 0
            )
          ) > 0.05
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover Qdrant error rate above 5%"
          description: "{{ $value | humanizePercentage }} of Qdrant operations errored over the last 10m (sustained 10m). RAG retrieval/indexing is degraded."
          runbook: "Check the qdrant container (`docker compose --profile qdrant ps/logs`) and loopover_qdrant_errors_total{op=...} to see whether upsert, query, or delete is failing. A reachable-but-erroring Qdrant often means a schema/collection or disk problem."

  # ── Orb usage-event export pipeline ────────────────────────────────────────
  - name: loopover-orb
    rules:
      - alert: LoopoverOrbExportErrorRateHigh
        # Failed Orb event exports vs total export attempts (errors + successes).
        # The `> 0` denominator guard avoids 0/0. Export failures mean billed usage
        # events aren't reaching Orb — a revenue/billing-integrity issue, so it warns
        # even at a modest 5% rate.
        expr: |
          (
            sum(rate(loopover_orb_export_errors_total[15m]))
            /
            (
              sum(rate(loopover_orb_events_exported_total[15m]))
              + sum(rate(loopover_orb_export_errors_total[15m]))
              > 0
            )
          ) > 0.05
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "loopover Orb export error rate above 5%"
          description: "{{ $value | humanizePercentage }} of Orb usage-event exports failed over the last 15m (sustained 15m). Billable events may not be reaching Orb."
          runbook: "Verify Orb API credentials/connectivity and check level=error logs around orb export. A sustained rise in loopover_orb_export_errors_total while loopover_orb_events_exported_total stays flat confirms a stuck exporter."

      - alert: LoopoverWebhookEnqueueFailures
        # A webhook accepted by the receiver but not durably queued becomes GitHub
        # redelivery pressure and can look like queue jitter. Any sustained enqueue
        # failures deserve attention.
        expr: sum(rate(loopover_webhook_enqueue_total{result="enqueue_failed"}[5m])) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "loopover webhook enqueue failures detected"
          description: "Webhook enqueue failures are occurring for over 2m. Labels event/action show which GitHub event class is failing."
          runbook: "Check WEBHOOKS queue availability and the durable queue backend. If this coincides with Orb relay drains, inspect loopover_orb_webhook_total{result=\"enqueue_failed\"}."

      - alert: LoopoverOrbRelayRegistrationStuck
        # #selfhost-runtime-drift follow-up: mirrors isOrbRelayRegistrationAlerting's own gate (3 consecutive
        # registration failures, OR the pull-mode drain loop gone quiet for 30m) so Prometheus escalates on
        # EXACTLY the same evidence the app itself uses to decide "actually stuck" vs "one hiccup, still
        # draining fine". A single transient registration timeout will not trip this on its own.
        expr: loopover_orb_relay_register_consecutive_failures >= 3 or loopover_orb_relay_drain_seconds_since_last > 1800
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover orb relay registration looks actually stuck, not just a transient hiccup"
          description: "Either the consecutive registration-failure streak has reached {{ $value | printf \"%.0f\" }}, or the pull-mode drain loop hasn't completed in over 30m. A lone registration timeout alone would not trigger this."
          runbook: "Check loopover_orb_relay_register_total{result=\"failed\"} by mode for the failure pattern, and confirm ORB_BROKER_URL / ORB_ENROLLMENT_SECRET are still valid. If pull mode, verify the drain loop itself isn't crash-looping (selfhost_orb_relay_register_failed logs at level=error)."

  # ── HTTP serving health (status label + duration histogram, both live in src/server.ts) ─
  # loopover_http_requests_total carries a status="2xx|3xx|4xx|5xx" label (seeded at zero per
  # class so every series exists from boot), and loopover_http_request_duration_seconds is a
  # histogram of request duration. Both rules below still tolerate an absent metric/label on an
  # older running build: the ratio uses a `> 0` denominator guard, and an absent series simply
  # yields no result (rule stays inactive) rather than erroring.
  - name: loopover-http
    rules:
      - alert: LoopoverHighHttp5xxRatio
        # Share of responses that are 5xx. status="5xx" selects only server errors;
        # the denominator sums all statuses. If the status label is missing entirely
        # (old build), the numerator selector matches nothing → no firing. 0.05 = 5%.
        #
        # Real-incident finding (2026-07-18): a quiet self-host install can see well
        # under 1 req/5m. At that volume a single stray 5xx alone swings the ratio to
        # 100% and pages on noise, not a real incident -- confirmed live (loopover
        # logs showed no errors at all while this fired repeatedly). The `and` clause
        # below requires at least 20 total requests in the same 5m window before the
        # ratio is trusted -- derived directly from the 5% threshold itself: with
        # fewer than 20 requests, one single 5xx response ALREADY exceeds 5% on
        # sample-size noise alone, before the failure rate means anything.
        expr: |
          (
            (
              sum(rate(loopover_http_requests_total{status="5xx"}[5m]))
              /
              sum(rate(loopover_http_requests_total[5m])) > 0
            ) > 0.05
          )
          and
          sum(increase(loopover_http_requests_total[5m])) >= 20
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "loopover HTTP 5xx ratio above 5%"
          description: "{{ $value | humanizePercentage }} of HTTP responses were 5xx over the last 5m (sustained 10m). The API is throwing server errors."
          runbook: "Tail logs for level=error around the request path. Correlate with deploys, DB availability, and AI-provider outages. If 5xx coincides with a target restart, it may be a crash-loop — see LoopoverTargetDown."

      - alert: LoopoverRequestLatencySLOBreach
        # p95 request latency from the duration histogram. histogram_quantile over the
        # per-le bucket rate gives the 95th percentile end-to-end. 1s p95 is a reasonable
        # SLO for a review API; raise for heavier AI-bound endpoints. `by (le)` is
        # required for histogram_quantile to work across bucket series.
        expr: |
          histogram_quantile(
            0.95,
            sum(rate(loopover_http_request_duration_seconds_bucket[5m])) by (le)
          ) > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover p95 request latency above 1s"
          description: "p95 HTTP request latency is {{ $value | printf \"%.2f\" }}s over the last 5m (sustained 10m), breaching the 1s SLO."
          runbook: "Check whether slowness is queue/DB/AI-bound: correlate with loopover_queue_pending and Qdrant/AI latency. A rising p95 with flat error rate usually means a saturated dependency, not a bug."

  # ── AI review reliability (dual-AI combiner + per-provider circuit breaker, #2540) ─
  - name: loopover-ai-review
    rules:
      - alert: LoopoverAiReviewInconclusiveSpike
        # `inconclusive` means the AI review pipeline could not produce a usable verdict (every reviewer
        # opinion missing/unparseable, or a required opinion never came back) -- the review still runs
        # deterministically, but dual-AI review is repeatedly failing to add value. Absolute-increase
        # threshold (matching LoopoverDeadLetterJobsGrowing above): there's no clean matching-cardinality
        # denominator (total review attempts aren't broken out per-mode the same way), so a ratio query
        # would need an unrelated series. > 5 in 30m tolerates the occasional one-off degrade.
        expr: increase(loopover_ai_review_inconclusive_total[30m]) > 5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover AI review is repeatedly inconclusive"
          description: "{{ $value | printf \"%.0f\" }} AI review(s) came back inconclusive over the last 30m (sustained 10m). Dual-AI review is repeatedly failing to produce a usable verdict."
          runbook: "Check provider health and circuit-breaker state (loopover_ai_provider_failures_total / loopover_ai_provider_circuit_open_total) and verify AI_PROVIDER credentials are still valid for every configured reviewer."

      - alert: LoopoverAiReviewOnMergeFloorBypassAttempted
        # A repo's .loopover.yml tried to loosen the operator's onMerge/combine/reviewer-count floor and
        # got clamped back to the operator's own configured minimum (#3901). A governance/config signal, not
        # an operational outage, but same absolute-increase style as LoopoverAiProviderCircuitOpen above:
        # any occurrence (> 0) over 1h is worth a look, sustained 5m so a single instantaneous blip doesn't
        # immediately page.
        expr: increase(loopover_ai_review_onmerge_clamped_total[1h]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "a repo's review-floor clamp fired"
          description: "{{ $value | printf \"%.0f\" }} onMerge/combine/reviewer-count clamp event(s) over the last 1h — a repo config tried to loosen below the operator's configured floor."
          runbook: "Check which repo's .loopover.yml requested the loosened setting (loopover_ai_review_onmerge_clamped_total is labeled by mode) and confirm whether that repo's floor should be adjusted or the request declined."

      - alert: LoopoverAiProviderCircuitOpen
        # A provider's circuit breaker opens after AI_PROVIDER_FAILURE_THRESHOLD consecutive failures and
        # short-circuits further attempts for a cooldown window -- ANY circuit-open event in 15m means that
        # provider has been failing repeatedly and calls are being skipped fast rather than retried at full
        # cost. Same absolute-increase style as LoopoverDeadLetterJobsGrowing: any occurrence is worth a look.
        expr: increase(loopover_ai_provider_circuit_open_total[15m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover AI provider {{ $labels.provider }} circuit breaker is open"
          description: "Provider {{ $labels.provider }} has failed repeatedly and its circuit breaker is skipping calls fast during its cooldown (sustained 5m)."
          runbook: "Check that provider's credentials/reachability (CLI auth for claude-code/codex, or the configured API key/base URL for HTTP providers) via loopover_ai_provider_failures_total{provider=\"...\"} and recent selfhost_ai_provider_failed logs."

  # ── Ops anomaly scan (review burst / review failure burst, #ops-anomaly-metric) ────
  - name: loopover-ops-anomalies
    rules:
      - alert: LoopoverOpsAnomalyDetected
        # runOpsAlerts (src/review/ops-wire.ts) scans loopover's own outcome data hourly and increments this
        # counter, labeled by kind, when it catches a review burst (a stuck-CI finalize loop or sweep retry
        # storm re-publishing the same PR far more than normal iteration ever does) or a review failure burst
        # (repeated inconclusive AI-review calls with zero successful publish -- the #3747 incident shape).
        # Same absolute-increase style as LoopoverDeadLetterJobsGrowing: any occurrence over the scan's own
        # 2h detection window is worth a look, not a rate/ratio.
        expr: increase(loopover_ops_anomaly_total[2h]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover ops anomaly scan flagged a {{ $labels.kind }} on {{ $labels.repo }}"
          description: "{{ $value | printf \"%.0f\" }} {{ $labels.kind }} detection(s) over the last 2h for {{ $labels.repo }} (sustained 5m). Check the ops_anomaly structured log for the full detail line."
          runbook: "Tail logs for level=error event=ops_anomaly repo={{ $labels.repo }} for the human-readable anomaly text. A review_burst or review_failure_burst usually means a stuck-CI finalize loop or a sweep retry storm -- see #orb-ci-stuck-repeat / #review-burst-blind-spot."

  # ── Host clock sync (#3811) ───────────────────────────────────────────────
  - name: loopover-system-health
    rules:
      - alert: LoopoverClockSkewWarning
        # GitHub App JWTs are signed with iat backdated 60s for skew tolerance and exp at now+540s
        # (createAppJwt, src/github/app.ts). A local clock running ahead erodes that backdate; once skew
        # approaches it, GitHub can reject the JWT as not-yet-valid ("Bad credentials"), breaking ALL
        # GitHub App auth fleet-wide -- exactly what happened when edge-us-01's sole NTP source died and
        # its clock drifted ~3 minutes unnoticed. 60s is well under that 3-minute drift, so this fires long
        # before auth actually breaks.
        expr: abs(loopover_clock_skew_seconds) > 60
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "loopover host clock has drifted {{ $value | printf \"%.0f\" }}s from GitHub's server time"
          description: "Clock skew has been over 60s (sustained 5m). GitHub App JWT auth starts failing once skew approaches the 60s iat backdate margin."
          runbook: "Check `chronyc sources` / `chronyc tracking` on the host. Confirm at least one NTP source shows a nonzero Reach (Reach: 0 means that source has never successfully synced). Configure redundant NTP sources -- a single dead source is a silent single point of failure."

      - alert: LoopoverClockSkewCritical
        # 120s is comfortably past the 60s JWT backdate margin -- GitHub App auth is very likely already
        # failing fleet-wide by this point.
        expr: abs(loopover_clock_skew_seconds) > 120
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "loopover host clock skew is CRITICAL ({{ $value | printf \"%.0f\" }}s) -- GitHub App auth is likely failing"
          description: "Clock skew has exceeded 120s (sustained 2m), well past the point GitHub App JWT auth (\"Bad credentials\") is expected to start failing fleet-wide."
          runbook: "Same as LoopoverClockSkewWarning, but treat as urgent: fix NTP sync immediately (chronyc sources, chronyc makestep, restart chrony if every source stays at Reach: 0). Check for github_app_jwt_rejected logs to confirm auth impact."

  # ── Cloudflare D1 (central cloud) size + signal_snapshots dedup regression (#3810) ────
  # loopover_d1_* metrics come from the OPT-IN Cloudflare Management API probe (src/selfhost/
  # d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars) -- absent/disabled reads -1 on every gauge below,
  # comfortably under every threshold here, so these rules never fire on an install that hasn't configured
  # the probe (most self-host installs run their own SQLite/Postgres backend and have nothing to monitor).
  - name: loopover-d1-storage
    rules:
      - alert: LoopoverD1DatabaseSizeWarning
        # 7e9 bytes is ~70% of D1's known ~10GB per-database cap that was hit on 2026-07-06 (#3810), which
        # caused real D1 writes (including an Orb relay registration handshake) to start failing/timing out.
        expr: loopover_d1_database_size_bytes > 7000000000
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Cloudflare D1 database size is approaching its cap"
          description: "The monitored D1 database is {{ $value | humanize }}B, over 70% of the ~10GB per-database cap (sustained 30m)."
          runbook: "Check the D1 panel's per-table row counts (Cloudflare D1 (Central Cloud) row) and confirm the daily signal_snapshots dedup job (dedupeSignalSnapshots, wired into prune-retention) is actually running. See #3810 for the 2026-07-06 incident this guards against."

      - alert: LoopoverD1DatabaseSizeCritical
        # 9e9 bytes is ~90% of the ~10GB cap -- D1 writes started failing fleet-wide the last time the
        # database actually hit the cap, so this is meant to fire well before that happens again.
        expr: loopover_d1_database_size_bytes > 9000000000
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Cloudflare D1 database size is CRITICAL -- writes may start failing soon"
          description: "The monitored D1 database is {{ $value | humanize }}B, over 90% of the ~10GB per-database cap (sustained 10m). D1 writes failed fleet-wide the last time this cap was hit (2026-07-06)."
          runbook: "Immediate: verify the signal_snapshots dedup job is running (POST /v1/internal/retention/preview, or check audit_events for its record) and identify + trim/archive any other unbounded table from the row-count panel. Contact Cloudflare to raise the account storage limit if cleanup alone doesn't recover enough headroom."

      - alert: LoopoverSignalSnapshotsDedupRegression
        # loopover_signal_snapshots_rows_per_key is rows-per-distinct-key scoped ONLY to the four
        # latest-only-dedup signal types dedupeSignalSnapshots (src/db/retention.ts) actually converges to
        # ~1 row per key -- NOT the whole signal_snapshots table, which intentionally keeps bounded
        # multi-row history for other signal types (queue-health, contributor-decision-pack, ...). Healthy
        # steady-state stays a small multiple of 1 (rows can accumulate for up to a day between the daily
        # dedup run); 10 is a wide margin above that, while the actual 2026-07-06 incident ratio (342243
        # rows / 2183 keys) was ~157 -- so a value anywhere near double digits means the dedup job has
        # stopped running, started erroring, or its allowlist regressed.
        expr: loopover_signal_snapshots_rows_per_key > 10
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "signal_snapshots dedup-by-key ratio is climbing"
          description: "signal_snapshots has {{ $value | printf \"%.1f\" }} rows per distinct dedup key (sustained 30m) -- the daily dedupeSignalSnapshots job may not be running."
          runbook: "Confirm the prune-retention cron (03:00 UTC daily) is completing (audit_events around that time) and that dedupeSignalSnapshots isn't throwing. See #3810 and src/db/retention.ts."

      - alert: LoopoverD1ProbeFailing
        # The probe itself (a Cloudflare Management API call) can fail independently of the database it
        # monitors -- an expired/rotated API token, a wrong account/database id, or a Cloudflare API outage.
        # Without this, an operator would see a flat/stale D1 panel and could mistake "the probe broke" for
        # "the database stopped growing".
        expr: increase(loopover_d1_probe_errors_total[1h]) > 3
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "the Cloudflare D1 size/row-count probe is failing"
          description: "{{ $value | printf \"%.0f\" }} D1 Management API probe failure(s) over the last 1h (sustained 15m, part={{ $labels.part }}). The size/row-count gauges below may be stale."
          runbook: "Check CLOUDFLARE_D1_MONITOR_API_TOKEN is still valid and CLOUDFLARE_D1_MONITOR_ACCOUNT_ID/DATABASE_ID are correct. Tail logs for level=error event=d1_size_probe_error."

  # ── Miner prediction-calibration drift (#5188) ────────────────────────────
  # UNLIKE every group above, this rule targets the loopover-MINER's own scrape surface, not the
  # loopover server's GET /metrics. The miner is a local CLI (not a daemon), so an operator renders
  # renderMinerPredictionMetrics (packages/loopover-engine/src/miner-prediction-metrics.ts, #4264, wired
  # into a command in #4838 over the calibration-report join built in #4849) to a file/pushgateway their
  # Prometheus scrapes. The rule is therefore DORMANT until such a target exists: an absent
  # loopover_miner_prediction_* series simply yields no result (and the `> 0` denominator guard also makes
  # the 0/0 case undefined), so it loads cleanly via `selfhost:validate-observability` and never false-fires
  # on an install that has no miner scrape configured yet -- exactly like loopover-d1-storage stays silent
  # without the D1 probe.
  - name: loopover-miner-prediction
    rules:
      - alert: LoopOverMinerPredictionCalibrationDrift
        # Calibration = how often the miner's predicted gate conclusion matched the realized outcome. This is
        # the miner counterpart of loopover-jobs' LoopoverHighJobFailureRatio and uses the identical
        # bad/(bad+good) shape: the fraction of RESOLVED predictions that came back INCORRECT over the window.
        # loopover_miner_prediction_{correct,incorrect}_total only move for rows carrying a resolved outcome
        # (renderMinerPredictionMetrics), so still-unresolved predictions never dilute the ratio. The trailing
        # `> 0` on the denominator guards the 0/0 = NaN case (no resolved predictions in the window -> the rule
        # stays inactive rather than firing on garbage), which is also what makes it degrade to silent when the
        # metric is absent/unset. A wide 6h rate window smooths the miner's naturally sparse, bursty resolution
        # cadence.
        #
        # TUNABLE THRESHOLD: 0.5 = alert once more than half of recently resolved predictions were wrong
        # (calibration has drifted badly). This is the one knob to tune -- lower it (e.g. 0.35) for a stricter
        # accuracy bar, raise it if the miner legitimately runs against harder repos. This follows the file's
        # inline-documented-threshold convention (there are no recording rules here to hang a named constant on).
        expr: |
          (
            sum(rate(loopover_miner_prediction_incorrect_total[6h]))
            /
            (
              sum(rate(loopover_miner_prediction_correct_total[6h]))
              + sum(rate(loopover_miner_prediction_incorrect_total[6h]))
              > 0
            )
          ) > 0.5
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "loopover miner prediction calibration has drifted"
          description: "{{ $value | humanizePercentage }} of the miner's recently resolved gate predictions were incorrect over the last 6h (sustained 30m). Predicted-gate accuracy has drifted out of tolerance."
          runbook: "Compare loopover_miner_prediction_correct_total vs loopover_miner_prediction_incorrect_total and review the miner's own calibration report (`loopover-miner calibration`). Sustained drift usually means the predicted-gate heuristics need recalibration against the repo(s) the miner now targets, or a recent upstream gate-behavior change the predictor hasn't caught up to."

  # ── Miner portfolio-queue backlog health (#5186) ──────────────────────────
  # Same DORMANT-until-a-scrape-target-exists posture as loopover-miner-prediction above: an operator runs
  # `loopover-miner queue metrics` (packages/loopover-miner/lib/portfolio-queue-cli.js's
  # renderPortfolioQueueMetrics) to a file/pushgateway their Prometheus scrapes. Absent
  # loopover_miner_portfolio_queue_* series simply yield no result, so this loads cleanly via
  # `selfhost:validate-observability` and never false-fires on an install with no miner scrape configured.
  - name: loopover-miner-portfolio-queue
    rules:
      - alert: LoopOverMinerPortfolioQueueItemStuck
        # The concrete "is a claimed item actually stuck" signal, distinct from #4827 (the lease/timeout/reclaim
        # CLI logic itself, which already self-heals a stuck lease past DEFAULT_MAX_LEASE_MS = 30m) and #4840
        # (alerting guidance docs, not a rule definition) -- this is the rule that pages an operator when the
        # self-heal ISN'T running (no cron/scheduled `queue claim-batch`/reclaim sweep configured) or is itself
        # stuck, so a lease sits well past its own 30m default with nothing automatically reclaiming it.
        # 3600s (1h) is a generous multiple of DEFAULT_MAX_LEASE_MS so a healthy install's own reclaim sweep
        # always clears this long before the alert would fire; tune down if your reclaim cadence is tighter.
        expr: loopover_miner_portfolio_queue_oldest_in_progress_lease_age_seconds > 3600
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "loopover miner has a claimed portfolio-queue item stuck in progress"
          description: "The oldest in-flight portfolio-queue claim lease is {{ $value | printf \"%.0f\" }}s old (sustained 15m), well past the 30m default reclaim window. A crashed/killed attempt process's lease isn't being automatically reclaimed."
          runbook: "Run `loopover-miner queue list --json` to find the stuck item, and `loopover-miner queue claim-batch` (or a scheduled reclaim sweep) to trigger the lease-expiry self-heal (portfolio-queue-expiry.js's sweepStuckItems). If it stays stuck after that, release it manually with `loopover-miner queue release <owner/repo> <identifier>`."

      - alert: LoopOverMinerPortfolioQueueBacklogHigh
        # Standing size of the queued (not yet claimed) backlog, the miner counterpart of
        # LoopoverQueueBacklogHigh above -- a sustained high queued count means discovery is outpacing
        # claiming, mirroring the main product's own "queue > threshold for N minutes" shape. 200 is a
        # generous default for a single-host laptop-mode install; tune to your normal steady-state depth.
        expr: loopover_miner_portfolio_queue_items{status="queued"} > 200
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "loopover miner portfolio-queue backlog above 200"
          description: "{{ $value | printf \"%.0f\" }} portfolio-queue items are queued (sustained 30m) -- claiming/attempting is falling behind discovery."
          runbook: "Check whether `queue next` / `loop` is running at all (a laptop-mode miner only drains the queue while actively invoked), or whether WIP caps (#4850's --global-wip/--per-repo-wip) are set low enough to bottleneck throughput relative to discovery volume."

  # ── Governor rate-limit/budget threshold pressure (#5187) ─────────────────
  # Same DORMANT-until-a-scrape-target-exists posture as the two miner groups above: an operator runs
  # `loopover-miner governor metrics` (packages/loopover-miner/lib/governor-metrics-cli.js's
  # renderGovernorMetrics) to a file/pushgateway their Prometheus scrapes. Absent loopover_miner_governor_*
  # series simply yield no result. Read-only: these rules alert on the governor's already-persisted rate-limit
  # (#5134/write-rate-limit.ts) and cap-usage (#5134/budget-cap.ts) state; they do not gate, retry, or modify
  # governor decision logic in any way (governor-chokepoint.js/governor-chokepoint-persisted.js are untouched).
  - name: loopover-miner-governor
    rules:
      - alert: LoopOverMinerGovernorRateLimitPressureHigh
        # remaining_ratio is 1 for an empty bucket and 0 once a write-rate-limit bucket (global or per-repo,
        # evaluated against DEFAULT_WRITE_RATE_LIMIT_POLICIES) is exhausted. Each bucket's own window is only
        # 60s (the default policy's windowMs), so staying under 0.1 for a full 10m means the bucket has been
        # refilled to >90% capacity across at least ten separate windows in a row -- sustained heavy write
        # pressure, not one spike.
        expr: loopover_miner_governor_rate_limit_remaining_ratio < 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover miner governor write-rate-limit bucket is nearly exhausted"
          description: "{{ $labels.scope }}/{{ $labels.action_class }}{{ with $labels.repo }} ({{ . }}){{ end }} has had under 10% of its write-rate-limit headroom remaining for 10m."
          runbook: "Check `loopover-miner governor metrics` for the affected scope/action_class and slow down that write class (fewer concurrent attempts, longer loop --cycle-delay-ms), or raise the relevant DEFAULT_WRITE_RATE_LIMIT_POLICIES ceiling if the current limit is genuinely too conservative for this install."

      - alert: LoopOverMinerGovernorCapUsageHigh
        # cap_usage_ratio compares governor-state's persisted cumulative capUsage (budget spent / turns taken /
        # elapsed session ms) against DEFAULT_AMS_POLICY_SPEC.capLimits, the same fleet-wide default loop-cli.js
        # itself falls back to when a repo has no `.loopover-ams.yml` override. 0.9 gives an operator a
        # heads-up before evaluateGovernorCaps' own exceeded/kill_switch verdict actually halts the run.
        expr: loopover_miner_governor_cap_usage_ratio > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "loopover miner governor is approaching a run cap ceiling"
          description: "The governor's {{ $labels.dimension }} cap usage has been over 90% of its configured limit for 10m."
          runbook: "Run `loopover-miner governor metrics` to see which dimension (budget/turns/elapsed_ms) is under pressure, and either let the current run finish (evaluateGovernorCaps will kill_switch/deny once the ceiling is actually reached) or raise the corresponding capLimits in `.loopover-ams.yml` if the ceiling is too tight for this repo's normal attempts."