diff --git a/.env.example b/.env.example index be30a8ff5b..5f2bd9973c 100644 --- a/.env.example +++ b/.env.example @@ -259,6 +259,22 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # # per-batch jobs above this so the per-login GitHub reads spread across the # # queue instead of bursting. Set 0 to disable the fan-out (single job). +# --- Maintenance-job backpressure (#selfhost-runtime-pressure) --- +# Periodic maintenance sweeps (contributor evidence, burden forecasts, RAG re-indexing, drift scans, product +# rollups, notifications...) already yield to an exhausted GitHub REST budget; these knobs add an orthogonal +# check at queue-claim time -- is the BOX itself under load right now -- so maintenance never competes with +# live webhook/review work for CPU/DB time. A denied maintenance job is deferred with jitter, never dropped; +# TRICKLE (MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS) force-admits it once it's waited long enough regardless of +# pressure, so sustained load can slow maintenance down but never starve it forever. All defaults are sane for +# a small single-node box; every value is optional. +# MAINTENANCE_ADMISSION_ENABLED=true # set false/0/off to fully disable this policy (old always-run behavior) +# MAINTENANCE_ADMISSION_MAX_LIVE_PENDING=5 # defer maintenance once this many live (webhook/regate) jobs are queued +# MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS=120000 # defer maintenance once the oldest live job has waited this long (2m) +# MAINTENANCE_ADMISSION_MAX_PENDING=15 # defer NEW maintenance admissions once this many maintenance jobs are already queued +# MAINTENANCE_ADMISSION_MAX_HOST_LOAD=1.5 # defer once 1-min load average per CPU core exceeds this (best-effort; see host-pressure.ts) +# MAINTENANCE_ADMISSION_DEFER_MS=180000 # base defer duration on denial, before jitter (3m) +# MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS=14400000 # trickle ceiling: force-admit a maintenance job that has waited this long (4h) + # --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- # DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert @@ -268,6 +284,18 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # TS_EXTRA_ARGS= # extra tailscale up flags, e.g. --advertise-tags=tag:self-host # --- Self-hosted GitHub Actions runner (#1205; requires --profile runners) --- +# NOT part of the default recommended stack -- it is an OPTIONAL profile. Pick a deployment mode: +# 1. App-only VPS (recommended default): gittensory + Postgres/Redis/Qdrant/observability, no runners here. +# Use GitHub-hosted CI for your own repos, or a separate runner host (below). +# 2. Separate runner host: run `--profile runners` on a DIFFERENT machine from the review stack. CI load +# never contends with the app for CPU/disk/network. +# 3. GitHub-hosted CI: simplest and safest for small teams -- no runner host to manage at all. +# CI runners sharing a VPS with the review stack CAN starve reviews: a burst of CI jobs (vitest coverage, +# wrangler builds, migration checks...) will happily consume every spare CPU cycle the app needs to process +# webhooks and AI review calls, discovered running exactly this pattern in production -- 3 uncapped runner +# containers on an 8-vCPU box left the app starved under load. If you DO need runners on this same host +# (mode 2 above, but co-located anyway, or you're testing this locally), see docker-compose.override.yml.example +# for a proven CPU-priority pattern (cpu_shares + a per-container cpus ceiling) before scaling replicas up. # Blank tokens/URLs are valid until --profile runners is enabled. # RUNNER_TOKEN= # runner registration token (Settings → Actions → Runners → New) # RUNNER_REPO_URL=https://github.com/org/repo @@ -276,6 +304,12 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # RUNNER_NAME=gittensory-runner # RUNNER_LABELS=self-hosted,linux +# --- Docker disk hygiene (#selfhost-runtime-pressure) --- +# Build cache and unused images accumulate fast on a box that builds from source or runs CI runners; a root +# disk over ~80-85% full slows down the WHOLE host (fsync latency, container scheduling), not just Docker. +# ./scripts/docker-prune.sh reports usage read-only by default; pass --yes to actually reclaim space (never +# touches volumes/application data). See the script's header for a cron example. + # --- Observability: metrics + alerts + logs (#1206; requires --profile observability) --- # The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index 3d4f881bb0..d0b823661d 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -11,11 +11,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "AI_EMBED_API_KEY", - firstReference: "src/server.ts:417", + firstReference: "src/server.ts:419", }, { name: "AI_EMBED_BASE_URL", - firstReference: "src/server.ts:414", + firstReference: "src/server.ts:416", }, { name: "AI_EMBED_MODEL", @@ -43,7 +43,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "BACKUP_ACKNOWLEDGED", - firstReference: "src/server.ts:356", + firstReference: "src/server.ts:358", }, { name: "BROWSER_WS_ENDPOINT", @@ -75,11 +75,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "CRON_INTERVAL_MS", - firstReference: "src/server.ts:819", + firstReference: "src/server.ts:839", }, { name: "DATABASE_PATH", - firstReference: "src/server.ts:239", + firstReference: "src/server.ts:241", }, { name: "DATABASE_URL", @@ -103,19 +103,23 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "GITHUB_CACHE_TTL_SECONDS", - firstReference: "src/server.ts:485", + firstReference: "src/server.ts:487", }, { name: "GITTENSORY_REPO_CONFIG_DIR", - firstReference: "src/server.ts:273", + firstReference: "src/server.ts:275", }, { name: "GITTENSORY_VERSION", firstReference: "src/selfhost/health.ts:29", }, + { + name: "MAINTENANCE_ADMISSION_ENABLED", + firstReference: "src/selfhost/maintenance-admission.ts:83", + }, { name: "MIGRATIONS_DIR", - firstReference: "src/server.ts:369", + firstReference: "src/server.ts:371", }, { name: "OBSERVABILITY_SMOKE_POLL_MS", @@ -175,7 +179,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_BROKER_URL", - firstReference: "src/server.ts:863", + firstReference: "src/server.ts:883", }, { name: "ORB_COLLECTOR_TOKEN", @@ -191,7 +195,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_RELAY_MODE", - firstReference: "src/server.ts:865", + firstReference: "src/server.ts:885", }, { name: "OTEL_EXPORTER_OTLP_ENDPOINT", @@ -223,11 +227,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "PGVECTOR_ENABLED", - firstReference: "src/server.ts:219", + firstReference: "src/server.ts:221", }, { name: "PORT", - firstReference: "src/server.ts:623", + firstReference: "src/server.ts:643", }, { name: "PUBLIC_API_ORIGIN", @@ -243,7 +247,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "QDRANT_URL", - firstReference: "src/server.ts:504", + firstReference: "src/server.ts:506", }, { name: "QUEUE_BACKGROUND_CONCURRENCY", @@ -255,7 +259,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "REVIEW_AUDIT_DIR", - firstReference: "src/server.ts:549", + firstReference: "src/server.ts:551", }, { name: "SELFHOST_BUNDLE_ALL", @@ -291,7 +295,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "SETUP_OUTPUT_PATH", - firstReference: "src/server.ts:740", + firstReference: "src/server.ts:760", }, ]; @@ -299,15 +303,15 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| Name | First reference |", "| --- | --- |", "| `AI_COMBINE` | `src/selfhost/ai.ts:848` |", - "| `AI_EMBED_API_KEY` | `src/server.ts:417` |", - "| `AI_EMBED_BASE_URL` | `src/server.ts:414` |", + "| `AI_EMBED_API_KEY` | `src/server.ts:419` |", + "| `AI_EMBED_BASE_URL` | `src/server.ts:416` |", "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:744` |", "| `AI_ON_MERGE` | `src/selfhost/ai.ts:850` |", "| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |", "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:748` |", "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:57` |", "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:747` |", - "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:356` |", + "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:358` |", "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |", "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:108` |", "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:49` |", @@ -315,17 +319,18 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts:112` |", "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:53` |", "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:112` |", - "| `CRON_INTERVAL_MS` | `src/server.ts:819` |", - "| `DATABASE_PATH` | `src/server.ts:239` |", + "| `CRON_INTERVAL_MS` | `src/server.ts:839` |", + "| `DATABASE_PATH` | `src/server.ts:241` |", "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |", "| `DISCORD_REPO_WEBHOOKS` | `src/selfhost/discord-notify.ts:31` |", "| `DISCORD_WEBHOOK_URL` | `src/selfhost/discord-notify.ts:40` |", "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |", - "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:485` |", - "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:273` |", + "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:487` |", + "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:275` |", "| `GITTENSORY_VERSION` | `src/selfhost/health.ts:29` |", - "| `MIGRATIONS_DIR` | `src/server.ts:369` |", + "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:83` |", + "| `MIGRATIONS_DIR` | `src/server.ts:371` |", "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |", "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |", "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:741` |", @@ -340,11 +345,11 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |", "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |", "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", - "| `ORB_BROKER_URL` | `src/server.ts:863` |", + "| `ORB_BROKER_URL` | `src/server.ts:883` |", "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |", "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |", "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |", - "| `ORB_RELAY_MODE` | `src/server.ts:865` |", + "| `ORB_RELAY_MODE` | `src/server.ts:885` |", "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |", "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |", "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |", @@ -352,15 +357,15 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts:40` |", "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |", "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |", - "| `PGVECTOR_ENABLED` | `src/server.ts:219` |", - "| `PORT` | `src/server.ts:623` |", + "| `PGVECTOR_ENABLED` | `src/server.ts:221` |", + "| `PORT` | `src/server.ts:643` |", "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |", "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |", "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |", - "| `QDRANT_URL` | `src/server.ts:504` |", + "| `QDRANT_URL` | `src/server.ts:506` |", "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:102` |", "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |", - "| `REVIEW_AUDIT_DIR` | `src/server.ts:549` |", + "| `REVIEW_AUDIT_DIR` | `src/server.ts:551` |", "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |", "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.mjs:5` |", "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts:186` |", @@ -369,5 +374,5 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `SENTRY_RELEASE` | `src/selfhost/otel.ts:62` |", "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts:373` |", "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts:161` |", - "| `SETUP_OUTPUT_PATH` | `src/server.ts:740` |", + "| `SETUP_OUTPUT_PATH` | `src/server.ts:760` |", ].join("\n"); diff --git a/docker-compose.override.yml.example b/docker-compose.override.yml.example index b95902ab94..83f7070091 100644 --- a/docker-compose.override.yml.example +++ b/docker-compose.override.yml.example @@ -25,6 +25,15 @@ # strongly you want review work to win contention (8:1 below is deliberately aggressive), and keep each # runner replica's `cpus` ceiling at roughly (host vCPUs) / (number of runner replicas you plan to run), # so a single busy runner can't monopolize the box even before shares kick in. +# +# This is the HOST-level (Docker CPU scheduling) half of the fix. The APPLICATION-level half is the +# MAINTENANCE_ADMISSION_* knobs in .env.example: gittensory itself defers its own maintenance sweeps +# (contributor evidence, RAG indexing, drift scans...) under queue/host pressure so live webhook/review +# work always wins there too. Also see ./scripts/docker-prune.sh for the disk-usage side of running CI +# builds on the same box (build cache/image growth, not just CPU). +# The RECOMMENDED alternative to all of this: don't co-locate runners with the review stack at all -- use +# GitHub-hosted CI, or a separate dedicated runner host (see the runner service's comment in +# docker-compose.yml). services: gittensory: diff --git a/docker-compose.yml b/docker-compose.yml index 16cdec0c68..a93e6d4dd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -609,12 +609,17 @@ services: network_mode: host # Tailscale needs host networking to advertise the host's address # ── Self-hosted GitHub Actions runner (--profile runners) ───────────────── + # NOT part of the default recommended stack -- purely optional. Three deployment modes (see .env.example + # for the full explanation): (1) app-only VPS + GitHub-hosted CI (recommended default, no runner here at + # all), (2) a separate dedicated runner host, (3) co-located runners on this SAME box (this profile). # Runs `runs-on: self-hosted` jobs on this machine. Set RUNNER_TOKEN= (or ACCESS_TOKEN=) # and RUNNER_REPO_URL= (e.g. https://github.com/your-org/your-repo) in .env. # Get a token at: https://github.com///settings/actions/runners/new # Running this alongside the main `gittensory` app on the SAME host? A burst of CI jobs can starve the - # app of CPU with no limits set (the default below). See docker-compose.override.yml.example for a - # proven CPU-priority pattern (relative cpu_shares + a per-container cpus ceiling) before scaling this up. + # app of CPU with no limits set (the default below) -- confirmed in production: 3 uncapped runner + # containers on an 8-vCPU box left the app starved under load. See docker-compose.override.yml.example + # for a proven CPU-priority pattern (relative cpu_shares + a per-container cpus ceiling) before scaling + # this up, and ./scripts/docker-prune.sh for reclaiming the disk space CI builds accumulate. runner: # Pin to a specific version tag — `latest` is mutable. Find a digest via: # docker pull myoung34/github-runner:ubuntu-jammy && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-jammy diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 039f85abab..1a5fdd0646 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -2118,6 +2118,284 @@ ], "title": "Backup Freshness", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 136 }, + "id": 128, + "title": "Runtime Pressure & Maintenance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 15 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 137 }, + "id": 129, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Live Queue Pending", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_live_pending", + "legendFormat": "live pending" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 15 }, + { "color": "red", "value": 40 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 137 }, + "id": 130, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Maintenance Queue Pending", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_maintenance_pending", + "legendFormat": "maintenance pending" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 120 }, + { "color": "red", "value": 600 } + ] + }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 137 }, + "id": 131, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Oldest Live Job Age", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_oldest_live_pending_age_seconds", + "legendFormat": "oldest live age" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 14400 }, + { "color": "red", "value": 43200 } + ] + }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 137 }, + "id": 132, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Oldest Maintenance Job Age", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_oldest_maintenance_pending_age_seconds", + "legendFormat": "oldest maintenance age" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 1.5 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 137 }, + "id": 133, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Host Load (per core, -1 = unavailable)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_host_load_avg1_per_core", + "legendFormat": "load/core" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 137 }, + "id": 134, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Branch-Protection Permission Denied (total)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_github_branch_protection_permission_denied_total or vector(0)", + "legendFormat": "permission denied" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 141 }, + "id": 135, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Live vs. Maintenance Queue Depth", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_live_pending", + "legendFormat": "live" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_maintenance_pending", + "legendFormat": "maintenance" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "ops" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 141 }, + "id": 136, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Maintenance Admission Deferrals by Reason", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (reason, job_type) (rate(gittensory_jobs_maintenance_admission_deferred_by_reason_total[5m])) or vector(0)", + "legendFormat": "{{reason}} {{job_type}}", + "refId": "A" + } + ] } ], "refresh": "30s", @@ -2142,5 +2420,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 5 + "version": 6 } diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 006d51a137..a98ac2a529 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -116,6 +116,38 @@ groups: description: "{{ $value | printf \"%.0f\" }} jobs pending for over 10m — the worker is falling behind the enqueue rate." runbook: "Compare rate(gittensory_jobs_enqueued_total) vs rate(gittensory_jobs_processed_total). If enqueue > processed persistently, the worker is the bottleneck (slow AI/DB, or it's stuck). Check worker logs." + # ── Maintenance-job backpressure (#selfhost-runtime-pressure) ────────────── + - name: gittensory-maintenance-pressure + rules: + - alert: GittensoryLiveQueueStarved + # Live/foreground work (webhooks, per-PR re-gates) is the whole point of the review stack -- it + # should clear in seconds to low minutes. A sustained old live job means something OTHER than + # maintenance admission is the bottleneck (AI latency, GitHub, DB, or genuine host CPU pressure), + # since maintenance-admission.ts already yields to live work at claim time. + expr: gittensory_queue_oldest_live_pending_age_seconds > 300 + for: 5m + labels: + severity: warning + annotations: + summary: "gittensory live/webhook work is stuck behind something other than maintenance" + description: "The oldest live-priority queue job has been pending for {{ $value | printf \"%.0f\" }}s. Maintenance admission already yields to live work, so check AI latency, GitHub rate limits, DB pressure, or host CPU (gittensory_host_load_avg1_per_core)." + runbook: "Open the Runtime Pressure & Maintenance row. If gittensory_host_load_avg1_per_core is elevated, a co-located CI runner or other host process is starving the app -- see docker-compose.yml's runner isolation guidance." + + - alert: GittensoryMaintenanceStarved + # Maintenance admission (maintenance-admission.ts) force-admits a maintenance job once it has waited + # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS (default 4h) regardless of pressure -- so under correct + # operation this should never sit much past that trickle ceiling. A value well beyond it means + # either the trickle isn't triggering (a bug) or the host is so overloaded even the trickle-forced + # job can't be claimed/processed. + expr: gittensory_queue_oldest_maintenance_pending_age_seconds > 21600 + for: 15m + labels: + severity: warning + annotations: + summary: "gittensory maintenance work has not run in over 6h" + description: "The oldest maintenance-lane queue job has been pending for {{ $value | printf \"%.0f\" }}s, past the default trickle ceiling. Contributor evidence, RAG indexing, drift scans, and similar sweeps are stale." + runbook: "Check gittensory_jobs_maintenance_admission_deferred_by_reason_total for the dominant defer reason, and confirm MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS wasn't raised. Sustained host_load_high suggests the box itself (not just this app) is overloaded." + # ── GitHub API budget / queue admission pressure ────────────────────────── - name: gittensory-github-rate-limits rules: diff --git a/scripts/docker-prune.sh b/scripts/docker-prune.sh new file mode 100755 index 0000000000..f7118f87f6 --- /dev/null +++ b/scripts/docker-prune.sh @@ -0,0 +1,82 @@ +#!/bin/sh +# Safe Docker disk hygiene for a self-host VPS (#selfhost-runtime-pressure). Build cache and unused images +# accumulate quickly on a box that rebuilds `gittensory` from source or runs GitHub Actions runners -- +# multi-GB `docker builder prune` growth per week is normal, and a root disk over ~80-85% full slows down +# EVERYTHING on the box (Postgres/SQLite fsync latency, container scheduling, log writes), not just Docker +# itself, well before it fills up completely. +# +# SAFE BY DESIGN: only prunes build cache, dangling/unused IMAGES, and stopped containers -- NEVER volumes +# (gittensory-data, gittensory-backups, postgres-data, qdrant-storage, etc.), so it cannot delete application +# data, backups, or vector-store state. Read-only by default (`--dry-run` -- or run without `--yes`, see +# below) so you can review what would be reclaimed before anything is deleted. +# +# Usage: +# ./scripts/docker-prune.sh # report current disk usage only, delete nothing +# ./scripts/docker-prune.sh --dry-run # same as above (explicit) +# ./scripts/docker-prune.sh --yes # actually prune (build cache + dangling images + stopped containers) +# ./scripts/docker-prune.sh --yes --images # also prune UNUSED (not just dangling) images -- more aggressive, +# # will re-pull/rebuild on next deploy if an image isn't running +# +# Cron example (weekly, Sunday 04:00, low-traffic window): +# 0 4 * * 0 cd /path/to/gittensory && ./scripts/docker-prune.sh --yes >> /var/log/gittensory-docker-prune.log 2>&1 +set -eu + +DRY_RUN=1 +PRUNE_UNUSED_IMAGES=0 +for arg in "$@"; do + case "$arg" in + --yes) DRY_RUN=0 ;; + --dry-run) DRY_RUN=1 ;; + --images) PRUNE_UNUSED_IMAGES=1 ;; + *) + echo "[docker-prune] unknown argument: $arg (expected --yes, --dry-run, and/or --images)" >&2 + exit 1 + ;; + esac +done + +if ! command -v docker >/dev/null 2>&1; then + echo "[docker-prune] docker not found on PATH" >&2 + exit 1 +fi + +echo "[docker-prune] disk usage before:" +docker system df + +echo "[docker-prune] root filesystem usage:" +df -h / 2>/dev/null || true + +if [ "$DRY_RUN" = 1 ]; then + echo "[docker-prune] DRY RUN (default) -- nothing will be deleted. Re-run with --yes to actually prune." + echo "[docker-prune] would run: docker builder prune -f" + echo "[docker-prune] would run: docker container prune -f" + if [ "$PRUNE_UNUSED_IMAGES" = 1 ]; then + echo "[docker-prune] would run: docker image prune -a -f" + else + echo "[docker-prune] would run: docker image prune -f (dangling only; pass --images for unused-but-tagged images too)" + fi + echo "[docker-prune] volumes are NEVER pruned by this script -- application data, backups, and vector-store state are always safe." + exit 0 +fi + +echo "[docker-prune] pruning build cache..." +docker builder prune -f + +echo "[docker-prune] pruning stopped containers..." +docker container prune -f + +if [ "$PRUNE_UNUSED_IMAGES" = 1 ]; then + echo "[docker-prune] pruning ALL unused images (not just dangling)..." + docker image prune -a -f +else + echo "[docker-prune] pruning dangling images..." + docker image prune -f +fi + +echo "[docker-prune] disk usage after:" +docker system df + +echo "[docker-prune] root filesystem usage:" +df -h / 2>/dev/null || true + +echo "[docker-prune] complete -- volumes were never touched." diff --git a/src/github/backfill.ts b/src/github/backfill.ts index 4d040a5409..219af2c2a1 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -325,6 +325,12 @@ const PR_DETAIL_BATCH_SIZE: Record = { light: 12, full: 40 // runs again (#audit-rate-headroom). Any PR left un-hydrated this run stays a candidate on the next page/run. const MERGED_PR_FILE_HYDRATION_BATCH_SIZE: Record = { light: 10, full: 20, resume: 20 }; const PULL_REQUEST_FILES_FETCH_METRIC = "gittensory_github_pull_request_files_fetch_total"; +// #selfhost-runtime-pressure: a bare 403 on the branch-protection probe (no admin:read on this installation/fork, +// the common case) is a PERMISSION/config gap, not GitHub rate-limit exhaustion -- GitHubApiError.rateLimited +// already makes that distinction (see isRateLimitedGitHubFailure below). Counted separately from the rate-limit +// metrics so a dashboard can tell "GitHub is throttling us" apart from "this token can't read branch protection +// (expected for most installations/forks)" instead of a permission gap inflating an apparent rate-limit signal. +const BRANCH_PROTECTION_PERMISSION_DENIED_METRIC = "gittensory_github_branch_protection_permission_denied_total"; type PullRequestFilesFetchCaller = "backfill_open_pr_details" | "backfill_merged_history" | "live_review"; // #2537: durable-cache counter for the bare PR-state read, mirroring PULL_REQUEST_FILES_FETCH_METRIC's bounded- // label style (no per-PR-number labels — cardinality-safe). @@ -2338,7 +2344,10 @@ export async function fetchRequiredStatusContexts( `/branches/${encodeURIComponent(baseRef)}/protection/required_status_checks`, token, githubRateLimitOptions(admissionKey), - ).catch(() => undefined); + ).catch((error) => { + recordBranchProtectionFetchFailure(error); + return undefined; + }); if (!result) return null; // 404 / 403 (no admin:read) / error → conservative fold-all. const names = new Set(); for (const ctx of result.data.contexts ?? []) { @@ -2350,6 +2359,17 @@ export async function fetchRequiredStatusContexts( return names; } +// A GitHubApiError's own `.rateLimited` flag (set at construction from status/retry-after/remaining/body, see +// GitHubApiError below) is ALREADY the correct rate-limit-vs-permission classification -- this just labels the +// permission case with its own metric instead of the fetch's `.catch` silently discarding that information. +// A non-GitHubApiError (a network/timeout failure) and a 404 (no branch protection configured) are equally +// "fold all checks" outcomes for the caller, but neither is a permission denial, so neither is counted here. +function recordBranchProtectionFetchFailure(error: unknown): void { + if (error instanceof GitHubApiError && error.statusCode === 403 && !error.rateLimited) { + incr(BRANCH_PROTECTION_PERMISSION_DENIED_METRIC); + } +} + /** * Best-effort fetch of ONE named check-run's conclusion on a head SHA (#2564, the CLA-bot check-run detection * mode of `gate.claMode`). Returns the conclusion string (lowercased; `"neutral"`/`"success"`/… or `""` when diff --git a/src/selfhost/host-pressure.ts b/src/selfhost/host-pressure.ts new file mode 100644 index 0000000000..ae86f3de18 --- /dev/null +++ b/src/selfhost/host-pressure.ts @@ -0,0 +1,22 @@ +// Optional host-CPU-pressure hint for maintenance-job admission (see maintenance-admission.ts). Node-only -- +// `node:os`'s loadavg() has no meaningful signal on Cloudflare Workers -- this module is imported ONLY by the +// self-host Node queue backends (sqlite-queue.ts / pg-queue.ts), never by src/index.ts's Worker bundle, so a +// static `node:os` import here is safe (mirrors the existing `hostname` import in selfhost/sentry.ts). +import { cpus, loadavg } from "node:os"; + +/** The 1-minute load average normalized per logical core, so the SAME threshold means the same thing on a + * 4-vCPU box as a 32-vCPU box. Best-effort and fail-open: any error, or a reading that can't possibly be a + * real load average, yields `null` ("signal unavailable") rather than a misleading 0 -- a caller must treat + * `null` as "skip this check", never as "load is zero". (On Windows, Node's loadavg() always returns + * `[0, 0, 0]` by design; that legitimately normalizes to 0, which just never trips a pressure threshold.) */ +export function hostLoadAvg1PerCore(): number | null { + try { + const load1 = loadavg()[0] ?? Number.NaN; + if (!Number.isFinite(load1) || load1 < 0) return null; + const coreCount = cpus().length; + if (!Number.isFinite(coreCount) || coreCount < 1) return null; + return load1 / coreCount; + } catch { + return null; + } +} diff --git a/src/selfhost/maintenance-admission.ts b/src/selfhost/maintenance-admission.ts new file mode 100644 index 0000000000..650229d12e --- /dev/null +++ b/src/selfhost/maintenance-admission.ts @@ -0,0 +1,169 @@ +// Maintenance-job backpressure / admission policy (#selfhost-runtime-pressure). User-facing work -- +// github-webhook, agent-regate-pr, the regate sweep trigger, recapture-preview (everything at or above +// FOREGROUND_QUEUE_PRIORITY_FLOOR, see queue-common.ts) -- must always win a resource race against periodic +// maintenance sweeps (contributor evidence, burden forecasts, RAG re-indexing, drift scans, product rollups, +// notifications...). Those sweeps already run on a conservative cadence (every 30min/hourly/6-hourly, see +// index.ts's enqueueScheduledJobs) and already yield to an EXHAUSTED GitHub REST budget +// (shouldWaitForGitHubRateLimit) -- this module adds an ORTHOGONAL signal: is the box itself under load RIGHT +// NOW (a live-work backlog, an aging live job, a hot host CPU), independent of whether GitHub's API happens to +// be rate-limited. The queue backends (sqlite-queue.ts / pg-queue.ts) consult this at CLAIM time, the same way +// they already consult GitHub rate-limit admission: a denied maintenance job is pushed back to 'pending' with +// a jittered future run_after -- its original enqueue time is left untouched, so the age-based trickle below +// still works -- never dropped and never run early. +// +// TRICKLE: a maintenance job that has been pending since `maxDeferAgeMs` is force-admitted regardless of +// current pressure, so a box under SUSTAINED load can never starve maintenance work forever -- it just runs at +// a bounded minimum rate instead of its normal cadence. +import { deterministicJitterMs, parsePositiveIntEnv } from "./queue-common"; + +// Periodic, repo/contributor-set-wide sweeps -- the heavy, deferrable maintenance lane. Deliberately EXCLUDES +// the targeted, per-PR/per-repo jobs fanned out FROM some of these (or that serve a specific in-flight +// PR/webhook directly): "backfill-repo-segment", "backfill-pr-details", "run-agent", "submit-draft", +// "retry-orb-relay" stay on the normal background lane, unthrottled by this policy. Foreground job types +// (github-webhook, agent-regate-pr, agent-regate-sweep, recapture-preview) are never listed here either -- they +// are already priority-gated (FOREGROUND_QUEUE_PRIORITY_FLOOR) and this policy only ever runs for a +// background-priority job. +export const MAINTENANCE_JOB_TYPES: ReadonlySet = new Set([ + "backfill-registered-repos", + "refresh-registry", + "refresh-installation-health", + "refresh-scoring-model", + "refresh-upstream-sources", + "build-upstream-ruleset", + "detect-upstream-drift", + "refresh-upstream-drift", + "file-upstream-drift-issues", + "build-contributor-evidence", + "build-contributor-decision-packs", + "refresh-contributor-activity", + "build-burden-forecasts", + "repair-data-fidelity", + "rollup-product-usage", + "prune-retention", + "generate-weekly-value-report", + "generate-signal-snapshots", + "notify-evaluate", + "notify-deliver", + "ops-alerts", + "selftune", + "rag-index-repo", +]); + +export function isMaintenanceJobType(type: string): boolean { + return MAINTENANCE_JOB_TYPES.has(type); +} + +export interface MaintenancePressureSignals { + livePendingCount: number; + oldestLivePendingAgeMs: number | null; + maintenancePendingCount: number; + oldestMaintenancePendingAgeMs: number | null; + /** Null when unavailable (see host-pressure.ts) -- a caller must treat null as "skip this check". */ + hostLoadAvg1PerCore: number | null; +} + +export interface MaintenanceAdmissionConfig { + enabled: boolean; + maxLivePendingCount: number; + maxLiveJobAgeMs: number; + maxMaintenancePendingCount: number; + maxHostLoadAvg1PerCore: number; + deferMs: number; + maxDeferAgeMs: number; +} + +const DEFAULT_MAX_LIVE_PENDING_COUNT = 5; +const DEFAULT_MAX_LIVE_JOB_AGE_MS = 2 * 60_000; +const DEFAULT_MAX_MAINTENANCE_PENDING_COUNT = 15; +const DEFAULT_MAX_HOST_LOAD_AVG1_PER_CORE = 1.5; +const DEFAULT_DEFER_MS = 3 * 60_000; +const DEFAULT_MAX_DEFER_AGE_MS = 4 * 60 * 60_000; + +function maintenanceAdmissionEnabled(): boolean { + const raw = (process.env.MAINTENANCE_ADMISSION_ENABLED ?? "").trim().toLowerCase(); + return raw !== "0" && raw !== "false" && raw !== "off" && raw !== "no"; +} + +function parsePositiveFloatEnv(name: string, fallback: number): number { + const supplied = process.env[name]; + if (supplied === undefined) return fallback; + const parsed = Number(supplied); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +/** Reads every MAINTENANCE_ADMISSION_* knob from process.env, each with a sane, protective default. Resolved + * ONCE per queue instance (mirrors queueBackgroundConcurrency / queueStartupJitterMs) rather than per job, so + * a misconfigured value only warns once at startup instead of on every claim. */ +export function resolveMaintenanceAdmissionConfig(): MaintenanceAdmissionConfig { + return { + enabled: maintenanceAdmissionEnabled(), + maxLivePendingCount: parsePositiveIntEnv("MAINTENANCE_ADMISSION_MAX_LIVE_PENDING", { + min: 0, + fallback: DEFAULT_MAX_LIVE_PENDING_COUNT, + }), + maxLiveJobAgeMs: parsePositiveIntEnv("MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS", { + min: 0, + fallback: DEFAULT_MAX_LIVE_JOB_AGE_MS, + }), + maxMaintenancePendingCount: parsePositiveIntEnv("MAINTENANCE_ADMISSION_MAX_PENDING", { + min: 0, + fallback: DEFAULT_MAX_MAINTENANCE_PENDING_COUNT, + }), + maxHostLoadAvg1PerCore: parsePositiveFloatEnv( + "MAINTENANCE_ADMISSION_MAX_HOST_LOAD", + DEFAULT_MAX_HOST_LOAD_AVG1_PER_CORE, + ), + deferMs: parsePositiveIntEnv("MAINTENANCE_ADMISSION_DEFER_MS", { min: 1_000, fallback: DEFAULT_DEFER_MS }), + maxDeferAgeMs: parsePositiveIntEnv("MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS", { + min: 60_000, + fallback: DEFAULT_MAX_DEFER_AGE_MS, + }), + }; +} + +export type MaintenanceAdmissionReason = + | "disabled" + | "trickle_max_defer_age" + | "live_pending_high" + | "live_job_age_high" + | "maintenance_pending_high" + | "host_load_high" + | "pressure_clear"; + +export interface MaintenanceAdmissionDecision { + admit: boolean; + reason: MaintenanceAdmissionReason; +} + +/** PURE policy decision: admit this maintenance job now, or defer it? Checked in priority order -- the + * trickle (age) escape hatch first, so a starved job is never re-denied by a later check, then each pressure + * signal in turn. `pendingSinceMs` is the job's ORIGINAL enqueue time (its row's created_at), not the time of + * its most recent deferral, so the trickle clock only resets on a genuine fresh request (a coalesced + * re-enqueue), never on a repeated denial of the same wait. */ +export function evaluateMaintenanceAdmission( + signals: MaintenancePressureSignals, + config: MaintenanceAdmissionConfig, + pendingSinceMs: number, + nowMs: number, +): MaintenanceAdmissionDecision { + if (!config.enabled) return { admit: true, reason: "disabled" }; + if (nowMs - pendingSinceMs >= config.maxDeferAgeMs) return { admit: true, reason: "trickle_max_defer_age" }; + if (signals.livePendingCount > config.maxLivePendingCount) return { admit: false, reason: "live_pending_high" }; + if (signals.oldestLivePendingAgeMs !== null && signals.oldestLivePendingAgeMs > config.maxLiveJobAgeMs) { + return { admit: false, reason: "live_job_age_high" }; + } + if (signals.maintenancePendingCount > config.maxMaintenancePendingCount) { + return { admit: false, reason: "maintenance_pending_high" }; + } + if (signals.hostLoadAvg1PerCore !== null && signals.hostLoadAvg1PerCore > config.maxHostLoadAvg1PerCore) { + return { admit: false, reason: "host_load_high" }; + } + return { admit: true, reason: "pressure_clear" }; +} + +/** Jittered defer duration for a denied maintenance job -- the base `deferMs` plus up to another `deferMs` of + * deterministic jitter (seeded by the job's own identity) so a whole cohort of denied jobs doesn't wake up on + * the same tick and immediately re-trip the same pressure check (mirrors rateLimitRetryDelayWithJitter). */ +export function maintenanceAdmissionDeferMs(config: MaintenanceAdmissionConfig, jitterSeed: string): number { + return config.deferMs + deterministicJitterMs(jitterSeed, config.deferMs); +} diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index e5e152cf14..5559e5b081 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -18,6 +18,7 @@ import { githubRateLimitMetricContext, githubRateLimitRetryDelayMs, buildSelfHostQueueSnapshot, + isForegroundJobPriority, jobCoalesceAbsorbedByKey, jobCoalesceKey, jobCoalesceSupersededKeyPrefix, @@ -35,6 +36,15 @@ import { type GitHubRateLimitAdmissionTarget, type SelfHostQueueSnapshot, } from "./queue-common"; +import { hostLoadAvg1PerCore } from "./host-pressure"; +import { + evaluateMaintenanceAdmission, + isMaintenanceJobType, + maintenanceAdmissionDeferMs, + resolveMaintenanceAdmissionConfig, + type MaintenanceAdmissionConfig, + type MaintenancePressureSignals, +} from "./maintenance-admission"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -53,6 +63,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE} ( ); ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0; ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS job_key TEXT; +ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS is_maintenance INTEGER NOT NULL DEFAULT 0; CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after, priority); CREATE INDEX IF NOT EXISTS ${TABLE}_pending_job_key ON ${TABLE}(job_key, status); CREATE TABLE IF NOT EXISTS ${STATS_TABLE} ( @@ -70,6 +81,9 @@ export interface PgDurableQueue { deadCount(): Promise; stats(): Promise>; snapshot(): Promise; + /** Live-vs-maintenance queue pressure, for the /metrics gauges (see server.ts) -- the SAME signals the + * maintenance-admission policy itself consults at claim time. */ + pressureSignals(): Promise; /** Requeues dead-lettered jobs still under the auto-retry attempts ceiling. Called on a timer while * running (see start()), and exposed directly so tests and an operator-triggered repair path don't have * to wait for the real interval. Returns the number of jobs revived. */ @@ -82,6 +96,7 @@ interface JobRow { attempts: number; job_key?: string | null; priority: number | string; + created_at: number | string; backgroundSlotReserved?: boolean; } @@ -122,6 +137,7 @@ export function createPgQueue( const activeJobIds = new Set(); let timer: ReturnType | null = null; let deadLetterReviveTimer: ReturnType | null = null; + const maintenanceAdmissionConfig: MaintenanceAdmissionConfig = resolveMaintenanceAdmissionConfig(); async function init(): Promise { await pool.query(DDL); @@ -141,6 +157,14 @@ export function createPgQueue( count: keyBackfilled, }), ); + const maintenanceFlagsBackfilled = await backfillJobMaintenanceFlags(); + if (maintenanceFlagsBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_maintenance_flags_backfilled", + count: maintenanceFlagsBackfilled, + }), + ); const recovered = await recoverProcessingJobs(); if (recovered) { await recordQueueMetric("gittensory_jobs_recovered_total", recovered); @@ -193,6 +217,43 @@ export function createPgQueue( return changed; } + async function backfillJobMaintenanceFlags(): Promise { + const res = await pool.query( + `SELECT id, payload, is_maintenance FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + ); + let changed = 0; + for (const row of res.rows as Array<{ id: string; payload: string; is_maintenance: number | string }>) { + const isMaintenance = isMaintenanceJobType(extractPayloadType(row.payload) ?? "") ? 1 : 0; + if (Number(row.is_maintenance ?? 0) === isMaintenance) continue; + await pool.query(`UPDATE ${TABLE} SET is_maintenance=$1 WHERE id=$2`, [isMaintenance, row.id]); + changed += 1; + } + return changed; + } + + /** Cheap aggregate reads behind the maintenance-admission policy (and the observability gauges in + * server.ts): how much LIVE (foreground) work is queued and how old the oldest of it is, and the same for + * the MAINTENANCE lane specifically (not "all background" -- targeted jobs like backfill-repo-segment + * don't count, see maintenance-admission.ts). Host load is an independent, optional signal. */ + async function maintenancePressureSignals(now: number): Promise { + const liveRes = await pool.query( + `SELECT COUNT(*) AS cnt, MIN(created_at) AS oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=$1`, + [FOREGROUND_QUEUE_PRIORITY_FLOOR], + ); + const maintenanceRes = await pool.query( + `SELECT COUNT(*) AS cnt, MIN(created_at) AS oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND is_maintenance=1`, + ); + const live = liveRes.rows[0] as { cnt: string | number; oldest: string | number | null }; + const maintenance = maintenanceRes.rows[0] as { cnt: string | number; oldest: string | number | null }; + return { + livePendingCount: Number(live.cnt), + oldestLivePendingAgeMs: live.oldest != null ? now - Number(live.oldest) : null, + maintenancePendingCount: Number(maintenance.cnt), + oldestMaintenancePendingAgeMs: maintenance.oldest != null ? now - Number(maintenance.oldest) : null, + hostLoadAvg1PerCore: hostLoadAvg1PerCore(), + }; + } + async function recoverProcessingJobs(): Promise { const res = await pool.query( `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing'`, @@ -364,8 +425,8 @@ export function createPgQueue( } } await pool.query( - `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key) VALUES ($1,'pending',0,$2,$3,$4,$5)`, - [payload, runAfter, now, priority, key], + `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) VALUES ($1,'pending',0,$2,$3,$4,$5,$6)`, + [payload, runAfter, now, priority, key, isMaintenanceJobType(message.type) ? 1 : 0], ); await recordQueueMetric("gittensory_jobs_enqueued_total"); kickOne(); @@ -400,7 +461,7 @@ export function createPgQueue( FOR UPDATE SKIP LOCKED LIMIT 1 ) - RETURNING id, payload, attempts, job_key, priority`, + RETURNING id, payload, attempts, job_key, priority, created_at`, [now, FOREGROUND_QUEUE_PRIORITY_FLOOR], ); return (res.rows[0] as JobRow | undefined) ?? null; @@ -494,6 +555,49 @@ export function createPgQueue( ); return true; } + if (!isForegroundJobPriority(Number(job.priority)) && isMaintenanceJobType(message.type)) { + const decision = evaluateMaintenanceAdmission( + await maintenancePressureSignals(Date.now()), + maintenanceAdmissionConfig, + Number(job.created_at), + Date.now(), + ); + if (!decision.admit) { + await withReviewSpan( + "selfhost.queue.maintenance_admission_deferred", + { "job.type": message.type, "queue.backend": "postgres", "maintenance_admission.reason": decision.reason }, + async () => { + const now = Date.now(); + const retryAfter = now + maintenanceAdmissionDeferMs( + maintenanceAdmissionConfig, + `${job.job_key ?? ""}:${job.id}:${job.payload}`, + ); + const update = await pool.query( + `UPDATE ${TABLE} SET status='pending', run_after=GREATEST(run_after, $1), last_error=COALESCE(last_error, $2) WHERE id=$3`, + [retryAfter, `maintenance admission deferred: ${decision.reason}`, job.id], + ); + if (update.rowCount) { + await recordQueueMetric("gittensory_jobs_maintenance_admission_deferred_total"); + incr("gittensory_jobs_maintenance_admission_deferred_by_reason_total", { + reason: decision.reason, + job_type: message.type, + }); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_maintenance_admission_deferred", + jobType: message.type, + reason: decision.reason, + retry_after_ms: Math.max(0, retryAfter - now), + }), + ); + } + }, + { parentTraceParent: jobTraceParent }, + ); + return true; + } + } try { await withReviewSpan( "selfhost.queue.job", @@ -722,6 +826,9 @@ export function createPgQueue( }, snapshot: binding.snapshot, reviveDeadLetterJobs, + pressureSignals() { + return maintenancePressureSignals(Date.now()); + }, }; async function reclaimExpiredProcessingJobs(): Promise { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index e56696a23d..82183d0b62 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -19,6 +19,7 @@ import { githubRateLimitMetricContext, githubRateLimitRetryDelayMs, buildSelfHostQueueSnapshot, + isForegroundJobPriority, jobCoalesceAbsorbedByKey, jobCoalesceKey, jobCoalesceSupersededKeyPrefix, @@ -36,6 +37,15 @@ import { type GitHubRateLimitAdmissionTarget, type SelfHostQueueSnapshot, } from "./queue-common"; +import { hostLoadAvg1PerCore } from "./host-pressure"; +import { + evaluateMaintenanceAdmission, + isMaintenanceJobType, + maintenanceAdmissionDeferMs, + resolveMaintenanceAdmissionConfig, + type MaintenanceAdmissionConfig, + type MaintenancePressureSignals, +} from "./maintenance-admission"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -72,6 +82,9 @@ export interface DurableQueue { deadCount(): number; stats(): Record; snapshot(): SelfHostQueueSnapshot; + /** Live-vs-maintenance queue pressure, for the /metrics gauges (see server.ts) -- the SAME signals the + * maintenance-admission policy itself consults at claim time. */ + pressureSignals(): MaintenancePressureSignals; /** Requeues dead-lettered jobs still under the auto-retry attempts ceiling. Called on a timer while * running (see start()), and exposed directly so tests and an operator-triggered repair path don't have * to wait for the real interval. Returns the number of jobs revived. */ @@ -84,6 +97,7 @@ interface JobRow { attempts: number; job_key?: string | null; priority: number; + created_at: number; backgroundSlotReserved?: boolean; } @@ -134,6 +148,11 @@ export function createSqliteQueue( } catch { /* column already present */ } + try { + driver.exec(`ALTER TABLE ${TABLE} ADD COLUMN is_maintenance INTEGER NOT NULL DEFAULT 0`); + } catch { + /* column already present */ + } driver.exec(CLAIM_INDEX_DDL); driver.exec(JOB_KEY_INDEX_DDL); const priorityBackfilled = backfillJobPriorities(driver); @@ -152,6 +171,15 @@ export function createSqliteQueue( count: keyBackfilled, }), ); + const maintenanceFlagsBackfilled = backfillJobMaintenanceFlags(driver); + if (maintenanceFlagsBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_maintenance_flags_backfilled", + count: maintenanceFlagsBackfilled, + }), + ); + const maintenanceAdmissionConfig: MaintenanceAdmissionConfig = resolveMaintenanceAdmissionConfig(); // Recover jobs a crashed previous run left mid-flight → make them claimable again. const recovered = recoverProcessingJobs(driver); if (recovered) { @@ -269,8 +297,8 @@ export function createSqliteQueue( } } driver.query( - `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, ?, ?, ?, ?)`, - [payload, runAfter, now, priority, key], + `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) VALUES (?, 'pending', 0, ?, ?, ?, ?, ?)`, + [payload, runAfter, now, priority, key, isMaintenanceJobType(message.type) ? 1 : 0], ); recordQueueMetric(driver, "gittensory_jobs_enqueued_total"); kickOne(); @@ -292,7 +320,7 @@ export function createSqliteQueue( function claimNextWhere(now: number, priorityPredicate: string): JobRow | null { const { rows } = driver.query( - `SELECT id, payload, attempts, job_key, priority + `SELECT id, payload, attempts, job_key, priority, created_at FROM ${TABLE} WHERE status='pending' AND run_after<=? AND ${priorityPredicate} ORDER BY priority DESC, run_after, id @@ -401,6 +429,49 @@ export function createSqliteQueue( ); return true; } + if (!isForegroundJobPriority(job.priority) && isMaintenanceJobType(message.type)) { + const decision = evaluateMaintenanceAdmission( + maintenancePressureSignals(driver, Date.now()), + maintenanceAdmissionConfig, + job.created_at, + Date.now(), + ); + if (!decision.admit) { + await withReviewSpan( + "selfhost.queue.maintenance_admission_deferred", + { "job.type": message.type, "queue.backend": "sqlite", "maintenance_admission.reason": decision.reason }, + async () => { + const now = Date.now(); + const retryAfter = now + maintenanceAdmissionDeferMs( + maintenanceAdmissionConfig, + `${job.job_key ?? ""}:${job.id}:${job.payload}`, + ); + const { changes } = driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=max(run_after, ?), last_error=coalesce(last_error, ?) WHERE id=?`, + [retryAfter, `maintenance admission deferred: ${decision.reason}`, job.id], + ); + if (changes) { + recordQueueMetric(driver, "gittensory_jobs_maintenance_admission_deferred_total"); + incr("gittensory_jobs_maintenance_admission_deferred_by_reason_total", { + reason: decision.reason, + job_type: message.type, + }); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_maintenance_admission_deferred", + jobType: message.type, + reason: decision.reason, + retry_after_ms: Math.max(0, retryAfter - now), + }), + ); + } + }, + { parentTraceParent: jobTraceParent }, + ); + return true; + } + } try { await withReviewSpan( "selfhost.queue.job", @@ -634,6 +705,9 @@ export function createSqliteQueue( }, snapshot: binding.snapshot, reviveDeadLetterJobs, + pressureSignals() { + return maintenancePressureSignals(driver, Date.now()); + }, }; } @@ -670,6 +744,43 @@ function backfillJobKeys(driver: SqliteDriver): number { return changed; } +function backfillJobMaintenanceFlags(driver: SqliteDriver): number { + const { rows } = driver.query( + `SELECT id, payload, is_maintenance FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + [], + ); + let changed = 0; + for (const row of rows as Array<{ id: number; payload: string; is_maintenance: number }>) { + const isMaintenance = isMaintenanceJobType(extractPayloadType(row.payload) ?? "") ? 1 : 0; + if (Number(row.is_maintenance) === isMaintenance) continue; + driver.query(`UPDATE ${TABLE} SET is_maintenance=? WHERE id=?`, [isMaintenance, row.id]); + changed += 1; + } + return changed; +} + +/** Cheap aggregate reads behind the maintenance-admission policy (and the observability gauges in server.ts): + * how much LIVE (foreground) work is queued and how old the oldest of it is, and the same for the + * MAINTENANCE lane specifically (not "all background" -- targeted jobs like backfill-repo-segment don't + * count, see maintenance-admission.ts). Host load is an independent, optional signal (see host-pressure.ts). */ +function maintenancePressureSignals(driver: SqliteDriver, now: number): MaintenancePressureSignals { + const live = driver.query( + `SELECT COUNT(*) as cnt, MIN(created_at) as oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=?`, + [FOREGROUND_QUEUE_PRIORITY_FLOOR], + ).rows[0] as { cnt: number; oldest: number | null }; + const maintenance = driver.query( + `SELECT COUNT(*) as cnt, MIN(created_at) as oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND is_maintenance=1`, + [], + ).rows[0] as { cnt: number; oldest: number | null }; + return { + livePendingCount: Number(live.cnt), + oldestLivePendingAgeMs: live.oldest != null ? now - Number(live.oldest) : null, + maintenancePendingCount: Number(maintenance.cnt), + oldestMaintenancePendingAgeMs: maintenance.oldest != null ? now - Number(maintenance.oldest) : null, + hostLoadAvg1PerCore: hostLoadAvg1PerCore(), + }; +} + function recoverProcessingJobs(driver: SqliteDriver): number { const { rows } = driver.query( `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing'`, diff --git a/src/server.ts b/src/server.ts index f686452f75..5bd6fbf0f0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -54,6 +54,7 @@ import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum } from "./se import { createPgQueue } from "./selfhost/pg-queue"; import { createPgVectorize, initPgVectorize } from "./selfhost/pg-vectorize"; import { resolvePostgresPoolMax } from "./selfhost/queue-common"; +import type { MaintenancePressureSignals } from "./selfhost/maintenance-admission"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; import { createSqliteVectorize } from "./selfhost/vectorize"; import { createFsBlobStore } from "./selfhost/blob-store"; @@ -126,6 +127,7 @@ interface Backend { size(): number | Promise; deadCount(): number | Promise; stats(): Record | Promise>; + pressureSignals(): MaintenancePressureSignals | Promise; }; vectorize?: Vectorize; shutdown(): Promise; @@ -585,11 +587,29 @@ async function main(): Promise { "gittensory_jobs_rate_limit_deferred_total", "gittensory_jobs_coalesced_total", "gittensory_jobs_recovered_total", + "gittensory_jobs_maintenance_admission_deferred_total", ]) { gauge(name.replace("_total", "_persisted_total"), () => durableJobMetric(name), ); } + // Runtime-pressure gauges (#selfhost-runtime-pressure): the SAME signals the maintenance-admission policy + // consults at claim time (see maintenance-admission.ts), so the dashboard shows exactly what's gating + // maintenance work right now -- live vs. maintenance queue depth, how stale the oldest of each is, and + // (best-effort) host CPU pressure. Distinguishes "the app queue is backed up" from "CI/other host load is + // starving the app" from "GitHub/AI latency", the ambiguity that made the original slowdown hard to diagnose. + const maintenancePressure = () => backend.queue.pressureSignals(); + gauge("gittensory_queue_live_pending", async () => (await maintenancePressure()).livePendingCount); + gauge("gittensory_queue_maintenance_pending", async () => (await maintenancePressure()).maintenancePendingCount); + gauge("gittensory_queue_oldest_live_pending_age_seconds", async () => + Math.floor(((await maintenancePressure()).oldestLivePendingAgeMs ?? 0) / 1000), + ); + gauge("gittensory_queue_oldest_maintenance_pending_age_seconds", async () => + Math.floor(((await maintenancePressure()).oldestMaintenancePendingAgeMs ?? 0) / 1000), + ); + // -1 (not 0) when unavailable -- a genuine idle host reads 0, so a dashboard can tell "known idle" apart + // from "no signal on this platform" (see host-pressure.ts). + gauge("gittensory_host_load_avg1_per_core", async () => (await maintenancePressure()).hostLoadAvg1PerCore ?? -1); gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000), ); diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 5bc8cd8d5a..23f3212548 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -54,6 +54,7 @@ import { } from "../../src/github/client"; import { normalizeRegistryPayload } from "../../src/registry/normalize"; import { persistRegistrySnapshot } from "../../src/registry/sync"; +import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { createTestEnv } from "../helpers/d1"; describe("GitHub backfill", () => { @@ -5050,6 +5051,37 @@ describe("GitHub backfill", () => { vi.stubGlobal("fetch", async () => new Response("forbidden", { status: 403 })); expect(await fetchRequiredStatusContexts(env, "JSONbored/gittensory", "main", "public-token")).toBeNull(); }); + + it("classifies a bare 403 (no admin:read) as permission-denied, not a rate limit (#selfhost-runtime-pressure)", async () => { + resetMetrics(); + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async () => new Response("forbidden", { status: 403 })); + expect(await fetchRequiredStatusContexts(env, "JSONbored/gittensory", "main", "public-token")).toBeNull(); + expect(await renderMetrics()).toContain("gittensory_github_branch_protection_permission_denied_total 1"); + }); + + it("does not count a 404 (no branch protection configured) as permission-denied", async () => { + resetMetrics(); + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async () => new Response("not found", { status: 404 })); + expect(await fetchRequiredStatusContexts(env, "JSONbored/gittensory", "main", "public-token")).toBeNull(); + expect(await renderMetrics()).not.toContain("gittensory_github_branch_protection_permission_denied_total"); + }); + + it("does not count a genuinely rate-limited 403 (x-ratelimit-remaining: 0) as permission-denied", async () => { + resetMetrics(); + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal( + "fetch", + async () => + new Response("secondary rate limit", { + status: 403, + headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "1780000000" }, + }), + ); + expect(await fetchRequiredStatusContexts(env, "JSONbored/gittensory", "main", "public-token")).toBeNull(); + expect(await renderMetrics()).not.toContain("gittensory_github_branch_protection_permission_denied_total"); + }, 15_000); }); describe("fetchNamedCheckRunConclusion (#2564)", () => { diff --git a/test/unit/selfhost-host-pressure.test.ts b/test/unit/selfhost-host-pressure.test.ts new file mode 100644 index 0000000000..557ad8b5a6 --- /dev/null +++ b/test/unit/selfhost-host-pressure.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("node:os", () => ({ + loadavg: vi.fn(), + cpus: vi.fn(), +})); + +describe("hostLoadAvg1PerCore", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + }); + + it("normalizes the 1-minute load average by logical core count", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([4, 3, 2]); + vi.mocked(os.cpus).mockReturnValue(Array.from({ length: 4 }, () => ({}) as never)); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBe(1); + }); + + it("returns null when loadavg() reports no samples at all (empty array)", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([]); + vi.mocked(os.cpus).mockReturnValue(Array.from({ length: 4 }, () => ({}) as never)); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); + + it("returns null when load1 is not finite", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([Number.NaN, 0, 0]); + vi.mocked(os.cpus).mockReturnValue(Array.from({ length: 4 }, () => ({}) as never)); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); + + it("returns null when load1 is negative", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([-1, 0, 0]); + vi.mocked(os.cpus).mockReturnValue(Array.from({ length: 4 }, () => ({}) as never)); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); + + it("returns null when cpus() reports zero cores", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([1, 1, 1]); + vi.mocked(os.cpus).mockReturnValue([]); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); + + it("returns 0 on Windows-style always-zero loadavg (a legitimate reading, not unavailable)", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([0, 0, 0]); + vi.mocked(os.cpus).mockReturnValue(Array.from({ length: 8 }, () => ({}) as never)); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBe(0); + }); + + it("returns null when loadavg() throws", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockImplementation(() => { + throw new Error("unsupported platform"); + }); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); + + it("returns null when cpus() throws", async () => { + const os = await import("node:os"); + vi.mocked(os.loadavg).mockReturnValue([1, 1, 1]); + vi.mocked(os.cpus).mockImplementation(() => { + throw new Error("unsupported platform"); + }); + const { hostLoadAvg1PerCore } = await import("../../src/selfhost/host-pressure"); + expect(hostLoadAvg1PerCore()).toBeNull(); + }); +}); diff --git a/test/unit/selfhost-maintenance-admission.test.ts b/test/unit/selfhost-maintenance-admission.test.ts new file mode 100644 index 0000000000..4bcf4365b3 --- /dev/null +++ b/test/unit/selfhost-maintenance-admission.test.ts @@ -0,0 +1,270 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + evaluateMaintenanceAdmission, + isMaintenanceJobType, + maintenanceAdmissionDeferMs, + MAINTENANCE_JOB_TYPES, + resolveMaintenanceAdmissionConfig, + type MaintenanceAdmissionConfig, + type MaintenancePressureSignals, +} from "../../src/selfhost/maintenance-admission"; + +const CLEAR_SIGNALS: MaintenancePressureSignals = { + livePendingCount: 0, + oldestLivePendingAgeMs: null, + maintenancePendingCount: 0, + oldestMaintenancePendingAgeMs: null, + hostLoadAvg1PerCore: null, +}; + +const CONFIG: MaintenanceAdmissionConfig = { + enabled: true, + maxLivePendingCount: 5, + maxLiveJobAgeMs: 120_000, + maxMaintenancePendingCount: 15, + maxHostLoadAvg1PerCore: 1.5, + deferMs: 180_000, + maxDeferAgeMs: 4 * 60 * 60_000, +}; + +describe("isMaintenanceJobType", () => { + it("classifies every listed maintenance sweep type", () => { + for (const type of MAINTENANCE_JOB_TYPES) { + expect(isMaintenanceJobType(type)).toBe(true); + } + }); + + it("does not classify targeted/foreground job types as maintenance", () => { + for (const type of [ + "github-webhook", + "agent-regate-pr", + "agent-regate-sweep", + "recapture-preview", + "backfill-repo-segment", + "backfill-pr-details", + "run-agent", + "submit-draft", + "retry-orb-relay", + ]) { + expect(isMaintenanceJobType(type)).toBe(false); + } + }); +}); + +describe("evaluateMaintenanceAdmission", () => { + const now = 1_000_000_000; + + it("admits when every pressure signal is clear", () => { + expect(evaluateMaintenanceAdmission(CLEAR_SIGNALS, CONFIG, now - 1_000, now)).toEqual({ + admit: true, + reason: "pressure_clear", + }); + }); + + it("admits unconditionally when disabled, even under extreme pressure", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 999 }, + { ...CONFIG, enabled: false }, + now - 1_000, + now, + ); + expect(decision).toEqual({ admit: true, reason: "disabled" }); + }); + + it("defers when live pending count exceeds the threshold", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 6 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision).toEqual({ admit: false, reason: "live_pending_high" }); + }); + + it("admits when live pending count is AT (not over) the threshold", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 5 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision.admit).toBe(true); + }); + + it("defers when the oldest live job has waited past the max age", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, oldestLivePendingAgeMs: 120_001 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision).toEqual({ admit: false, reason: "live_job_age_high" }); + }); + + it("admits when there is no live job at all (null oldest age)", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, oldestLivePendingAgeMs: null }, + CONFIG, + now - 1_000, + now, + ); + expect(decision.admit).toBe(true); + }); + + it("defers when the maintenance lane itself is already backed up", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, maintenancePendingCount: 16 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision).toEqual({ admit: false, reason: "maintenance_pending_high" }); + }); + + it("defers when host load per core exceeds the threshold", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, hostLoadAvg1PerCore: 1.51 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision).toEqual({ admit: false, reason: "host_load_high" }); + }); + + it("admits when host load is unavailable (null), never treating null as high", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, hostLoadAvg1PerCore: null }, + CONFIG, + now - 1_000, + now, + ); + expect(decision.admit).toBe(true); + }); + + it("force-admits via trickle once pending since exceeds the max defer age, even under pressure", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 999, hostLoadAvg1PerCore: 99 }, + CONFIG, + now - CONFIG.maxDeferAgeMs, + now, + ); + expect(decision).toEqual({ admit: true, reason: "trickle_max_defer_age" }); + }); + + it("does not trickle-admit a job that hasn't waited long enough yet", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 999 }, + CONFIG, + now - (CONFIG.maxDeferAgeMs - 1), + now, + ); + expect(decision).toEqual({ admit: false, reason: "live_pending_high" }); + }); + + it("checks live pressure before maintenance-lane pressure (priority order)", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, livePendingCount: 6, maintenancePendingCount: 16 }, + CONFIG, + now - 1_000, + now, + ); + expect(decision.reason).toBe("live_pending_high"); + }); +}); + +describe("maintenanceAdmissionDeferMs", () => { + it("returns at least the base defer and at most double it (base + jitter)", () => { + const value = maintenanceAdmissionDeferMs(CONFIG, "seed-a"); + expect(value).toBeGreaterThanOrEqual(CONFIG.deferMs); + expect(value).toBeLessThanOrEqual(CONFIG.deferMs * 2); + }); + + it("is deterministic for the same seed", () => { + expect(maintenanceAdmissionDeferMs(CONFIG, "seed-b")).toBe(maintenanceAdmissionDeferMs(CONFIG, "seed-b")); + }); + + it("varies (in general) across different seeds", () => { + const values = new Set( + Array.from({ length: 10 }, (_, i) => maintenanceAdmissionDeferMs(CONFIG, `seed-${i}`)), + ); + expect(values.size).toBeGreaterThan(1); + }); +}); + +describe("resolveMaintenanceAdmissionConfig", () => { + const envKeys = [ + "MAINTENANCE_ADMISSION_ENABLED", + "MAINTENANCE_ADMISSION_MAX_LIVE_PENDING", + "MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS", + "MAINTENANCE_ADMISSION_MAX_PENDING", + "MAINTENANCE_ADMISSION_MAX_HOST_LOAD", + "MAINTENANCE_ADMISSION_DEFER_MS", + "MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS", + ] as const; + const saved: Record = {}; + + beforeEach(() => { + for (const key of envKeys) { + saved[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(() => { + for (const key of envKeys) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + }); + + it("returns protective defaults with no env overrides", () => { + expect(resolveMaintenanceAdmissionConfig()).toEqual({ + enabled: true, + maxLivePendingCount: 5, + maxLiveJobAgeMs: 120_000, + maxMaintenancePendingCount: 15, + maxHostLoadAvg1PerCore: 1.5, + deferMs: 180_000, + maxDeferAgeMs: 4 * 60 * 60_000, + }); + }); + + it("reads every knob from the environment when set", () => { + process.env.MAINTENANCE_ADMISSION_MAX_LIVE_PENDING = "10"; + process.env.MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS = "60000"; + process.env.MAINTENANCE_ADMISSION_MAX_PENDING = "30"; + process.env.MAINTENANCE_ADMISSION_MAX_HOST_LOAD = "2.25"; + process.env.MAINTENANCE_ADMISSION_DEFER_MS = "5000"; + process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS = "3600000"; + const config = resolveMaintenanceAdmissionConfig(); + expect(config.maxLivePendingCount).toBe(10); + expect(config.maxLiveJobAgeMs).toBe(60_000); + expect(config.maxMaintenancePendingCount).toBe(30); + expect(config.maxHostLoadAvg1PerCore).toBe(2.25); + expect(config.deferMs).toBe(5_000); + expect(config.maxDeferAgeMs).toBe(3_600_000); + }); + + it.each(["0", "false", "off", "no"])("treats MAINTENANCE_ADMISSION_ENABLED=%s as disabled", (value) => { + process.env.MAINTENANCE_ADMISSION_ENABLED = value; + expect(resolveMaintenanceAdmissionConfig().enabled).toBe(false); + }); + + it.each(["1", "true", "on", "yes", "anything-else"])( + "treats MAINTENANCE_ADMISSION_ENABLED=%s as enabled", + (value) => { + process.env.MAINTENANCE_ADMISSION_ENABLED = value; + expect(resolveMaintenanceAdmissionConfig().enabled).toBe(true); + }, + ); + + it("falls back to the default host-load threshold on an invalid float", () => { + process.env.MAINTENANCE_ADMISSION_MAX_HOST_LOAD = "not-a-number"; + expect(resolveMaintenanceAdmissionConfig().maxHostLoadAvg1PerCore).toBe(1.5); + }); + + it("falls back to the default host-load threshold on a negative float", () => { + process.env.MAINTENANCE_ADMISSION_MAX_HOST_LOAD = "-1"; + expect(resolveMaintenanceAdmissionConfig().maxHostLoadAvg1PerCore).toBe(1.5); + }); +}); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 3b5a465ef7..0bfa800d38 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -6,8 +6,14 @@ import { createPgQueue } from "../../src/selfhost/pg-queue"; import { queueSnapshotFromBinding } from "../../src/selfhost/queue-common"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { RetryableJobError } from "../../src/queue/retryable"; +import { hostLoadAvg1PerCore } from "../../src/selfhost/host-pressure"; import type { JobMessage } from "../../src/types"; +// Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every +// maintenance-admission test in this file would be flaky against the real node:os signal. Default to +// "unavailable" (null, never gates) here; individual host-load tests override the mock explicitly. +vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null) })); + const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; const webhook = (sender: { login: string; type: string }, eventName = "issue_comment", action = "edited"): JobMessage => ({ @@ -63,6 +69,12 @@ interface MockPool { * on a specific row (rowCount 0) while another succeeds (rowCount 1). */ setReviveUpdateRowCounts(rowCounts: number[]): void; setRateLimitRows(rows: Array<{ admission_key?: string | null; repo_full_name?: string | null; remaining: number | string | null; reset_at: string | null; observed_at?: string | null }>): void; + /** Configures the two maintenance-admission pressure aggregate queries (live + maintenance lane). Defaults + * to zero pending / null oldest in both lanes (pressure clear) until set. */ + setPressureSignals(signals: { + live?: { cnt: number; oldest: number | null }; + maintenance?: { cnt: number; oldest: number | null }; + }): void; } function makePool(): MockPool { @@ -70,8 +82,14 @@ function makePool(): MockPool { let deferUpdateRowCount = 1; const reviveUpdateRowCounts: number[] = []; let rateLimitRows: Array<{ admission_key?: string | null; repo_full_name?: string | null; remaining: number | string | null; reset_at: string | null; observed_at?: string | null }> = []; + let pressureLive: { cnt: number; oldest: number | null } = { cnt: 0, oldest: null }; + let pressureMaintenance: { cnt: number; oldest: number | null } = { cnt: 0, oldest: null }; const fn = vi.fn().mockImplementation(async (sql: unknown, params?: unknown[]) => { const q = String(sql); + if (q.includes("AS cnt, MIN(created_at) AS oldest")) { + const signal = q.includes("is_maintenance=1") ? pressureMaintenance : pressureLive; + return { rows: [{ cnt: String(signal.cnt), oldest: signal.oldest }], rowCount: 1 }; + } if (q.includes("FROM github_rate_limit_observations")) { const admissionKey = typeof params?.[0] === "string" ? params[0] : null; const newest = (rows: typeof rateLimitRows) => @@ -118,6 +136,10 @@ function makePool(): MockPool { reviveUpdateRowCounts.length = 0; reviveUpdateRowCounts.push(...rowCounts); }, + setPressureSignals(signals) { + if (signals.live) pressureLive = signals.live; + if (signals.maintenance) pressureMaintenance = signals.maintenance; + }, setRateLimitRows(rows) { rateLimitRows = rows; }, @@ -131,6 +153,7 @@ describe("createPgQueue (durable #977)", () => { vi.useRealTimers(); resetMetrics(); vi.restoreAllMocks(); + vi.mocked(hostLoadAvg1PerCore).mockReturnValue(null); }); it("init() creates the table and recovers stuck-processing jobs", async () => { @@ -1592,4 +1615,108 @@ describe("createPgQueue (durable #977)", () => { ); expect(bindingSnapshot).toEqual(snapshot); }); + + describe("maintenance-admission pressure gating (#selfhost-runtime-pressure)", () => { + const now = Date.now(); + const maintenanceRow = { id: "m1", payload: JSON.stringify(msg("build-contributor-evidence")), attempts: 0, job_key: "build-contributor-evidence:all", priority: 0, created_at: now }; + + it("defers a maintenance job when live queue pressure is high", async () => { + const m = makePool(); + m.setPressureSignals({ live: { cnt: 6, oldest: now } }); // default threshold is 5 + m.enqueueResult({ rows: [], rowCount: 0 }); // empty foreground claim + m.enqueueResult({ rows: [maintenanceRow], rowCount: 1 }); // background claim + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + + expect(started).not.toContain("build-contributor-evidence"); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', run_after=GREATEST"), + expect.arrayContaining([expect.stringContaining("live_pending_high")]), + ); + expect(await renderMetrics()).toContain( + 'gittensory_jobs_maintenance_admission_deferred_by_reason_total{job_type="build-contributor-evidence",reason="live_pending_high"} 1', + ); + }); + + it("admits a maintenance job immediately when pressure is clear", async () => { + const m = makePool(); + m.enqueueResult({ rows: [], rowCount: 0 }); // empty foreground claim + m.enqueueResult({ rows: [maintenanceRow], rowCount: 1 }); // background claim + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + expect(started).toEqual(["build-contributor-evidence"]); + }); + + it("never defers a foreground job even under maintenance-triggering pressure", async () => { + const m = makePool(); + m.setPressureSignals({ live: { cnt: 6, oldest: now } }); + m.enqueueResult({ + rows: [{ id: "w1", payload: JSON.stringify(msg("github-webhook")), attempts: 0, job_key: null, priority: 10, created_at: now }], + rowCount: 1, + }); + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + expect(started).toEqual(["github-webhook"]); + }); + + it("defers a maintenance job when the maintenance lane itself is backed up", async () => { + const m = makePool(); + m.setPressureSignals({ maintenance: { cnt: 16, oldest: now } }); // default threshold is 15 + m.enqueueResult({ rows: [], rowCount: 0 }); + m.enqueueResult({ rows: [maintenanceRow], rowCount: 1 }); + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + }); + + it("defers a maintenance job when host load per core is high", async () => { + vi.mocked(hostLoadAvg1PerCore).mockReturnValue(5); + const m = makePool(); + m.enqueueResult({ rows: [], rowCount: 0 }); + m.enqueueResult({ rows: [maintenanceRow], rowCount: 1 }); + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + }); + + it("force-admits via trickle once a maintenance job has waited past the max defer age", async () => { + const oldEnv = process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS; + process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS = "60000"; // 1m (parsePositiveIntEnv floor) + try { + const m = makePool(); + m.setPressureSignals({ live: { cnt: 6, oldest: now } }); + m.enqueueResult({ rows: [], rowCount: 0 }); + m.enqueueResult({ + rows: [{ ...maintenanceRow, created_at: now - 61_000 }], + rowCount: 1, + }); + const started: string[] = []; + const q = createPgQueue(m.pool, async (j) => void started.push(typeOf(j))); + await q.drain(); + expect(started).toEqual(["build-contributor-evidence"]); + } finally { + if (oldEnv === undefined) delete process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS; + else process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS = oldEnv; + } + }); + + it("pressureSignals() surfaces the live and maintenance aggregate reads", async () => { + const m = makePool(); + m.setPressureSignals({ live: { cnt: 2, oldest: now - 1_000 }, maintenance: { cnt: 4, oldest: now - 2_000 } }); + const q = createPgQueue(m.pool, async () => undefined); + const signals = await q.pressureSignals(); + expect(signals).toEqual({ + livePendingCount: 2, + oldestLivePendingAgeMs: expect.any(Number), + maintenancePendingCount: 4, + oldestMaintenancePendingAgeMs: expect.any(Number), + hostLoadAvg1PerCore: null, + }); + }); + }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 68dcaffc98..1b7b5851fa 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -5,8 +5,14 @@ import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; import { queueSnapshotFromBinding } from "../../src/selfhost/queue-common"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { RetryableJobError } from "../../src/queue/retryable"; +import { hostLoadAvg1PerCore } from "../../src/selfhost/host-pressure"; import type { JobMessage } from "../../src/types"; +// Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every +// maintenance-admission test in this file would be flaky against the real node:os signal. Default to +// "unavailable" (null, never gates) here; individual host-load tests override the mock explicitly. +vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null) })); + function makeDriver(): ReturnType { return nodeSqliteDriver(new DatabaseSync(":memory:") as never); } @@ -69,6 +75,7 @@ describe("createSqliteQueue (durable #980)", () => { vi.useRealTimers(); resetMetrics(); vi.restoreAllMocks(); + vi.mocked(hostLoadAvg1PerCore).mockReturnValue(null); }); it("persists + drains FIFO through the consumer", async () => { @@ -2034,4 +2041,292 @@ describe("createSqliteQueue (durable #980)", () => { await q.stop(); // waits for the in-flight consume to finish expect(done).toBe(true); }); + + describe("maintenance-admission pressure gating (#selfhost-runtime-pressure)", () => { + const envKeys = [ + "MAINTENANCE_ADMISSION_ENABLED", + "MAINTENANCE_ADMISSION_MAX_LIVE_PENDING", + "MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS", + "MAINTENANCE_ADMISSION_MAX_PENDING", + "MAINTENANCE_ADMISSION_MAX_HOST_LOAD", + "MAINTENANCE_ADMISSION_DEFER_MS", + "MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS", + ] as const; + const saved: Record = {}; + + beforeEach(() => { + for (const key of envKeys) { + saved[key] = process.env[key]; + delete process.env[key]; + } + }); + afterEach(() => { + for (const key of envKeys) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + }); + + // Directly occupies rows the way currently-pending/processing live work would, without needing worker-slot + // choreography -- the admission check reads real table state, not the consumer callback. + function seedLiveRows(driver: ReturnType, count: number, status: "pending" | "processing" = "processing"): void { + const now = Date.now(); + for (let i = 0; i < count; i += 1) { + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, ?, 0, ?, ?, 10, NULL, 0)`, + [JSON.stringify({ type: "github-webhook", deliveryId: `seed-${i}`, eventName: "x", payload: {} }), status, now, now], + ); + } + } + + it("defers a maintenance job when live queue pressure is high", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); // default threshold is 5 + const before = Date.now(); + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + + expect(started).not.toContain("build-contributor-evidence"); + const row = driver.query( + "SELECT status, run_after, last_error FROM _selfhost_jobs WHERE payload LIKE '%build-contributor-evidence%'", + [], + ).rows[0] as { status: string; run_after: number; last_error: string }; + expect(row.status).toBe("pending"); + expect(row.run_after).toBeGreaterThan(before); + expect(row.last_error).toContain("live_pending_high"); + expect(q.stats()).toMatchObject({ gittensory_jobs_maintenance_admission_deferred_total: 1 }); + expect(await renderMetrics()).toContain( + 'gittensory_jobs_maintenance_admission_deferred_by_reason_total{job_type="build-contributor-evidence",reason="live_pending_high"} 1', + ); + }); + + it("admits a maintenance job immediately when pressure is clear", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + expect(started).toEqual(["build-contributor-evidence"]); + expect(q.size()).toBe(0); + }); + + it("never defers live/foreground work, even under the same pressure that defers maintenance work", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + await q.binding.send(msg("build-contributor-evidence")); + await q.binding.send(msg("github-webhook")); + await q.drain(); + expect(started).toEqual(["github-webhook"]); + expect(started).not.toContain("build-contributor-evidence"); + }); + + it("does not defer a targeted background job that is not classified as maintenance", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + await q.binding.send({ + type: "backfill-repo-segment", + requestedBy: "api", + repoFullName: "jsonbored/gittensory", + segment: "labels", + } as unknown as JobMessage); + await q.drain(); + expect(started).toEqual(["backfill-repo-segment"]); + }); + + it("defers a maintenance job when the maintenance lane itself is already backed up", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + const now = Date.now(); + for (let i = 0; i < 16; i += 1) { // default threshold is 15 + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 0, NULL, 1)`, + [JSON.stringify({ type: "rollup-product-usage", requestedBy: "test" }), now + 3_600_000, now], + ); + } + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + const row = driver.query( + "SELECT last_error FROM _selfhost_jobs WHERE payload LIKE '%build-contributor-evidence%'", + [], + ).rows[0] as { last_error: string }; + expect(row.last_error).toContain("maintenance_pending_high"); + }); + + it("defers a maintenance job when host load per core exceeds the threshold", async () => { + vi.mocked(hostLoadAvg1PerCore).mockReturnValue(5); + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + const row = driver.query( + "SELECT last_error FROM _selfhost_jobs WHERE payload LIKE '%build-contributor-evidence%'", + [], + ).rows[0] as { last_error: string }; + expect(row.last_error).toContain("host_load_high"); + }); + + it("admits unconditionally when MAINTENANCE_ADMISSION_ENABLED is disabled, even under high pressure", async () => { + process.env.MAINTENANCE_ADMISSION_ENABLED = "false"; + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + expect(started).toEqual(["build-contributor-evidence"]); + }); + + it("force-admits via trickle once a maintenance job has waited past the max defer age, even under pressure", async () => { + process.env.MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS = "60000"; // 1m (parsePositiveIntEnv floor) + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + const staleCreatedAt = Date.now() - 61_000; + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, 0, ?, 0, NULL, 1)`, + [JSON.stringify({ type: "build-contributor-evidence", requestedBy: "schedule" }), staleCreatedAt], + ); + await q.drain(); + expect(started).toEqual(["build-contributor-evidence"]); + }); + + it("pressureSignals() reports live/maintenance pending counts and oldest ages", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + seedLiveRows(driver, 2, "pending"); + const now = Date.now(); + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 0, NULL, 1)`, + [JSON.stringify({ type: "rollup-product-usage", requestedBy: "test" }), now + 3_600_000, now - 5_000], + ); + const signals = q.pressureSignals(); + expect(signals.livePendingCount).toBe(2); + expect(signals.oldestLivePendingAgeMs).toBeGreaterThanOrEqual(0); + expect(signals.maintenancePendingCount).toBe(1); + expect(signals.oldestMaintenancePendingAgeMs).toBeGreaterThanOrEqual(5_000); + expect(signals.hostLoadAvg1PerCore).toBeNull(); + }); + + it("pressureSignals() reports null oldest ages when a lane has no pending work", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const signals = q.pressureSignals(); + expect(signals.livePendingCount).toBe(0); + expect(signals.oldestLivePendingAgeMs).toBeNull(); + expect(signals.maintenancePendingCount).toBe(0); + expect(signals.oldestMaintenancePendingAgeMs).toBeNull(); + }); + + it("backfills the is_maintenance flag on startup for jobs enqueued by an older version", async () => { + const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); + driver.exec(` + CREATE TABLE _selfhost_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_error TEXT, + priority INTEGER NOT NULL DEFAULT 0, + job_key TEXT + ); + `); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + [JSON.stringify(msg("build-contributor-evidence"))], + ); + createSqliteQueue(driver, async () => undefined); + const row = driver.query("SELECT is_maintenance FROM _selfhost_jobs", []).rows[0] as { is_maintenance: number }; + expect(Number(row.is_maintenance)).toBe(1); + }); + + it("does not crash the startup backfill on an unparseable pending payload", async () => { + const driver = makeDriver(); + const q1 = createSqliteQueue(driver, async () => undefined); // creates the table + await q1.stop(); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + ["not valid json"], + ); + expect(() => createSqliteQueue(driver, async () => undefined)).not.toThrow(); + const row = driver.query( + "SELECT is_maintenance FROM _selfhost_jobs WHERE payload = ?", + ["not valid json"], + ).rows[0] as { is_maintenance: number }; + expect(Number(row.is_maintenance)).toBe(0); + }); + + it("leaves an already-correct is_maintenance flag alone (backfill no-op branch)", async () => { + const driver = makeDriver(); + const q1 = createSqliteQueue(driver, async () => undefined); + await q1.binding.send(msg("build-contributor-evidence")); + await q1.stop(); + // Re-open a queue over the same driver: the row's is_maintenance is already correct (set 1 at enqueue), + // so the startup backfill should count it as unchanged. + const writes: string[] = []; + vi.mocked(process.stdout.write).mockImplementation((chunk) => { + writes.push(String(chunk)); + return true; + }); + createSqliteQueue(driver, async () => undefined); + expect(writes.some((w) => w.includes("selfhost_queue_maintenance_flags_backfilled"))).toBe(false); + }); + + it("defers a maintenance job with no job_key (a raw/legacy row) using an empty jitter seed segment", async () => { + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + const before = Date.now(); + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, 0, ?, 0, NULL, 1)`, + [JSON.stringify({ type: "build-contributor-evidence", requestedBy: "test" }), before], + ); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + const row = driver.query( + "SELECT run_after FROM _selfhost_jobs WHERE payload LIKE '%build-contributor-evidence%'", + [], + ).rows[0] as { run_after: number }; + expect(row.run_after).toBeGreaterThan(before); + }); + + it("skips the maintenance-admission-deferred metric when the defer update changes no rows", async () => { + const base = makeDriver(); + const driver = { + exec: base.exec.bind(base), + query: vi.fn((sql: string, params: unknown[]) => { + if (sql.includes("SET status='pending', run_after=max(run_after, ?), last_error=coalesce(last_error, ?)")) { + return { rows: [], changes: 0 }; + } + return base.query(sql, params); + }), + } as ReturnType; + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + seedLiveRows(driver, 6); + await q.binding.send(msg("build-contributor-evidence")); + await q.drain(); + expect(started).not.toContain("build-contributor-evidence"); + expect(q.stats()).not.toHaveProperty("gittensory_jobs_maintenance_admission_deferred_total"); + expect(await renderMetrics()).not.toContain("gittensory_jobs_maintenance_admission_deferred_by_reason_total"); + }); + }); });