fix(selfhost): add memory limits to docker-compose services (#2569)

* fix(selfhost): add memory limits to docker-compose services

No service declared a memory or CPU limit, including the core gittensory
app itself. Any --profile combination let one runaway optional service
(Ollama loading a multi-GB model, Postgres, Qdrant, or the whole
observability stack) exhaust host RAM and OOM-kill or starve the core
review pipeline container instead of the intended optional service --
the exact gap #1828 calls out, with zero mitigation anywhere in the
compose file.

Add deploy.resources.limits.memory to the core gittensory/redis
services and every heavyweight optional service (postgres, qdrant,
ollama, prometheus, loki, tempo, grafana), each overridable via a
documented .env variable so an operator can size a bigger/smaller host
without editing the compose file.

* fix(selfhost): guard the possibly-undefined service lookup in the resource-limits test

Author: JSONbored

.env.example

@@ -192,6 +192,20 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # MIGRATIONS_DIR=/app/migrations
 # CRON_INTERVAL_MS=120000                    # maintain/sweep + sync cadence (default ~2 min)
 
+# --- Per-service memory limits (docker-compose.yml; #1828/#2495) ---
+# Every service below gets a beginner-friendly default `deploy.resources.limits.memory` cap so one runaway
+# optional profile (e.g. Ollama loading a multi-GB model) can't OOM-kill or starve the core review pipeline.
+# Override any of these for a bigger/smaller host; values use Docker's byte-size shorthand (e.g. 512m, 4g).
+# GITTENSORY_MEM_LIMIT=2g                    # core app (always runs)
+# REDIS_MEM_LIMIT=512m                       # core app (always runs)
+# POSTGRES_MEM_LIMIT=2g                      # --profile postgres / --profile pgbouncer
+# QDRANT_MEM_LIMIT=2g                        # --profile qdrant
+# OLLAMA_MEM_LIMIT=8g                        # --profile ollama; raise this before pulling a large local model
+# PROMETHEUS_MEM_LIMIT=1g                    # --profile observability
+# LOKI_MEM_LIMIT=1g                          # --profile observability
+# TEMPO_MEM_LIMIT=1g                         # --profile observability
+# GRAFANA_MEM_LIMIT=512m                     # --profile observability
+
 # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) ---
 # Blank is valid until --profile litestream is enabled.
 # LITESTREAM_ACCESS_KEY_ID=

docker-compose.yml

@@ -98,6 +98,12 @@ services:
       pgbouncer:
         condition: service_healthy
         required: false
+    # Sane defaults so a runaway optional service (below) can't OOM-kill or starve the core review pipeline;
+    # override via .env for a bigger/smaller host (#1828).
+    deploy:
+      resources:
+        limits:
+          memory: "${GITTENSORY_MEM_LIMIT:-2g}"
     healthcheck:
       # Probe /ready (not /health): /health is a liveness stub that is 200 even when the DB is down,
       # whereas /ready returns 503 until the DB answers AND migrations are applied — so dependents that
@@ -131,6 +137,10 @@ services:
       - ""
       - --appendonly
       - "no"
+    deploy:
+      resources:
+        limits:
+          memory: "${REDIS_MEM_LIMIT:-512m}"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s
@@ -147,6 +157,10 @@ services:
       POSTGRES_DB: gittensory
     volumes:
       - gittensory-pg:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          memory: "${POSTGRES_MEM_LIMIT:-2g}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U gittensory"]
       interval: 10s
@@ -218,6 +232,10 @@ services:
       - "127.0.0.1:6334:6334" # gRPC (localhost only)
     volumes:
       - qdrant-data:/qdrant/storage
+    deploy:
+      resources:
+        limits:
+          memory: "${QDRANT_MEM_LIMIT:-2g}"
     # The image is Debian (bash present) but ships no curl/wget/nc. bash's /dev/tcp pseudo-device speaks
     # HTTP without an external binary, so we hit Qdrant's k8s-style /readyz probe (200 once it accepts
     # traffic). This lets the app's depends_on gate on a serving Qdrant. /readyz is unauthenticated even
@@ -251,6 +269,12 @@ services:
       timeout: 5s
       start_period: 15s
       retries: 5
+    # Ollama can load multi-GB models into memory; default high enough for common 7-8B quantized models but
+    # cap it so a large model pull can't exhaust host RAM and take down the core review pipeline (#1828).
+    deploy:
+      resources:
+        limits:
+          memory: "${OLLAMA_MEM_LIMIT:-8g}"
 
   # ── Litestream (--profile litestream) ─────────────────────────────────────
   # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example
@@ -308,6 +332,10 @@ services:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.path=/prometheus"
       - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME:-180d}"
+    deploy:
+      resources:
+        limits:
+          memory: "${PROMETHEUS_MEM_LIMIT:-1g}"
 
   # Routes Prometheus alerts to your notification channel. Ships SILENT: alerts go to a
   # null receiver until you fill in a receiver in alertmanager/alertmanager.yml.
@@ -352,6 +380,10 @@ services:
       GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource
       # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env.
       GITHUB_TOKEN: "${GITHUB_TOKEN:-}"
+    deploy:
+      resources:
+        limits:
+          memory: "${GRAFANA_MEM_LIMIT:-512m}"
     entrypoint:
       - /bin/sh
       - -ec
@@ -419,6 +451,10 @@ services:
     volumes:
       - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro
       - loki-data:/loki
+    deploy:
+      resources:
+        limits:
+          memory: "${LOKI_MEM_LIMIT:-1g}"
     # No healthcheck: grafana/loki:latest is distroless (no shell/wget/curl), so an in-container probe
     # can't run — a CMD-SHELL check would leave the container stuck "starting". Readiness is observable
     # at GET /ready; Promtail retries/backs off until Loki accepts pushes and Grafana retries the
@@ -494,6 +530,10 @@ services:
     volumes:
       - ./tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
       - tempo-data:/var/tempo
+    deploy:
+      resources:
+        limits:
+          memory: "${TEMPO_MEM_LIMIT:-1g}"
     healthcheck:
       test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"]
       interval: 10s

test/unit/selfhost-compose-resource-limits.test.ts

@@ -0,0 +1,58 @@
+import { readFileSync } from "node:fs";
+import { parse } from "yaml";
+import { describe, expect, it } from "vitest";
+
+function readYaml(path: string): Record<string, unknown> {
+  const value = parse(readFileSync(path, "utf8"));
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be a YAML object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+// Pure structural checks only (no `docker` CLI invocation): the self-hosted runner container this actually
+// runs on does not have Docker-in-Docker access, so a test that shells out to `docker compose config`
+// would be unreliable/environment-dependent here (same constraint as docker-compose-override-example.test.ts).
+describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () => {
+  const EXPECTED_LIMITS: Record<string, string> = {
+    gittensory: "${GITTENSORY_MEM_LIMIT:-2g}",
+    redis: "${REDIS_MEM_LIMIT:-512m}",
+    postgres: "${POSTGRES_MEM_LIMIT:-2g}",
+    qdrant: "${QDRANT_MEM_LIMIT:-2g}",
+    ollama: "${OLLAMA_MEM_LIMIT:-8g}",
+    prometheus: "${PROMETHEUS_MEM_LIMIT:-1g}",
+    loki: "${LOKI_MEM_LIMIT:-1g}",
+    tempo: "${TEMPO_MEM_LIMIT:-1g}",
+    grafana: "${GRAFANA_MEM_LIMIT:-512m}",
+  };
+
+  it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => {
+    const compose = readYaml("docker-compose.yml");
+    const services = (compose.services as Record<string, Record<string, unknown>>) ?? {};
+
+    for (const [name, expected] of Object.entries(EXPECTED_LIMITS)) {
+      const service = services[name];
+      expect(service, name).toBeTruthy();
+      const deploy = service?.deploy as { resources?: { limits?: { memory?: unknown } } } | undefined;
+      expect(deploy?.resources?.limits?.memory, name).toBe(expected);
+    }
+  });
+
+  it("documents every memory-limit override variable in .env.example", () => {
+    const env = readFileSync(".env.example", "utf8");
+
+    for (const key of [
+      "GITTENSORY_MEM_LIMIT",
+      "REDIS_MEM_LIMIT",
+      "POSTGRES_MEM_LIMIT",
+      "QDRANT_MEM_LIMIT",
+      "OLLAMA_MEM_LIMIT",
+      "PROMETHEUS_MEM_LIMIT",
+      "LOKI_MEM_LIMIT",
+      "TEMPO_MEM_LIMIT",
+      "GRAFANA_MEM_LIMIT",
+    ]) {
+      expect(env, key).toContain(key);
+    }
+  });
+});