From 4dc1e2de1a71532c9185092efd49c1cdf7782108 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 2 Jul 2026 01:45:27 -0700 Subject: [PATCH 1/2] fix(selfhost): add memory limits to docker-compose services No service declared a memory or CPU limit, including the core gittensory app itself. Any --profile combination let one runaway optional service (Ollama loading a multi-GB model, Postgres, Qdrant, or the whole observability stack) exhaust host RAM and OOM-kill or starve the core review pipeline container instead of the intended optional service -- the exact gap #1828 calls out, with zero mitigation anywhere in the compose file. Add deploy.resources.limits.memory to the core gittensory/redis services and every heavyweight optional service (postgres, qdrant, ollama, prometheus, loki, tempo, grafana), each overridable via a documented .env variable so an operator can size a bigger/smaller host without editing the compose file. --- .env.example | 14 +++++ docker-compose.yml | 40 +++++++++++++ .../selfhost-compose-resource-limits.test.ts | 58 +++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 test/unit/selfhost-compose-resource-limits.test.ts diff --git a/.env.example b/.env.example index 79106403f6..144291e6dd 100644 --- a/.env.example +++ b/.env.example @@ -189,6 +189,20 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # MIGRATIONS_DIR=/app/migrations # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) +# --- Per-service memory limits (docker-compose.yml; #1828/#2495) --- +# Every service below gets a beginner-friendly default `deploy.resources.limits.memory` cap so one runaway +# optional profile (e.g. Ollama loading a multi-GB model) can't OOM-kill or starve the core review pipeline. +# Override any of these for a bigger/smaller host; values use Docker's byte-size shorthand (e.g. 512m, 4g). +# GITTENSORY_MEM_LIMIT=2g # core app (always runs) +# REDIS_MEM_LIMIT=512m # core app (always runs) +# POSTGRES_MEM_LIMIT=2g # --profile postgres / --profile pgbouncer +# QDRANT_MEM_LIMIT=2g # --profile qdrant +# OLLAMA_MEM_LIMIT=8g # --profile ollama; raise this before pulling a large local model +# PROMETHEUS_MEM_LIMIT=1g # --profile observability +# LOKI_MEM_LIMIT=1g # --profile observability +# TEMPO_MEM_LIMIT=1g # --profile observability +# GRAFANA_MEM_LIMIT=512m # --profile observability + # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) --- # Blank is valid until --profile litestream is enabled. # LITESTREAM_ACCESS_KEY_ID= diff --git a/docker-compose.yml b/docker-compose.yml index b454b857bd..7f3e4b1cef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,6 +97,12 @@ services: pgbouncer: condition: service_healthy required: false + # Sane defaults so a runaway optional service (below) can't OOM-kill or starve the core review pipeline; + # override via .env for a bigger/smaller host (#1828). + deploy: + resources: + limits: + memory: "${GITTENSORY_MEM_LIMIT:-2g}" healthcheck: # Probe /ready (not /health): /health is a liveness stub that is 200 even when the DB is down, # whereas /ready returns 503 until the DB answers AND migrations are applied — so dependents that @@ -130,6 +136,10 @@ services: - "" - --appendonly - "no" + deploy: + resources: + limits: + memory: "${REDIS_MEM_LIMIT:-512m}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -146,6 +156,10 @@ services: POSTGRES_DB: gittensory volumes: - gittensory-pg:/var/lib/postgresql/data + deploy: + resources: + limits: + memory: "${POSTGRES_MEM_LIMIT:-2g}" healthcheck: test: ["CMD-SHELL", "pg_isready -U gittensory"] interval: 10s @@ -217,6 +231,10 @@ services: - "127.0.0.1:6334:6334" # gRPC (localhost only) volumes: - qdrant-data:/qdrant/storage + deploy: + resources: + limits: + memory: "${QDRANT_MEM_LIMIT:-2g}" # The image is Debian (bash present) but ships no curl/wget/nc. bash's /dev/tcp pseudo-device speaks # HTTP without an external binary, so we hit Qdrant's k8s-style /readyz probe (200 once it accepts # traffic). This lets the app's depends_on gate on a serving Qdrant. /readyz is unauthenticated even @@ -250,6 +268,12 @@ services: timeout: 5s start_period: 15s retries: 5 + # Ollama can load multi-GB models into memory; default high enough for common 7-8B quantized models but + # cap it so a large model pull can't exhaust host RAM and take down the core review pipeline (#1828). + deploy: + resources: + limits: + memory: "${OLLAMA_MEM_LIMIT:-8g}" # ── Litestream (--profile litestream) ───────────────────────────────────── # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example @@ -307,6 +331,10 @@ services: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME:-180d}" + deploy: + resources: + limits: + memory: "${PROMETHEUS_MEM_LIMIT:-1g}" # Routes Prometheus alerts to your notification channel. Ships SILENT: alerts go to a # null receiver until you fill in a receiver in alertmanager/alertmanager.yml. @@ -351,6 +379,10 @@ services: GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env. GITHUB_TOKEN: "${GITHUB_TOKEN:-}" + deploy: + resources: + limits: + memory: "${GRAFANA_MEM_LIMIT:-512m}" entrypoint: - /bin/sh - -ec @@ -418,6 +450,10 @@ services: volumes: - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro - loki-data:/loki + deploy: + resources: + limits: + memory: "${LOKI_MEM_LIMIT:-1g}" # No healthcheck: grafana/loki:latest is distroless (no shell/wget/curl), so an in-container probe # can't run — a CMD-SHELL check would leave the container stuck "starting". Readiness is observable # at GET /ready; Promtail retries/backs off until Loki accepts pushes and Grafana retries the @@ -493,6 +529,10 @@ services: volumes: - ./tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo-data:/var/tempo + deploy: + resources: + limits: + memory: "${TEMPO_MEM_LIMIT:-1g}" healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] interval: 10s diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts new file mode 100644 index 0000000000..746ed9d383 --- /dev/null +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -0,0 +1,58 @@ +import { readFileSync } from "node:fs"; +import { parse } from "yaml"; +import { describe, expect, it } from "vitest"; + +function readYaml(path: string): Record { + const value = parse(readFileSync(path, "utf8")); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${path} must be a YAML object`); + } + return value as Record; +} + +// Pure structural checks only (no `docker` CLI invocation): the self-hosted runner container this actually +// runs on does not have Docker-in-Docker access, so a test that shells out to `docker compose config` +// would be unreliable/environment-dependent here (same constraint as docker-compose-override-example.test.ts). +describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () => { + const EXPECTED_LIMITS: Record = { + gittensory: "${GITTENSORY_MEM_LIMIT:-2g}", + redis: "${REDIS_MEM_LIMIT:-512m}", + postgres: "${POSTGRES_MEM_LIMIT:-2g}", + qdrant: "${QDRANT_MEM_LIMIT:-2g}", + ollama: "${OLLAMA_MEM_LIMIT:-8g}", + prometheus: "${PROMETHEUS_MEM_LIMIT:-1g}", + loki: "${LOKI_MEM_LIMIT:-1g}", + tempo: "${TEMPO_MEM_LIMIT:-1g}", + grafana: "${GRAFANA_MEM_LIMIT:-512m}", + }; + + it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => { + const compose = readYaml("docker-compose.yml"); + const services = (compose.services as Record>) ?? {}; + + for (const [name, expected] of Object.entries(EXPECTED_LIMITS)) { + const service = services[name]; + expect(service, name).toBeTruthy(); + const deploy = service.deploy as { resources?: { limits?: { memory?: unknown } } } | undefined; + expect(deploy?.resources?.limits?.memory, name).toBe(expected); + } + }); + + it("documents every memory-limit override variable in .env.example", () => { + const env = readFileSync(".env.example", "utf8"); + + for (const key of [ + "GITTENSORY_MEM_LIMIT", + "REDIS_MEM_LIMIT", + "POSTGRES_MEM_LIMIT", + "QDRANT_MEM_LIMIT", + "OLLAMA_MEM_LIMIT", + "PROMETHEUS_MEM_LIMIT", + "LOKI_MEM_LIMIT", + "TEMPO_MEM_LIMIT", + "GRAFANA_MEM_LIMIT", + ]) { + expect(env, key).toContain(key); + } + }); +}); From 7c16a3f492fdc666041c9504a76e570b8280afcc Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 2 Jul 2026 02:04:07 -0700 Subject: [PATCH 2/2] fix(selfhost): guard the possibly-undefined service lookup in the resource-limits test --- test/unit/selfhost-compose-resource-limits.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts index 746ed9d383..29ccc36500 100644 --- a/test/unit/selfhost-compose-resource-limits.test.ts +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -33,7 +33,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () = for (const [name, expected] of Object.entries(EXPECTED_LIMITS)) { const service = services[name]; expect(service, name).toBeTruthy(); - const deploy = service.deploy as { resources?: { limits?: { memory?: unknown } } } | undefined; + const deploy = service?.deploy as { resources?: { limits?: { memory?: unknown } } } | undefined; expect(deploy?.resources?.limits?.memory, name).toBe(expected); } });