fix(selfhost): prune stopped containers by actual stop time, not creation time

Docker's `container prune --filter until=<n>h` filters by creation
time, not stop time, so a long-lived container stopped moments ago
was already eligible for deletion on the very next scheduled run.
prune_stopped_containers() now inspects each exited container's
State.FinishedAt directly, comparing it against a cutoff computed
from epoch arithmetic and formatted portably for both GNU and
BusyBox date, so only containers actually stopped for at least
RETAIN_HOURS are ever removed.

Author: JSONbored

apps/gittensory-ui/src/lib/selfhost-env-reference.ts

@@ -11,11 +11,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "AI_EMBED_API_KEY",
-    firstReference: "src/server.ts:419",
+    firstReference: "src/server.ts:420",
   },
   {
     name: "AI_EMBED_BASE_URL",
-    firstReference: "src/server.ts:416",
+    firstReference: "src/server.ts:417",
   },
   {
     name: "AI_EMBED_MODEL",
@@ -43,7 +43,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "BACKUP_ACKNOWLEDGED",
-    firstReference: "src/server.ts:358",
+    firstReference: "src/server.ts:359",
   },
   {
     name: "BROWSER_WS_ENDPOINT",
@@ -75,11 +75,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "CRON_INTERVAL_MS",
-    firstReference: "src/server.ts:839",
+    firstReference: "src/server.ts:845",
   },
   {
     name: "DATABASE_PATH",
-    firstReference: "src/server.ts:241",
+    firstReference: "src/server.ts:242",
   },
   {
     name: "DATABASE_URL",
@@ -103,11 +103,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "GITHUB_CACHE_TTL_SECONDS",
-    firstReference: "src/server.ts:487",
+    firstReference: "src/server.ts:488",
   },
   {
     name: "GITTENSORY_REPO_CONFIG_DIR",
-    firstReference: "src/server.ts:275",
+    firstReference: "src/server.ts:276",
   },
   {
     name: "GITTENSORY_VERSION",
@@ -119,7 +119,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "MIGRATIONS_DIR",
-    firstReference: "src/server.ts:371",
+    firstReference: "src/server.ts:372",
   },
   {
     name: "OBSERVABILITY_SMOKE_POLL_MS",
@@ -179,7 +179,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_BROKER_URL",
-    firstReference: "src/server.ts:883",
+    firstReference: "src/server.ts:889",
   },
   {
     name: "ORB_COLLECTOR_TOKEN",
@@ -195,7 +195,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_RELAY_MODE",
-    firstReference: "src/server.ts:885",
+    firstReference: "src/server.ts:891",
   },
   {
     name: "OTEL_EXPORTER_OTLP_ENDPOINT",
@@ -227,11 +227,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "PGVECTOR_ENABLED",
-    firstReference: "src/server.ts:221",
+    firstReference: "src/server.ts:222",
   },
   {
     name: "PORT",
-    firstReference: "src/server.ts:643",
+    firstReference: "src/server.ts:644",
   },
   {
     name: "PUBLIC_API_ORIGIN",
@@ -247,7 +247,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "QDRANT_URL",
-    firstReference: "src/server.ts:506",
+    firstReference: "src/server.ts:507",
   },
   {
     name: "QUEUE_BACKGROUND_CONCURRENCY",
@@ -259,7 +259,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "REVIEW_AUDIT_DIR",
-    firstReference: "src/server.ts:551",
+    firstReference: "src/server.ts:552",
   },
   {
     name: "SELFHOST_BUNDLE_ALL",
@@ -295,42 +295,42 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "SETUP_OUTPUT_PATH",
-    firstReference: "src/server.ts:760",
+    firstReference: "src/server.ts:761",
   },
 ];
 
 export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| Name | First reference |",
   "| --- | --- |",
   "| `AI_COMBINE` | `src/selfhost/ai.ts:888` |",
-  "| `AI_EMBED_API_KEY` | `src/server.ts:419` |",
-  "| `AI_EMBED_BASE_URL` | `src/server.ts:416` |",
+  "| `AI_EMBED_API_KEY` | `src/server.ts:420` |",
+  "| `AI_EMBED_BASE_URL` | `src/server.ts:417` |",
   "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:784` |",
   "| `AI_ON_MERGE` | `src/selfhost/ai.ts:890` |",
   "| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |",
   "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:788` |",
   "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:57` |",
   "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:787` |",
-  "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:358` |",
+  "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:359` |",
   "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |",
   "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:108` |",
   "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:49` |",
   "| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:108` |",
   "| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts:112` |",
   "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:53` |",
   "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:112` |",
-  "| `CRON_INTERVAL_MS` | `src/server.ts:839` |",
-  "| `DATABASE_PATH` | `src/server.ts:241` |",
+  "| `CRON_INTERVAL_MS` | `src/server.ts:845` |",
+  "| `DATABASE_PATH` | `src/server.ts:242` |",
   "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |",
   "| `DISCORD_REPO_WEBHOOKS` | `src/selfhost/discord-notify.ts:31` |",
   "| `DISCORD_WEBHOOK_URL` | `src/selfhost/discord-notify.ts:40` |",
   "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
   "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |",
-  "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:487` |",
-  "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:275` |",
+  "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:488` |",
+  "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:276` |",
   "| `GITTENSORY_VERSION` | `src/selfhost/health.ts:29` |",
   "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:83` |",
-  "| `MIGRATIONS_DIR` | `src/server.ts:371` |",
+  "| `MIGRATIONS_DIR` | `src/server.ts:372` |",
   "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |",
   "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |",
   "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:781` |",
@@ -345,27 +345,27 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |",
   "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |",
   "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
-  "| `ORB_BROKER_URL` | `src/server.ts:883` |",
+  "| `ORB_BROKER_URL` | `src/server.ts:889` |",
   "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |",
   "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |",
   "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |",
-  "| `ORB_RELAY_MODE` | `src/server.ts:885` |",
+  "| `ORB_RELAY_MODE` | `src/server.ts:891` |",
   "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |",
   "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |",
   "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |",
   "| `OTEL_SERVICE_NAME` | `src/selfhost/otel.ts:59` |",
   "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts:40` |",
   "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |",
   "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |",
-  "| `PGVECTOR_ENABLED` | `src/server.ts:221` |",
-  "| `PORT` | `src/server.ts:643` |",
+  "| `PGVECTOR_ENABLED` | `src/server.ts:222` |",
+  "| `PORT` | `src/server.ts:644` |",
   "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |",
   "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |",
   "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |",
-  "| `QDRANT_URL` | `src/server.ts:506` |",
+  "| `QDRANT_URL` | `src/server.ts:507` |",
   "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:102` |",
   "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |",
-  "| `REVIEW_AUDIT_DIR` | `src/server.ts:551` |",
+  "| `REVIEW_AUDIT_DIR` | `src/server.ts:552` |",
   "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |",
   "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.mjs:5` |",
   "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts:186` |",
@@ -374,5 +374,5 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `SENTRY_RELEASE` | `src/selfhost/otel.ts:62` |",
   "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts:373` |",
   "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts:161` |",
-  "| `SETUP_OUTPUT_PATH` | `src/server.ts:760` |",
+  "| `SETUP_OUTPUT_PATH` | `src/server.ts:761` |",
 ].join("\n");

scripts/selfhost-docker-prune.sh

@@ -6,10 +6,20 @@
 # service comments on why raw docker.sock exposure into a container is avoided).
 #
 # Age-filtered so nothing built/pulled recently is touched -- a rollback within the retention window still
-# has its image available. `docker container prune`, `docker image prune -a`, and `docker builder prune`
-# only ever remove resources Docker itself already reports as unused (a running container, its own image,
-# or an active build-cache entry a build is currently using, are never candidates) -- this script does not
-# change that safety property, it only adds the age floor on top of it.
+# has its image available. `docker image prune -a` and `docker builder prune` only ever remove resources
+# Docker itself already reports as unused (a running container's own image, or an active build-cache entry
+# a build is currently using, are never candidates) -- this script does not change that safety property, it
+# only adds the age floor on top of it. Docker's own `until=` filter for both is CREATION time, which is
+# fine there: an in-use image/cache entry can never be a prune candidate in the first place regardless of
+# how old it is, so the age floor only ever protects a recently built one that isn't in use yet.
+#
+# Containers are handled differently and deliberately do NOT use `docker container prune --filter until=`:
+# that filter is ALSO creation time, not stop time (verified against Docker's own docs) -- a long-running
+# container stopped moments ago (e.g. an operator's `docker compose stop <svc>` to inspect a live issue)
+# would already be older than the retention window by creation date, so a creation-time filter deletes it
+# on the very next scheduled run instead of giving the intended grace period. prune_stopped_containers()
+# below inspects each exited container's ACTUAL State.FinishedAt instead, so only a container that has
+# itself been stopped for at least RETAIN_HOURS is ever removed.
 #
 # SAFE BY DESIGN: only prunes stopped containers, unused images, and build cache -- NEVER volumes
 # (gittensory-data, gittensory-backups, postgres-data, qdrant-storage, runner-work, etc.), so it cannot
@@ -38,23 +48,70 @@ if ! command -v docker >/dev/null 2>&1; then
   exit 1
 fi
 
+# Prunes (real mode) or reports (dry-run) stopped containers whose ACTUAL stop time is at least
+# RETAIN_HOURS in the past -- see the header comment for why `docker container prune --filter until=`
+# itself is not used. Walks each exited container's `docker inspect` State.FinishedAt individually.
+#
+# Deliberately never parses FinishedAt's ISO-8601 string with `date -d` -- GNU coreutils' `date -d` accepts
+# it, but BusyBox `date` (Alpine and other minimal Linux hosts, a common lightweight Docker host choice)
+# only understands a small fixed set of `-d` inputs (`@<epoch>`, `hh:mm[:ss]`, `YYYY-MM-DD hh:mm[:ss]`, ...)
+# and rejects FinishedAt's fractional-second form outright. Because that failure was wrapped in
+# `2>/dev/null || continue`, it used to fail SILENTLY -- every container would be skipped forever, with no
+# error, defeating this feature's whole purpose on a BusyBox host without so much as a warning.
+#
+# Instead: format the cutoff (an EPOCH INTEGER computed by plain arithmetic, never parsed from a string) as
+# an ISO-8601-prefix string via `date -d @<epoch>` -- the `@<epoch>` form IS in BusyBox's small supported
+# set, unlike arbitrary ISO-8601 parsing -- then compare that against FinishedAt's own first-19-characters
+# prefix LEXICOGRAPHICALLY. That comparison is chronologically correct because both sides are the same
+# fixed-width, zero-padded, UTC "YYYY-MM-DDTHH:MM:SS" shape, and needs no date-string PARSING at all.
+prune_stopped_containers() {
+  now_epoch=$(date -u +%s)
+  cutoff_epoch=$((now_epoch - RETAIN_HOURS * 3600))
+  cutoff_iso=$(date -u -d "@${cutoff_epoch}" +%Y-%m-%dT%H:%M:%S)
+  container_list=$(mktemp)
+  docker ps -a --filter status=exited --format '{{.ID}}' > "$container_list"
+  # Reads from a FILE (not a pipe) deliberately: `cmd | while read; do ...; done` runs the loop body in a
+  # subshell under POSIX sh, silently discarding any variable set inside it once the loop ends -- harmless
+  # today (nothing here is read after the loop), but a real trap for a future edit that adds e.g. a
+  # removed-count summary. `done < file` has no such subshell.
+  while IFS= read -r cid; do
+    [ -n "$cid" ] || continue
+    finished_at=$(docker inspect -f '{{.State.FinishedAt}}' "$cid" 2>/dev/null) || continue
+    finished_prefix=$(printf '%s' "$finished_at" | cut -c1-19)
+    case "$finished_prefix" in
+      [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]) ;;
+      *) continue ;; # not a recognizable timestamp shape -- skip rather than guess
+    esac
+    # `[ a \< b ]` is not defined by POSIX test for strings; `expr` IS specified to compare non-numeric
+    # operands lexicographically, so it's used here for a portable string-ordering check.
+    expr "$finished_prefix" '<' "$cutoff_iso" >/dev/null || continue
+    if [ "$DRY_RUN" = 1 ]; then
+      echo "[docker-prune] DRY RUN -- would remove stopped container $cid (stopped before ${cutoff_iso}Z)"
+    elif docker rm "$cid" >/dev/null 2>&1; then
+      echo "[docker-prune] removed stopped container $cid (stopped before ${cutoff_iso}Z)"
+    else
+      echo "[docker-prune] WARNING: failed to remove stopped container $cid" >&2
+    fi
+  done < "$container_list"
+  rm -f "$container_list"
+}
+
 echo "[docker-prune] $(date -u +%FT%TZ) starting (retain: ${RETAIN_HOURS}h, dry-run: ${DRY_RUN})"
 echo "[docker-prune] before:"
 docker system df
 echo "[docker-prune] root filesystem usage:"
 df -h / 2>/dev/null || true
 
+echo "[docker-prune] pruning stopped containers older than ${RETAIN_HOURS}h (by actual stop time)..."
+prune_stopped_containers
+
 if [ "$DRY_RUN" = 1 ]; then
-  echo "[docker-prune] DRY RUN -- would run: docker container prune -f --filter until=${RETAIN_HOURS}h"
   echo "[docker-prune] DRY RUN -- would run: docker image prune -af --filter until=${RETAIN_HOURS}h"
   echo "[docker-prune] DRY RUN -- would run: docker builder prune -af --filter until=${RETAIN_HOURS}h"
   echo "[docker-prune] volumes are NEVER pruned by this script -- application data, backups, and runner state are always safe."
   exit 0
 fi
 
-echo "[docker-prune] pruning stopped containers older than ${RETAIN_HOURS}h..."
-docker container prune -f --filter "until=${RETAIN_HOURS}h"
-
 echo "[docker-prune] pruning unused images older than ${RETAIN_HOURS}h..."
 docker image prune -af --filter "until=${RETAIN_HOURS}h"