Merge pull request #5977 from JSONbored/fix/selfhost-config-dir-empty-guardrail

feat(selfhost): warn loudly when the private config mount is empty

Author: JSONbored

apps/loopover-ui/src/lib/selfhost-env-reference.ts

@@ -129,6 +129,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "CODEX_HOME",
     firstReference: "src/selfhost/ai.ts",
   },
+  {
+    name: "CONFIG_DIR_EMPTY_ACKNOWLEDGED",
+    firstReference: "src/server.ts",
+  },
   {
     name: "CRON_INTERVAL_MS",
     firstReference: "src/server.ts",
@@ -513,6 +517,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts` |",
   "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts` |",
   "| `CODEX_HOME` | `src/selfhost/ai.ts` |",
+  "| `CONFIG_DIR_EMPTY_ACKNOWLEDGED` | `src/server.ts` |",
   "| `CRON_INTERVAL_MS` | `src/server.ts` |",
   "| `DATABASE_PATH` | `src/server.ts` |",
   "| `DATABASE_URL` | `src/selfhost/preflight.ts` |",

apps/loopover-ui/src/routes/docs.self-hosting-operations.tsx

@@ -1142,7 +1142,8 @@ git merge --ff-only origin/main
 curl -sf http://localhost:8787/ready
 docker compose ps loopover
 grep -E '^(LOOPOVER_IMAGE|LOOPOVER_VERSION|SENTRY_RELEASE)=' .env
-docker inspect --format '{{.Config.Image}}' "$(docker compose ps -q loopover)"`}
+docker inspect --format '{{.Config.Image}}' "$(docker compose ps -q loopover)"
+docker exec "$(docker compose ps -q loopover)" sh -c 'ls -A "\${LOOPOVER_REPO_CONFIG_DIR:-/config}" | wc -l'`}
       />
       <p>
         If any check fails, see <Link to="/docs/self-hosting-troubleshooting">Troubleshooting</Link>

scripts/selfhost-post-update-check.sh

@@ -4,7 +4,8 @@
 # Run after deploy-selfhost-image.sh, deploy-selfhost-prebuilt.sh, or any manual
 # `docker compose up -d --no-deps loopover` that ships a new app image.
 #
-# Checks /ready, compose health, .env release metadata, and the running container image.
+# Checks /ready, compose health, .env release metadata, the running container image, and whether the
+# container-private config mount (LOOPOVER_REPO_CONFIG_DIR) is unexpectedly empty.
 # Does not modify .env, volumes, loopover-config/, or any profile service.
 set -euo pipefail
 
@@ -93,4 +94,18 @@ fi
 
 running_image="$(docker inspect --format '{{.Config.Image}}' "$container_id")"
 echo "selfhost post-update check: running image=$running_image"
+
+# Config-drift guardrail (a live incident during the gittensory->loopover rename): docker-compose.yml's
+# LOOPOVER_REPO_CONFIG_DIR bind mount silently degrades to an empty directory -- not an error -- when its host
+# source directory doesn't exist (e.g. renamed/moved without updating the mount, or simply never created). Every
+# per-repo and global setting then falls back to built-in defaults with zero visible symptoms until someone
+# notices the behavior change. This is a READ-ONLY check (matches the file header above: never modifies
+# loopover-config/) run every time this script runs, i.e. after every deploy -- exactly when a mount-path change
+# would land. Non-fatal: an empty mount is also the correct, expected state for a fresh install with no private
+# config written yet, so this warns rather than exits non-zero.
+config_dir_entries="$(docker exec "$container_id" sh -c 'dir="${LOOPOVER_REPO_CONFIG_DIR:-/config}"; [ -d "$dir" ] && ls -A "$dir" | wc -l || echo 0' 2>/dev/null || true)"
+if [[ "$config_dir_entries" =~ ^[0-9]+$ ]] && [ "$config_dir_entries" -eq 0 ]; then
+  echo "selfhost post-update check: warning — the container's private config directory (LOOPOVER_REPO_CONFIG_DIR, default /config) is empty; every per-repo and global setting is silently using built-in defaults. If you expected private config to apply, verify the host directory wasn't renamed or moved without updating docker-compose.yml's bind mount." >&2
+fi
+
 echo "selfhost post-update check: ok"

src/selfhost/health.ts

@@ -231,3 +231,26 @@ export function publicOriginAcknowledgedGaugeValue(opts: {
 }): 0 | 1 {
   return publicOriginReachabilityAdvisory(opts) === null ? 1 : 0;
 }
+
+/** Boot-time advisory (a live incident during the gittensory->loopover rename): `LOOPOVER_REPO_CONFIG_DIR`
+ *  points the focus-manifest loader at a container-private per-repo config mount (`private-config.ts`) that
+ *  silently and validly degrades to "no local config" when the mounted directory is empty — every setting
+ *  (labels, gate, autonomy, ...) then falls back to built-in defaults with NO error, because an empty mount is
+ *  also the correct, expected state for a brand-new install that hasn't written any `.loopover.yml` yet. The
+ *  incident: a docker-compose.yml change renamed the bind-mount source directory convention
+ *  (`./gittensory-config` -> `./loopover-config`) as a documented breaking change requiring operators to `mv`
+ *  their existing directory to match — that manual step was missed on deploy, Docker silently created an empty
+ *  directory at the new path, and every repo's config-driven settings (including `autoLabelEnabled`) reverted
+ *  to defaults for about a day before anyone noticed. Mirrors {@link sqliteBackupAdvisory}'s shape: warns
+ *  rather than blocks (an empty dir is legitimate for a fresh install), and the operator can silence it with
+ *  `CONFIG_DIR_EMPTY_ACKNOWLEDGED=true` once they've confirmed it's intentional. */
+export function emptyConfigDirAdvisory(opts: { configured: boolean; entryCount: number; acknowledged: boolean }): string | null {
+  if (!opts.configured || opts.acknowledged || opts.entryCount > 0) return null;
+  return `LOOPOVER_REPO_CONFIG_DIR is set but the mounted directory is empty — every per-repo and global setting (labels, gate, autonomy, ...) is silently using built-in defaults instead of your .loopover.yml config. This usually means the host directory was renamed or moved without updating the bind mount (see docker-compose.yml's "volumes:" comment), or the volume didn't mount as expected. If this is intentional — a fresh install with no config written yet — set CONFIG_DIR_EMPTY_ACKNOWLEDGED=true to silence this warning.`;
+}
+
+/** Prometheus gauge value mirroring {@link emptyConfigDirAdvisory}: 1 when the mount isn't configured, has
+ *  entries, or the operator acknowledged it, 0 when the advisory would fire. */
+export function emptyConfigDirAcknowledgedGaugeValue(opts: { configured: boolean; entryCount: number; acknowledged: boolean }): 0 | 1 {
+  return emptyConfigDirAdvisory(opts) === null ? 1 : 0;
+}

src/selfhost/metrics.ts

@@ -54,6 +54,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["loopover_clock_skew_seconds", { help: "Clock skew in seconds between this process and GitHub's server time (positive = ahead), sampled from GitHub App JWT-mint response Date headers.", type: "gauge" }],
   ["loopover_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }],
   ["loopover_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }],
+  ["loopover_config_dir_empty_acknowledged", { help: "1 when LOOPOVER_REPO_CONFIG_DIR is unset, has entries, or is acknowledged; 0 when it's configured but the mounted directory is empty.", type: "gauge" }],
   ["loopover_http_requests_total", { help: "HTTP app requests by response status class.", type: "counter" }],
   ["loopover_http_request_duration_seconds", { help: "HTTP app request duration in seconds.", type: "histogram" }],
   ["loopover_webhook_dedup_total", { help: "Webhook deliveries deduplicated before enqueue.", type: "counter" }],

src/server.ts

@@ -6,7 +6,7 @@
 // Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same
 // scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare
 // Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles.
-import { existsSync, writeFileSync } from "node:fs";
+import { existsSync, readdirSync, writeFileSync } from "node:fs";
 import { delimiter, join } from "node:path";
 import { randomUUID } from "node:crypto";
 import { DatabaseSync } from "node:sqlite";
@@ -48,6 +48,8 @@ import {
   backupAcknowledgedGaugeValue,
   buildHealthBody,
   codexAuthReadinessProbe,
+  emptyConfigDirAcknowledgedGaugeValue,
+  emptyConfigDirAdvisory,
   githubAppReadinessProbe,
   publicOriginAcknowledgedGaugeValue,
   publicOriginReachabilityAdvisory,
@@ -113,6 +115,16 @@ function nonBlank(value: string | undefined): string | undefined {
   return trimmed ? trimmed : undefined;
 }
 
+/** Top-level entry count of `dir` (files + subdirectories, dotfiles included), or 0 on any read error --
+ *  a missing/unreadable directory is reported the same as an empty one rather than crashing boot. */
+function safeReaddirCount(dir: string): number {
+  try {
+    return readdirSync(dir).length;
+  } catch {
+    return 0;
+  }
+}
+
 
 interface Backend {
   db: D1Database;
@@ -298,15 +310,36 @@ async function main(): Promise<void> {
   // Boot-time visibility (config-drift guardrail): state which config dir is actually in effect, unconditionally
   // -- neither reader above logs anything, so an operator previously had no way to confirm from the logs alone
   // which directory (if any) was live, which is exactly the ambiguity that let a stale, no-longer-mounted config
-  // path get mistaken for the real one during a past incident. Never touches or validates any file; this is
-  // purely a log line, same "state what's in effect" shape as the sentry/otel boot logs below.
+  // path get mistaken for the real one during a past incident. `entryCount` is a cheap, one-time top-level
+  // listing (never recursive, never touches file contents) so a SECOND incident of the same shape -- the mount
+  // resolving but landing on an empty directory -- is visible in the log line itself, not just "some path is
+  // configured" (see emptyConfigDirAdvisory below for the loud version of this same signal).
+  const configDirOpts = {
+    configured: Boolean(repoConfigDir),
+    // A missing (as opposed to merely empty) directory is treated the same as zero entries -- both mean "no
+    // local config was actually read" -- rather than letting a bad path crash the whole server at boot.
+    entryCount: repoConfigDir ? safeReaddirCount(repoConfigDir) : 0,
+    acknowledged: process.env.CONFIG_DIR_EMPTY_ACKNOWLEDGED === "true",
+  };
   console.log(
     JSON.stringify({
       event: "selfhost_config_dir",
-      configured: Boolean(repoConfigDir),
+      configured: configDirOpts.configured,
       dir: repoConfigDir ?? null,
+      entryCount: repoConfigDir ? configDirOpts.entryCount : null,
     }),
   );
+  // Config-drift advisory: warn LOUDLY (not just the log line above) when the mount resolves but is empty --
+  // see emptyConfigDirAdvisory's own doc comment for the incident this guards against.
+  const configDirAdvisory = emptyConfigDirAdvisory(configDirOpts);
+  if (configDirAdvisory)
+    console.warn(
+      JSON.stringify({
+        level: "warn",
+        event: "selfhost_config_dir_empty_advisory",
+        message: configDirAdvisory,
+      }),
+    );
   // Error tracking (#1468): opt-in via SENTRY_DSN — a complete no-op when unset. When on, capture uncaught crashes
   // + unhandled rejections (flush before exit for the fatal case); per-subsystem captures (queue dead-letter,
   // review failures) are wired at their sites.
@@ -781,6 +814,7 @@ async function main(): Promise<void> {
   );
   gauge("loopover_backup_acknowledged", () => backupAcknowledgedGaugeValue(sqliteBackupOpts));
   gauge("loopover_public_origin_acknowledged", () => publicOriginAcknowledgedGaugeValue(publicOriginOpts));
+  gauge("loopover_config_dir_empty_acknowledged", () => emptyConfigDirAcknowledgedGaugeValue(configDirOpts));
   // Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters
   // created on first use would otherwise cause "No data" in Grafana until the first job event).
   for (const c of [

test/unit/selfhost-health.test.ts

@@ -8,6 +8,8 @@ import {
   backupAcknowledgedGaugeValue,
   buildHealthBody,
   codexAuthReadinessProbe,
+  emptyConfigDirAcknowledgedGaugeValue,
+  emptyConfigDirAdvisory,
   githubAppReadinessProbe,
   publicOriginAcknowledgedGaugeValue,
   publicOriginReachabilityAdvisory,
@@ -311,6 +313,35 @@ describe("publicOriginAcknowledgedGaugeValue (#4180)", () => {
   });
 });
 
+describe("emptyConfigDirAdvisory (gittensory->loopover rename incident)", () => {
+  it("is silent when LOOPOVER_REPO_CONFIG_DIR is unset entirely — a normal, unconfigured install", () => {
+    expect(emptyConfigDirAdvisory({ configured: false, entryCount: 0, acknowledged: false })).toBeNull();
+  });
+
+  it("is silent when the mounted directory has at least one entry", () => {
+    expect(emptyConfigDirAdvisory({ configured: true, entryCount: 1, acknowledged: false })).toBeNull();
+  });
+
+  it("is silent when acknowledged, even if configured and empty", () => {
+    expect(emptyConfigDirAdvisory({ configured: true, entryCount: 0, acknowledged: true })).toBeNull();
+  });
+
+  it("regression: warns when configured but the mounted directory is empty — the exact incident shape", () => {
+    const message = emptyConfigDirAdvisory({ configured: true, entryCount: 0, acknowledged: false });
+    expect(message).toMatch(/mounted directory is empty/);
+    expect(message).toMatch(/CONFIG_DIR_EMPTY_ACKNOWLEDGED/);
+  });
+});
+
+describe("emptyConfigDirAcknowledgedGaugeValue (gittensory->loopover rename incident)", () => {
+  it("mirrors the advisory: 0 only when configured and empty and unacknowledged", () => {
+    expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 0, acknowledged: false })).toBe(0);
+    expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 0, acknowledged: true })).toBe(1);
+    expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 1, acknowledged: false })).toBe(1);
+    expect(emptyConfigDirAcknowledgedGaugeValue({ configured: false, entryCount: 0, acknowledged: false })).toBe(1);
+  });
+});
+
 describe("readiness (#982)", () => {
   afterEach(() => {
     vi.restoreAllMocks();

test/unit/selfhost-post-update-check-script.test.ts

@@ -44,6 +44,10 @@ if [ "$1" = "inspect" ] && [ "$2" = "--format" ]; then
   fi
   exit 0
 fi
+if [ "$1" = "exec" ]; then
+  printf '%s\\n' "\${CONFIG_DIR_ENTRIES:-3}"
+  exit 0
+fi
 exit 0
 `,
   );
@@ -100,4 +104,19 @@ describe("selfhost-post-update-check.sh", () => {
     expect(result.status).toBe(1);
     expect(result.stderr).toContain("after 2 attempts (6s)");
   });
+
+  it("regression: warns (without failing) when the container's private config mount is empty", () => {
+    const result = run({ CONFIG_DIR_ENTRIES: "0" });
+
+    expect(result.status, result.stderr).toBe(0);
+    expect(result.stderr).toContain("private config directory (LOOPOVER_REPO_CONFIG_DIR, default /config) is empty");
+    expect(result.stdout).toContain("selfhost post-update check: ok");
+  });
+
+  it("is silent when the container's private config mount has entries", () => {
+    const result = run({ CONFIG_DIR_ENTRIES: "4" });
+
+    expect(result.status, result.stderr).toBe(0);
+    expect(result.stderr).not.toContain("private config directory");
+  });
 });