fix(selfhost): keep runner temp files off docker overlay

apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx

@@ -154,6 +154,70 @@ sudo systemctl enable --now gittensory-docker-prune.timer`}
         Run it manually at any time with <code>docker system df</code> before and after to see what
         it reclaimed: <code>sh scripts/selfhost-docker-prune.sh</code>.
       </p>
+      <p>
+        This should always prune <strong>containers, images, and build cache</strong> — never
+        volumes. Pruning a volume deletes real application state (the database, backups, vector
+        index, or a runner&apos;s registration and job data), not disposable build output, so it is
+        never part of routine cleanup unless you intentionally want to delete that state.
+      </p>
+
+      <h2>Self-hosted runner temp storage</h2>
+      <p>
+        If you run <code>--profile runners</code>, keep every runner job&apos;s scratch/temp writes
+        on the mounted <code>runner-work</code> volume, never the container&apos;s plain{" "}
+        <code>/tmp</code>. A container&apos;s own <code>/tmp</code> lives in Docker&apos;s
+        overlay/containerd snapshot storage — a CI job that writes high-volume temp data there
+        (language toolchain caches, build artifacts, ad hoc <code>mktemp</code> calls) grows the
+        host&apos;s Docker root storage directly, not the volume, so it is invisible to
+        volume-scoped cleanup and can fill the disk out from under the whole stack. The shipped{" "}
+        <code>runner</code> service points <code>TMPDIR</code>, <code>TMP</code>, and{" "}
+        <code>TEMP</code> at <code>/tmp/runner/tmp</code> (a subdirectory of the mounted{" "}
+        <code>runner-work</code> volume) and keeps <code>RUNNER_WORKDIR</code> at{" "}
+        <code>/tmp/runner</code> on the same volume. A one-shot <code>runner-tmp-init</code> service
+        creates that directory on the volume (and makes it world-writable, matching real{" "}
+        <code>/tmp</code> permissions) before the runner container starts, so this works out of the
+        box on a fresh volume with no manual steps.
+      </p>
+      <p>
+        Adding a second or third runner service in <code>docker-compose.override.yml</code> for
+        higher CI throughput? Each one needs its own <code>runner-work</code>-style volume, its own
+        init step, and the same temp env — YAML anchors don&apos;t cross separate compose files, so
+        repeat the extension block in your override file:
+      </p>
+      <CodeBlock
+        lang="yaml"
+        code={`x-runner-tmp-env: &runner-tmp-env
+  TMPDIR: /tmp/runner/tmp
+  TMP: /tmp/runner/tmp
+  TEMP: /tmp/runner/tmp
+
+services:
+  runner-2-tmp-init:
+    image: alpine:3.20
+    profiles: ["runners"]
+    volumes:
+      - runner-work-2:/tmp/runner
+    command: ["sh", "-c", "mkdir -p /tmp/runner/tmp && chmod 1777 /tmp/runner/tmp"]
+
+  runner-2:
+    image: myoung34/github-runner:ubuntu-jammy
+    profiles: ["runners"]
+    depends_on:
+      runner-2-tmp-init:
+        condition: service_completed_successfully
+    environment:
+      <<: *runner-tmp-env
+      RUNNER_NAME: gittensory-runner-2
+      RUNNER_SCOPE: \${RUNNER_SCOPE:-repo}
+      REPO_URL: \${RUNNER_REPO_URL:-}
+      RUNNER_TOKEN: \${RUNNER_TOKEN:-}
+      RUNNER_WORKDIR: /tmp/runner
+    volumes:
+      - runner-work-2:/tmp/runner
+
+volumes:
+  runner-work-2:`}
+      />
 
       <h2>Sentry tracing</h2>
       <p>

docker-compose.override.yml.example

@@ -34,6 +34,16 @@
 # The RECOMMENDED alternative to all of this: don't co-locate runners with the review stack at all -- use
 # GitHub-hosted CI, or a separate dedicated runner host (see the runner service's comment in
 # docker-compose.yml).
+#
+# SCALING TO MULTIPLE RUNNERS: adding a second/third runner service here (e.g. to raise CI throughput)?
+# Each one MUST set the same TMPDIR/TMP/TEMP env (pointed at its own mounted runner-work-style volume,
+# never left at the container's plain /tmp) and mount its own named volume at /tmp/runner -- otherwise its
+# CI job temp writes fall back to the container's overlay storage (the exact growth/disk-pressure problem
+# this repo's runner service is set up to avoid). YAML anchors don't cross separate compose files, so you
+# cannot reference docker-compose.yml's `x-runner-tmp-env` anchor from this file directly -- either repeat
+# the three env vars literally, or redeclare the SAME `x-runner-tmp-env: &runner-tmp-env` extension block
+# at the top of your own override file and merge it with `<<: *runner-tmp-env`. See the self-hosting
+# operations docs for a complete, copy-pasteable multi-runner service snippet.
 
 services:
   gittensory:

docker-compose.yml

@@ -41,6 +41,23 @@ x-logging: &default-logging
       max-size: "10m"
       max-file: "3"
 
+# Self-hosted GitHub Actions runner temp env (#selfhost-runner-tmp): a container's plain /tmp lives in
+# Docker's overlay/containerd snapshot storage, NOT any mounted volume -- high-volume CI scratch writes
+# there (language toolchains, build caches, ad hoc `mktemp` calls) grow the HOST's Docker root storage
+# directly and are invisible to volume-scoped cleanup (confirmed in production: unbounded overlay growth,
+# disk pressure, and `docker system df` races against disappearing temp files). Every self-hosted-runner
+# service -- the `runner` service below, merged in via `<<: *runner-tmp-env`, AND any additional runner
+# service you add for a multi-runner deployment -- must point TMPDIR/TMP/TEMP at its own mounted
+# runner-work-style volume the same way, so job temp writes always land on mounted storage instead of the
+# container's writable layer. A second compose file (e.g. docker-compose.override.yml) CANNOT reference
+# this anchor directly -- YAML anchors don't cross files -- so an override adding another runner service
+# must redeclare this same x-runner-tmp-env block in its own file before merging it. See
+# docker-compose.override.yml.example and the self-hosting operations docs for the full pattern.
+x-runner-tmp-env: &runner-tmp-env
+  TMPDIR: /tmp/runner/tmp
+  TMP: /tmp/runner/tmp
+  TEMP: /tmp/runner/tmp
+
 services:
   # ── Core app (always runs) ─────────────────────────────────────────────────
   gittensory:
@@ -620,14 +637,32 @@ services:
   # containers on an 8-vCPU box left the app starved under load. See docker-compose.override.yml.example
   # for a proven CPU-priority pattern (relative cpu_shares + a per-container cpus ceiling) before scaling
   # this up, and ./scripts/docker-prune.sh for reclaiming the disk space CI builds accumulate.
+  # One-shot bootstrap for the runner's TMPDIR (below): guarantees /tmp/runner/tmp exists (and is
+  # world-writable, matching real /tmp's mode -- the runner image's runtime user isn't guaranteed to be
+  # root) on the runner-work volume BEFORE the runner container starts. A fresh named volume has nothing
+  # at that path otherwise, and pointing TMPDIR at a directory that doesn't exist yet would break every
+  # job step that shells out expecting $TMPDIR to be usable. `mkdir -p` is idempotent, so this safely
+  # no-ops on every subsequent `docker compose up`; it never touches the runner image's own entrypoint.
+  runner-tmp-init:
+    image: alpine:3.20
+    <<: *default-logging
+    profiles: ["runners"]
+    volumes:
+      - runner-work:/tmp/runner
+    command: ["sh", "-c", "mkdir -p /tmp/runner/tmp && chmod 1777 /tmp/runner/tmp"]
+
   runner:
     # Pin to a specific version tag — `latest` is mutable. Find a digest via:
     # docker pull myoung34/github-runner:ubuntu-jammy && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-jammy
     image: myoung34/github-runner:ubuntu-jammy
     restart: unless-stopped
     <<: *default-logging
     profiles: ["runners"]
+    depends_on:
+      runner-tmp-init:
+        condition: service_completed_successfully
     environment:
+      <<: *runner-tmp-env
       RUNNER_SCOPE: ${RUNNER_SCOPE:-repo}
       REPO_URL: ${RUNNER_REPO_URL:-}
       RUNNER_TOKEN: ${RUNNER_TOKEN:-}

test/unit/docs-selfhost-operations-runner-tmpdir.test.ts

@@ -0,0 +1,74 @@
+import { readFileSync } from "node:fs";
+import { parseDocument } from "yaml";
+import { describe, expect, it } from "vitest";
+
+// Drift guard (#selfhost-runner-tmp): the self-hosting operations doc's multi-runner override snippet
+// must stay in sync with docker-compose.yml's real x-runner-tmp-env anchor + runner-tmp-init pattern --
+// mirrors the same source-of-truth-diff approach as docs-selfhost-troubleshooting-metric-names.test.ts.
+// If the real compose pattern ever changes without the docs snippet changing too, this fails instead of
+// operators copy-pasting a stale/incorrect multi-runner example.
+
+const DOC_PATH = "apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx";
+
+function readYamlWithMerge(path: string): Record<string, unknown> {
+  const doc = parseDocument(readFileSync(path, "utf8"), { merge: true });
+  const value = doc.toJS();
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be a YAML object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+describe("self-hosting-operations doc: runner temp storage guidance matches docker-compose.yml (#selfhost-runner-tmp)", () => {
+  it("explains why runner temp must stay off overlay storage", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    expect(doc).toMatch(/runner-work/);
+    expect(doc).toMatch(/overlay/);
+    expect(doc).toContain("TMPDIR");
+  });
+
+  it("mentions the runner-tmp-init bootstrap mechanism by concept, not just by name", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    expect(doc).toContain("runner-tmp-init");
+    expect(doc).toMatch(/before the runner container\s+starts/);
+  });
+
+  it("says Docker cleanup should prune containers/images/build cache, never volumes", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    expect(doc).toMatch(/containers, images, and build cache/i);
+    expect(doc).toMatch(/never\s+volumes/i);
+  });
+
+  it("includes a multi-runner override snippet whose x-runner-tmp-env values match the real anchor in docker-compose.yml", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    const codeBlockMatch = /x-runner-tmp-env: &runner-tmp-env[\s\S]*?volumes:\n {2}runner-work-2:/.exec(doc);
+    expect(codeBlockMatch, "expected a multi-runner CodeBlock snippet in the doc").not.toBeNull();
+    const snippet = codeBlockMatch![0];
+
+    const compose = readYamlWithMerge("docker-compose.yml");
+    const realAnchor = compose["x-runner-tmp-env"] as Record<string, string>;
+    expect(realAnchor.TMPDIR).toBeTruthy();
+
+    for (const key of ["TMPDIR", "TMP", "TEMP"] as const) {
+      expect(snippet, `docs snippet missing ${key}: ${realAnchor[key]}`).toContain(`${key}: ${realAnchor[key]}`);
+    }
+  });
+
+  it("the doc's multi-runner snippet uses the same depends_on condition as the real runner service", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    const compose = readYamlWithMerge("docker-compose.yml");
+    const services = compose.services as Record<string, Record<string, unknown>>;
+    const runner = services.runner!;
+    const dependsOn = runner.depends_on as Record<string, { condition?: string }>;
+    const condition = Object.values(dependsOn)[0]?.condition;
+    expect(condition).toBeTruthy();
+    expect(doc).toContain(`condition: ${condition}`);
+  });
+
+  it("the doc's multi-runner snippet never mounts the Docker socket", () => {
+    const doc = readFileSync(DOC_PATH, "utf8");
+    const codeBlockMatch = /x-runner-tmp-env: &runner-tmp-env[\s\S]*?volumes:\n {2}runner-work-2:/.exec(doc);
+    expect(codeBlockMatch).not.toBeNull();
+    expect(codeBlockMatch![0]).not.toMatch(/docker\.sock/);
+  });
+});

test/unit/selfhost-compose-runner-tmpdir.test.ts

@@ -0,0 +1,129 @@
+import { readFileSync } from "node:fs";
+import { parseDocument } from "yaml";
+import { describe, expect, it } from "vitest";
+
+function readYamlWithMerge(path: string): Record<string, unknown> {
+  const doc = parseDocument(readFileSync(path, "utf8"), { merge: true });
+  const value = doc.toJS();
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${path} must be a YAML object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+type ComposeService = Record<string, unknown>;
+
+function composeServices(): Record<string, ComposeService> {
+  const compose = readYamlWithMerge("docker-compose.yml");
+  return (compose.services as Record<string, ComposeService>) ?? {};
+}
+
+function runnerService(): ComposeService {
+  const services = composeServices();
+  expect(services.runner, "docker-compose.yml must define a runner service").toBeTruthy();
+  return services.runner as ComposeService;
+}
+
+// Runner CI job temp writes must stay on the mounted runner-work volume, never the container's plain
+// /tmp (which lives in Docker's overlay/containerd snapshot storage and grows the HOST's Docker root
+// storage directly -- invisible to volume-scoped cleanup, confirmed in production as unbounded overlay
+// growth, disk pressure, and `docker system df` races against disappearing temp files). Pure structural
+// checks only (no `docker` CLI invocation): the self-hosted runner container this actually runs on does
+// not have Docker-in-Docker access, so a test that shells out to `docker compose config` would be
+// unreliable/environment-dependent here (same constraint as the other selfhost-compose-*.test.ts files).
+// `{ merge: true }` resolves `<<: *runner-tmp-env` the same way Docker Compose's own YAML 1.1 merge-key
+// support does -- verified once by hand against `docker compose config` with every profile active.
+describe("docker-compose.yml — runner temp storage stays off overlay (#selfhost-runner-tmp)", () => {
+  it("mounts the runner-work volume at /tmp/runner", () => {
+    const runner = runnerService();
+    const volumes = (runner.volumes as string[]) ?? [];
+    expect(volumes).toContain("runner-work:/tmp/runner");
+  });
+
+  it("keeps RUNNER_WORKDIR on the mounted volume", () => {
+    const runner = runnerService();
+    const env = runner.environment as Record<string, unknown>;
+    expect(env.RUNNER_WORKDIR).toBe("/tmp/runner");
+  });
+
+  it.each(["TMPDIR", "TMP", "TEMP"])("sets %s for the runner", (key) => {
+    const runner = runnerService();
+    const env = runner.environment as Record<string, unknown>;
+    expect(env[key], key).toBeDefined();
+  });
+
+  it.each(["TMPDIR", "TMP", "TEMP"])(
+    "points %s at mounted runner storage, not the container's plain /tmp",
+    (key) => {
+      const runner = runnerService();
+      const env = runner.environment as Record<string, unknown>;
+      const value = env[key];
+      expect(value, key).not.toBe("/tmp");
+      expect(typeof value, key).toBe("string");
+      expect(value as string, `${key}=${String(value)} must live under the mounted /tmp/runner volume`).toMatch(
+        /^\/tmp\/runner(\/|$)/,
+      );
+    },
+  );
+
+  it("never mounts the Docker socket into the runner service", () => {
+    const runner = runnerService();
+    const volumes = (runner.volumes as string[]) ?? [];
+    expect(volumes.some((v) => v.includes("docker.sock"))).toBe(false);
+  });
+
+  it("guarantees the configured TMPDIR exists before the runner starts, via a depends_on init service", () => {
+    const services = composeServices();
+    const runner = runnerService();
+    const dependsOn = runner.depends_on as Record<string, { condition?: string }> | undefined;
+    expect(dependsOn, "runner must declare depends_on for a bootstrap/init service").toBeTruthy();
+
+    const initServiceNames = Object.keys(dependsOn ?? {});
+    expect(initServiceNames.length, "runner must depend on exactly the init service").toBeGreaterThan(0);
+    const initServiceName = initServiceNames[0]!;
+    expect(dependsOn?.[initServiceName]?.condition, "the dependency must wait for successful completion").toBe(
+      "service_completed_successfully",
+    );
+
+    const initService = services[initServiceName];
+    expect(initService, `the depends_on target "${initServiceName}" must exist as a service`).toBeTruthy();
+    const initVolumes = (initService?.volumes as string[]) ?? [];
+    expect(
+      initVolumes.some((v) => v.startsWith("runner-work:")),
+      "the init service must mount the same runner-work volume it bootstraps",
+    ).toBe(true);
+    expect(JSON.stringify(initService?.command)).toMatch(/mkdir -p/);
+  });
+
+  it("does not introduce volume-deleting behavior in the runner temp bootstrap step", () => {
+    const services = composeServices();
+    const runner = runnerService();
+    const dependsOn = runner.depends_on as Record<string, unknown>;
+    const initServiceName = Object.keys(dependsOn)[0]!;
+    const initService = services[initServiceName];
+    const command = JSON.stringify(initService?.command ?? "");
+    expect(command).not.toMatch(/\brm\b|\bprune\b|\bdown\b\s+-v/);
+  });
+
+  it("keeps the runner service gated behind the runners profile (opt-in, not part of the default stack)", () => {
+    const runner = runnerService();
+    expect(runner.profiles).toEqual(["runners"]);
+  });
+
+  it("exposes a reusable x-runner-tmp-env anchor that the runner service actually merges in", () => {
+    const compose = readYamlWithMerge("docker-compose.yml");
+    const anchorBlock = compose["x-runner-tmp-env"] as Record<string, unknown>;
+    expect(anchorBlock, "x-runner-tmp-env extension field must exist for multi-runner reuse").toBeTruthy();
+    for (const key of ["TMPDIR", "TMP", "TEMP"]) {
+      expect(anchorBlock[key], key).toBe("/tmp/runner/tmp");
+    }
+
+    // The runner service's resolved env must equal the anchor's values (not a separately hand-written
+    // duplicate that could drift) -- `{ merge: true }` above already resolved `<<: *runner-tmp-env`.
+    const runner = runnerService();
+    const env = runner.environment as Record<string, unknown>;
+    for (const key of ["TMPDIR", "TMP", "TEMP"]) {
+      expect(env[key], key).toBe(anchorBlock[key]);
+    }
+  });
+});