feat(diag): persistent memory trajectory + query-leak CI soak (#581 observability)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -103,6 +103,21 @@ body:
         Please check for private paths/strings before pasting.
       render: text
 
+  - type: textarea
+    id: diagnostics
+    attributes:
+      label: Diagnostics trajectory (memory / performance / leak issues)
+      description: |
+        We collect **no telemetry**, so for a memory or performance problem we
+        need the diagnostics trajectory from your machine. Set
+        `CBM_DIAGNOSTICS=1`, reproduce the issue, then attach (or paste) the file
+        `cbm-diagnostics-<pid>.ndjson` from your temp dir (`$TMPDIR` / `/tmp`, or
+        `%TEMP%` on Windows). It records only resource counters (rss, committed,
+        fds, query counts) over time — no code or queries. See the README
+        "Troubleshooting & Diagnostics" section. Pasting an agent's summary of
+        the trajectory is also fine.
+      render: text
+
   - type: input
     id: scale
     attributes:

.github/workflows/_soak.yml

@@ -62,13 +62,27 @@ jobs:
         run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
       - name: Soak (${{ inputs.duration_minutes }} min)
         run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
+      # #581 guard: read-only soak (never reindex/mutate) so any memory growth is
+      # a query-path leak, not WAL/indexing. Reuses the build above.
+      - name: Query-leak soak (#581, read-only)
+        env:
+          CBM_SOAK_MODE: query-leak
+          RESULTS_DIR: soak-results-query-leak
+        run: scripts/soak-test.sh build/c/codebase-memory-mcp ${{ inputs.duration_minutes }} --skip-crash-test
       - name: Upload metrics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }}
           path: soak-results/
           retention-days: 14
+      - name: Upload query-leak metrics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: soak-query-leak-${{ matrix.goos }}-${{ matrix.goarch }}
+          path: soak-results-query-leak/
+          retention-days: 14
 
   soak-quick-windows:
     runs-on: windows-latest
@@ -98,13 +112,30 @@ jobs:
           BIN=build/c/codebase-memory-mcp
           [ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
           scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }}
+      # #581 guard on Windows — the platform the bug is reported on.
+      - name: Query-leak soak (#581, read-only)
+        shell: msys2 {0}
+        env:
+          CBM_SOAK_MODE: query-leak
+          RESULTS_DIR: soak-results-query-leak
+        run: |
+          BIN=build/c/codebase-memory-mcp
+          [ -f "${BIN}.exe" ] && BIN="${BIN}.exe"
+          scripts/soak-test.sh "$BIN" ${{ inputs.duration_minutes }} --skip-crash-test
       - name: Upload metrics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: soak-quick-windows-amd64
           path: soak-results/
           retention-days: 14
+      - name: Upload query-leak metrics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: soak-query-leak-windows-amd64
+          path: soak-results-query-leak/
+          retention-days: 14
 
   soak-asan:
     if: ${{ inputs.run_asan }}

README.md

@@ -231,6 +231,31 @@ Benchmarked on Apple M3 Pro:
 
 **Token efficiency**: Five structural queries consumed ~3,400 tokens via codebase-memory-mcp versus ~412,000 tokens via file-by-file grep exploration — a **99.2% reduction**.
 
+## Troubleshooting & Diagnostics
+
+codebase-memory-mcp runs **100% locally and collects no telemetry** — your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself.
+
+### Capture a diagnostics log
+
+Set `CBM_DIAGNOSTICS=1` before the MCP server starts, then reproduce the problem (let it run as long as it takes — a slow leak needs time to show in the trend). The server writes two files to your system temp directory (`$TMPDIR` or `/tmp` on macOS/Linux, `%TEMP%` on Windows):
+
+| File | What it is |
+|------|------------|
+| `cbm-diagnostics-<pid>.ndjson` | **The memory trajectory** — one JSON line every 5 s with `rss`, `committed` (Windows commit charge), `peak_*`, `page_faults`, `fd`, and `queries`. **This is the file we need for memory/leak reports** — the *trend over time* is what pinpoints a leak. It is **kept on disk after the server exits** (so you can grab it post-mortem) and rotates to `.ndjson.1` past ~8 MB. |
+| `cbm-diagnostics-<pid>.json` | The latest snapshot only — handy for a quick live check. Removed on clean exit. |
+
+The startup log prints both paths, e.g.:
+
+```
+level=info msg=diagnostics.start snapshot=/tmp/cbm-diagnostics-12345.json trajectory=/tmp/cbm-diagnostics-12345.ndjson interval=5s
+```
+
+Set the variable in the `env` block of your agent's MCP server config, or export it before launching the server.
+
+### What to share
+
+When you open a memory/performance issue, **attach the `.ndjson` trajectory** — it contains no source code or query text, only resource counters. If you'd rather not attach a file, paste it (or an agent's summary of it) into the issue: your assistant can read the NDJSON directly and report whether `rss`/`committed` grow monotonically, how fast, and relative to query count — which is exactly what we need to find the cause.
+
 ## Installation
 
 ### Pre-built Binaries

scripts/soak-test.sh

@@ -34,7 +34,7 @@ BINARY=$(cd "$(dirname "$BINARY")" && pwd)/$(basename "$BINARY")
 #                 skipped in this mode because it reindexes (which would mask #581).
 CBM_SOAK_MODE="${CBM_SOAK_MODE:-default}"
 
-RESULTS_DIR="soak-results"
+RESULTS_DIR="${RESULTS_DIR:-soak-results}"
 mkdir -p "$RESULTS_DIR"
 
 METRICS_CSV="$RESULTS_DIR/metrics.csv"

src/foundation/diagnostics.c

@@ -35,6 +35,12 @@ static cbm_thread_t g_diag_thread;
 static bool g_diag_started = false;
 static time_t g_start_time = 0;
 static char g_diag_path[CBM_SZ_256] = "";
+/* Persistent NDJSON time-series (the memory TRAJECTORY users send us to diagnose
+ * slow leaks like #581). Unlike g_diag_path (a latest-snapshot file overwritten
+ * every interval and deleted on stop), this is appended-to and KEPT on exit. */
+static char g_diag_ndjson_path[CBM_SZ_256] = "";
+static size_t g_diag_ndjson_size = 0;
+#define DIAG_NDJSON_CAP_BYTES (8u * 1024u * 1024u) /* rotate to .1 past this */
 
 /* ── Query stats ─────────────────────────────────────────────────── */
 
@@ -89,6 +95,35 @@ static int count_open_fds(void) {
 #define DIAG_INTERVAL_S 5
 #define DIAG_PATH_EXTRA 24 /* ".tmp" + safety margin */
 
+/* Append one compact JSON line to the persistent NDJSON trajectory, rotating to
+ * <path>.1 once it passes the cap. Best-effort: a failed append never disrupts
+ * the server. The trajectory (a monotonic rss/committed climb over hours) is
+ * what reveals a slow leak like #581 — the single latest-snapshot file cannot. */
+static void append_trajectory(long uptime, size_t rss, size_t peak_rss, size_t commit,
+                              size_t peak_commit, size_t page_faults, int fds, int qcount) {
+    if (g_diag_ndjson_path[0] == '\0') {
+        return;
+    }
+    if (g_diag_ndjson_size > DIAG_NDJSON_CAP_BYTES) {
+        char rot[sizeof(g_diag_ndjson_path) + DIAG_PATH_EXTRA];
+        snprintf(rot, sizeof(rot), "%s.1", g_diag_ndjson_path);
+        (void)rename(g_diag_ndjson_path, rot); /* keep one previous generation */
+        g_diag_ndjson_size = 0;
+    }
+    FILE *f = fopen(g_diag_ndjson_path, "a");
+    if (!f) {
+        return;
+    }
+    int n = fprintf(f,
+                    "{\"uptime_s\":%ld,\"rss\":%zu,\"peak_rss\":%zu,\"committed\":%zu,"
+                    "\"peak_committed\":%zu,\"page_faults\":%zu,\"fd\":%d,\"queries\":%d}\n",
+                    uptime, rss, peak_rss, commit, peak_commit, page_faults, fds, qcount);
+    if (n > 0) {
+        g_diag_ndjson_size += (size_t)n;
+    }
+    (void)fclose(f);
+}
+
 static void write_diagnostics(void) {
     /* Collect mimalloc stats */
     size_t elapsed_ms = 0;
@@ -151,6 +186,10 @@ static void write_diagnostics(void) {
         return;
     }
     (void)rename(tmp_path, g_diag_path);
+
+    /* Also append to the persistent trajectory (kept on exit for users to send). */
+    append_trajectory(uptime, current_rss, peak_rss, current_commit, peak_commit, page_faults, fds,
+                      qcount);
 }
 
 static void *diag_thread_fn(void *arg) {
@@ -179,14 +218,18 @@ bool cbm_diag_start(void) {
 
     snprintf(g_diag_path, sizeof(g_diag_path), "%s/cbm-diagnostics-%d.json", cbm_tmpdir(),
              (int)getpid());
+    snprintf(g_diag_ndjson_path, sizeof(g_diag_ndjson_path), "%s/cbm-diagnostics-%d.ndjson",
+             cbm_tmpdir(), (int)getpid());
+    g_diag_ndjson_size = 0;
 
     if (cbm_thread_create(&g_diag_thread, 0, diag_thread_fn, NULL) != 0) {
         return false;
     }
 
     g_diag_started = true;
-    (void)fprintf(stderr, "level=info msg=diagnostics.start path=%s interval=%ds\n", g_diag_path,
-                  DIAG_INTERVAL_S);
+    (void)fprintf(stderr,
+                  "level=info msg=diagnostics.start snapshot=%s trajectory=%s interval=%ds\n",
+                  g_diag_path, g_diag_ndjson_path, DIAG_INTERVAL_S);
     return true;
 }
 
@@ -198,9 +241,14 @@ void cbm_diag_stop(void) {
     cbm_thread_join(&g_diag_thread);
     g_diag_started = false;
 
-    /* Clean up file */
+    /* Remove the live latest-snapshot file (and its .tmp), but KEEP the
+     * trajectory NDJSON so the user can send it after the server exits. */
     cbm_unlink(g_diag_path);
     char tmp_path[sizeof(g_diag_path) + DIAG_PATH_EXTRA];
     snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", g_diag_path);
     cbm_unlink(tmp_path);
+    if (g_diag_ndjson_path[0] != '\0') {
+        (void)fprintf(stderr, "level=info msg=diagnostics.trajectory_kept path=%s\n",
+                      g_diag_ndjson_path);
+    }
 }