feat(mcps): probe MCP servers for real, and tell the agent what is dead

resources/hooks/mcp-health-check.json

@@ -0,0 +1,16 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash ${CLAUDE_CONFIG_DIR}/hooks/mcp-health-check.sh",
+            "description": "Once-a-day MCP liveness probe for the active profile: spawns each server and runs the real MCP handshake, then names any that are down — plus the skills that declare a dependency on them — so the agent knows a capability is broken rather than concluding it does not exist. Fail-open; throttled by ~/.config/cue/mcp-health-stamp; disable with CUE_MCP_HEALTH_OFF=1.",
+            "id": "cue:sessionstart:mcp-health-check"
+          }
+        ]
+      }
+    ]
+  }
+}

resources/hooks/mcp-health-check.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# SessionStart hook — tell the agent which MCP servers in the active profile
+# are actually dead.
+#
+# A broken MCP is invisible: the server fails to start, its tools never appear,
+# and the agent simply concludes the capability does not exist — then works
+# around it or tells the user it is unavailable. secret-mcp sat dead in eight
+# profiles for weeks that way, and `cue mcps health` reported it green the whole
+# time because the old check only ran `which` on the wrapper command.
+#
+# So: run the real handshake probe once a day, and if anything is down, say so
+# in context where the agent can act on it — including which skills declare a
+# dependency on the dead server, and the exact command to drop it.
+#
+# Always safe: emits nothing and exits 0 when jq or cue are missing, the probe
+# fails, or it already ran today.
+#
+# Tunables:
+#   CUE_MCP_HEALTH_OFF=1   disable entirely
+#   Stamp: ~/.config/cue/mcp-health-stamp (date of last run; delete to re-run)
+
+set -uo pipefail
+
+[ "${CUE_MCP_HEALTH_OFF:-}" = "1" ] && exit 0
+command -v jq >/dev/null 2>&1 || exit 0
+command -v cue >/dev/null 2>&1 || exit 0
+
+# Once per day. The probe spawns every server in the profile, so it is far too
+# expensive to run on every session.
+stamp_dir="${XDG_CONFIG_HOME:-$HOME/.config}/cue"
+stamp="$stamp_dir/mcp-health-stamp"
+today="$(date +%F)" || exit 0
+[ -n "$today" ] || exit 0
+[ -f "$stamp" ] && [ "$(cat "$stamp" 2>/dev/null)" = "$today" ] && exit 0
+
+# Exits 1 when something is down, which is the interesting case — so ignore the
+# status and read the payload.
+health="$(timeout 90 cue mcps health --json 2>/dev/null)" || true
+[ -n "$health" ] || exit 0
+echo "$health" | jq -e 'type == "array"' >/dev/null 2>&1 || exit 0
+
+mkdir -p "$stamp_dir" 2>/dev/null && printf '%s' "$today" > "$stamp" 2>/dev/null
+
+dead="$(echo "$health" | jq -r '[.[] | select(.status == "down")]')"
+count="$(echo "$dead" | jq -r 'length')"
+[ "${count:-0}" -gt 0 ] 2>/dev/null || exit 0
+
+# Skills that declare a dependency on a dead server are dead too, in the sense
+# that their documented flow cannot run. Naming them saves the agent from
+# discovering it mid-task.
+catalog=""
+for candidate in \
+  "$HOME/Documents/cue/resources/skills/catalog/index.json" \
+  "${CUE_HOME:-}/resources/skills/catalog/index.json"; do
+  [ -f "$candidate" ] && { catalog="$candidate"; break; }
+done
+
+printf '⚠️  MCP health: %s server(s) in this profile are not running.\n\n' "$count"
+
+echo "$dead" | jq -r '.[] | "   ✗ \(.id) — \(.reason // "no response")"'
+
+if [ -n "$catalog" ]; then
+  affected="$(
+    echo "$dead" | jq -r '.[].id' | while read -r id; do
+      [ -n "$id" ] || continue
+      # The parens around the index() are load-bearing: `|` binds looser than
+      # `and`, so without them jq pipes a boolean into index() and dies.
+      jq -r --arg id "$id" '
+        (if type == "object" and has("skills") then .skills else . end)
+        | (if type == "object" then [.[]] else . end)
+        | map(select(type == "object" and ((.requires.mcps // []) | index($id))))
+        | .[] | "   ↳ skill \(.id // .name) needs \($id)"
+      ' "$catalog" 2>/dev/null
+    done
+  )"
+  [ -n "$affected" ] && { printf '\n'; printf '%s\n' "$affected"; }
+fi
+
+cat <<'EOF'
+
+These tools are unavailable this session — do not report their capability as
+missing without saying why. Offer the user the removal, do not run it unasked:
+  cue mcps remove <id>
+EOF
+
+exit 0

src/commands/mcps.ts

@@ -9,11 +9,13 @@
  *   health [--json]     — ping each MCP in active profile
  */
 
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
 
+import { findMissingExecutable, probeServer } from "../lib/mcp-probe";
+import type { McpServerConfig, ProbeResult } from "../lib/mcp-probe";
 import { loadProfile } from "../lib/profile-loader";
 import { resolveActiveProfile } from "../lib/cwd-resolver";
 import { repoRoot } from "../lib/repo-root";
@@ -162,57 +164,66 @@ async function cmdRemove(id: string): Promise<number> {
   return 0;
 }
 
-async function cmdHealth(json: boolean): Promise<number> {
+async function cmdHealth(json: boolean, shallow: boolean): Promise<number> {
   const ids = await getActiveProfileMcpIds();
-  const results: { id: string; status: "up" | "down" | "unknown"; latency_ms?: number }[] = [];
-
-  for (const id of ids) {
-    const start = Date.now();
-    // Try to check if the MCP process/command exists
-    const allConfigs = loadMcpConfig(id);
-    if (!allConfigs) {
-      results.push({ id, status: "unknown" });
-      continue;
-    }
-
-    const cmd = allConfigs.command as string | undefined;
-    if (!cmd) {
-      results.push({ id, status: "unknown" });
-      continue;
-    }
 
-    // For stdio MCPs, check if the command binary exists
-    const expandedCmd = cmd.replace(/^~/, process.env.HOME ?? "~");
-    const check = spawnSync("which", [expandedCmd.split("/").pop() ?? cmd], {
-      encoding: "utf8",
-      timeout: 2000,
-    });
-    const latency = Date.now() - start;
-
-    if (check.status === 0) {
-      results.push({ id, status: "up", latency_ms: latency });
-    } else {
-      // Try the full path
-      const { existsSync } = await import("node:fs");
-      if (existsSync(expandedCmd)) {
-        results.push({ id, status: "up", latency_ms: latency });
-      } else {
-        results.push({ id, status: "down", latency_ms: latency });
-      }
-    }
-  }
+  // Probe concurrently — each one spends most of its time waiting on a child
+  // process, and a profile with a dozen servers would otherwise take a minute.
+  const results: ProbeResult[] = await Promise.all(
+    ids.map((id) => {
+      const config = loadMcpConfig(id) as McpServerConfig | null;
+      if (shallow) return Promise.resolve(shallowCheck(id, config));
+      return probeServer(id, config);
+    }),
+  );
 
   if (json) {
     process.stdout.write(JSON.stringify(results, null, 2) + "\n");
-  } else {
-    process.stdout.write(`MCP Health Check (${results.length} servers):\n\n`);
-    for (const r of results) {
-      const icon = r.status === "up" ? "✅" : r.status === "down" ? "❌" : "❓";
-      const lat = r.latency_ms !== undefined ? ` (${r.latency_ms}ms)` : "";
-      process.stdout.write(`  ${icon} ${r.id}${lat}\n`);
-    }
+    return results.some(r => r.status === "down") ? 1 : 0;
   }
-  return 0;
+
+  const mode = shallow ? " — shallow" : "";
+  process.stdout.write(`MCP Health Check (${results.length} servers${mode}):\n\n`);
+  for (const r of results) {
+    const icon = r.status === "up" ? "✅" : r.status === "down" ? "❌" : "❓";
+    const lat = r.latency_ms !== undefined ? ` (${r.latency_ms}ms` : "";
+    const tools = r.tools !== undefined ? `, ${r.tools} tools)` : lat ? ")" : "";
+    const why = r.reason ? `\n      ${r.reason}` : "";
+    process.stdout.write(`  ${icon} ${r.id}${lat}${tools}${why}\n`);
+  }
+
+  const dead = results.filter(r => r.status === "down");
+  if (dead.length > 0) {
+    process.stdout.write(
+      `\n${dead.length} server(s) down. Remove one with:  cue mcps remove <id>\n`,
+    );
+  }
+  return dead.length > 0 ? 1 : 0;
+}
+
+/**
+ * The pre-existing check, kept behind `--shallow` for when spawning every
+ * server is too slow (a hot loop, CI). It only answers "does an executable by
+ * this name exist", so it cannot see a server that starts and then dies —
+ * hence `findMissingExecutable`, which at least catches a broken interpreter
+ * hiding inside a wrapper's arguments.
+ */
+function shallowCheck(id: string, config: McpServerConfig | null): ProbeResult {
+  if (!config?.command) return { id, status: "unknown", reason: "no command in MCP config" };
+
+  const missing = findMissingExecutable(config);
+  if (missing) return { id, status: "down", reason: `missing executable: ${missing}` };
+
+  const expanded = config.command.replace(/^~/, process.env.HOME ?? "~");
+  if (existsSync(expanded)) return { id, status: "up" };
+
+  const found = spawnSync("which", [expanded.split("/").pop() ?? expanded], {
+    encoding: "utf8",
+    timeout: 2000,
+  });
+  return found.status === 0
+    ? { id, status: "up" }
+    : { id, status: "down", reason: `not on PATH: ${config.command}` };
 }
 
 function loadMcpConfig(id: string): Record<string, unknown> | null {
@@ -240,18 +251,26 @@ Subcommands:
   available         MCPs NOT in active profile
   add <id>          Add MCP to active profile
   remove <id>       Remove MCP from active profile
-  health            Ping each MCP, show status
+  health            Spawn each MCP and run the MCP handshake; show status
+
+Flags:
+  --json            Machine-readable output
+  --shallow         Only check that the executable exists (fast, less certain)
+
+Exit code is 1 when any server is down, so CI and hooks can gate on it.
 
 Examples:
   cue mcps add coolify
   cue mcps health
+  cue mcps health --json
 `);
     return 0;
   }
 
   const sub = args[0] ?? "list";
   const json = args.includes("--json");
-  const rest = args.filter(a => a !== "--json");
+  const shallow = args.includes("--shallow");
+  const rest = args.filter(a => a !== "--json" && a !== "--shallow");
 
   switch (sub) {
     case "list":
@@ -263,7 +282,7 @@ Examples:
     case "remove":
       return cmdRemove(rest[1] ?? "");
     case "health":
-      return cmdHealth(json);
+      return cmdHealth(json, shallow);
     default:
       process.stderr.write(`Unknown subcommand: ${sub}. Use: list, available, add, remove, health\n`);
       return 1;

src/lib/mcp-probe.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, symlinkSync, writeFileSync, chmodSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { findMissingExecutable, probeServer } from "./mcp-probe";
+
+const tmp = mkdtempSync(join(tmpdir(), "mcp-probe-"));
+
+describe("findMissingExecutable", () => {
+  test("catches a dangling symlink hidden inside a bash -lc wrapper", () => {
+    // This is the secret-mcp shape verbatim: the launcher is `bash`, and the
+    // interpreter that actually matters is a symlink to a uv-managed Python
+    // that was uninstalled. `which bash` passes, so the old check said "up".
+    const dangling = join(tmp, "venv", "bin", "python");
+    symlinkSync("/nonexistent/cpython-3.14/bin/python3.14", ensureDir(dangling));
+
+    expect(
+      findMissingExecutable({
+        command: "bash",
+        args: ["-lc", `cd /srv && export FOO=1; exec ${dangling} -m secret_mcp`],
+      }),
+    ).toBe(dangling);
+  });
+
+  test("passes a wrapper whose interpreter exists", () => {
+    const real = join(tmp, "real", "bin", "python");
+    writeFileSync(ensureDir(real), "#!/bin/sh\n");
+    chmodSync(real, 0o755);
+
+    expect(
+      findMissingExecutable({ command: "bash", args: ["-lc", `exec ${real} -m thing`] }),
+    ).toBeNull();
+  });
+
+  test("reports an absolute command that does not exist", () => {
+    expect(findMissingExecutable({ command: "/opt/gone/server", args: [] })).toBe(
+      "/opt/gone/server",
+    );
+  });
+
+  test("leaves PATH-resolved commands to the spawn", () => {
+    // Not our job to second-guess PATH — only absolute paths are checked.
+    expect(findMissingExecutable({ command: "npx", args: ["-y", "some-mcp"] })).toBeNull();
+  });
+});
+
+describe("probeServer", () => {
+  test("reports up with a tool count when the server completes the handshake", async () => {
+    const server = join(tmp, "fake-mcp.mjs");
+    writeFileSync(
+      server,
+      `
+let buf = "";
+process.stdin.on("data", (c) => {
+  buf += c;
+  const lines = buf.split("\\n"); buf = lines.pop();
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    const msg = JSON.parse(line);
+    if (msg.method === "initialize") {
+      process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {} }) + "\\n");
+    } else if (msg.method === "tools/list") {
+      process.stdout.write(JSON.stringify({
+        jsonrpc: "2.0", id: msg.id, result: { tools: [{ name: "a" }, { name: "b" }] },
+      }) + "\\n");
+    }
+  }
+});
+`,
+    );
+
+    const result = await probeServer("fake", { command: process.execPath, args: [server] }, 5000);
+    expect(result.status).toBe("up");
+    expect(result.tools).toBe(2);
+  });
+
+  test("reports down when the process dies immediately", async () => {
+    const result = await probeServer(
+      "dead",
+      { command: process.execPath, args: ["-e", "process.exit(1)"] },
+      5000,
+    );
+    expect(result.status).toBe("down");
+    expect(result.reason).toContain("exited with code 1");
+  });
+
+  test("reports down, naming the missing interpreter, rather than a bare timeout", async () => {
+    const gone = join(tmp, "also-gone", "bin", "python");
+    symlinkSync("/nonexistent/python3.14", ensureDir(gone));
+
+    const result = await probeServer(
+      "broken",
+      { command: "bash", args: ["-lc", `exec ${gone} -m x`] },
+      5000,
+    );
+    expect(result.status).toBe("down");
+    expect(result.reason).toContain(gone);
+  });
+
+  test("reports unknown when the MCP has no command", async () => {
+    const result = await probeServer("nocmd", { args: [] }, 1000);
+    expect(result.status).toBe("unknown");
+  });
+});
+
+/** mkdir -p the parent of `file`, then hand `file` back. */
+function ensureDir(file: string): string {
+  require("node:fs").mkdirSync(join(file, ".."), { recursive: true });
+  return file;
+}