Bring back MCP endpoint, scoped to the two /ask tools

README.md

@@ -81,6 +81,14 @@ npm run generate-embeddings   # Generate vector embeddings
 
 Automated jobs run daily to keep content updated.
 
+## MCP
+
+The backend also exposes a [Model Context Protocol](https://modelcontextprotocol.io)
+endpoint at `/mcp` with two tools (`search_all`, `get_full_document`) — the
+same tools the `/ask` assistant uses. Point any MCP-compatible client at
+`https://search-api.fluffylabs.dev/mcp` to query the index. See
+[`backend/README.md`](./backend/README.md#mcp) for details.
+
 ## Deployment
 
 - **Backend**: Deployed to <https://search-api.fluffylabs.dev>

backend/README.md

@@ -72,3 +72,95 @@ The application includes scheduled cron jobs that run daily to:
 - `GET /search/graypaper` - Search graypaper sections
 - `GET /search/discords` - Search Discord messages
 - `GET /embeddings` - Get embeddings for a query
+- `POST /ask` - Streaming AI assistant (SSE). See [Ask API](#ask-api).
+- `POST|GET|DELETE /mcp` - Model Context Protocol endpoint. See [MCP](#mcp).
+
+## Ask API
+
+`POST /ask` runs an agent loop that answers questions using the JAM knowledge
+base. Responses stream back as Server-Sent Events.
+
+Request body:
+
+```json
+{
+  "messages": [{ "role": "user", "content": "What is a refinement context?" }],
+  "model": "openai/gpt-4o-mini",
+  "openrouterKey": "sk-or-..."
+}
+```
+
+The agent has access to two tools — `search_all` (unified search across all
+sources) and `get_full_document` (fetch full markdown by id). These are the
+same tools exposed via [MCP](#mcp).
+
+## MCP
+
+The backend serves a [Model Context Protocol](https://modelcontextprotocol.io)
+endpoint at `/mcp` using the Streamable HTTP transport. It exposes the same
+two tools the `/ask` agent uses:
+
+- `search_all(query, limit?)` — unified search across graypaper, discord,
+  matrix and pages. Returns an array of result chunks, each with a stable `id`,
+  `sourceType`, and content preview.
+- `get_full_document(id)` — fetch the full markdown of a document by id
+  returned from `search_all`.
+
+### Design choices
+
+- **Stateless.** A fresh `Server` + `Transport` is created per request. No
+  session state is kept between calls; `initialize` does not return an
+  `mcp-session-id` header and there is no `GET` / `DELETE` lifecycle. Any
+  MCP client that supports stateless Streamable HTTP works.
+- **No CORS.** `/mcp` intentionally sets no `Access-Control-Allow-Origin`
+  header. It is meant for server-to-server / local MCP clients, not browser
+  origins. The other endpoints still apply CORS for the frontend.
+- **No embeddings.** Tool calls run fulltext-only search; the server's
+  OpenAI quota is never spent on anonymous MCP traffic. `/ask` still uses
+  hybrid search because the caller supplies their own OpenRouter key.
+
+### Connecting a client
+
+Point any MCP client at the public deployment or your local dev server:
+
+- Production: `https://search-api.fluffylabs.dev/mcp`
+- Local dev: `http://localhost:3000/mcp`
+
+Example Claude Desktop (`claude_desktop_config.json`) entry:
+
+```json
+{
+  "mcpServers": {
+    "jam-search": {
+      "url": "https://search-api.fluffylabs.dev/mcp"
+    }
+  }
+}
+```
+
+### Manual probe
+
+Stateless mode means a single POST per interaction — no session bookkeeping.
+
+```bash
+# Initialize and read the server's advertised capabilities:
+curl -s -X POST http://localhost:3000/mcp \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json, text/event-stream' \
+  -d '{"jsonrpc":"2.0","id":1,"method":"initialize",
+       "params":{"protocolVersion":"2025-03-26","capabilities":{},
+                 "clientInfo":{"name":"probe","version":"0.0.1"}}}'
+
+# List the two tools:
+curl -s -X POST http://localhost:3000/mcp \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json, text/event-stream' \
+  -d '{"jsonrpc":"2.0","id":2,"method":"tools/list"}'
+
+# Call search_all:
+curl -s -X POST http://localhost:3000/mcp \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: application/json, text/event-stream' \
+  -d '{"jsonrpc":"2.0","id":3,"method":"tools/call",
+       "params":{"name":"search_all","arguments":{"query":"refine"}}}'
+```

backend/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.19.13",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "@octokit/rest": "^22.0.1",
     "@orama/orama": "^3.1.18",
     "cheerio": "^1.2.0",

backend/src/__tests__/ask/tools.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from "vitest";
-import { executeGetFullDocument, executeSearchAll } from "../../ask/tools.js";
+import {
+  executeGetFullDocument,
+  executeSearchAll,
+  TOOL_DEFINITIONS,
+  TOOL_SPECS,
+} from "../../ask/tools.js";
 import { createSearchDB, insertDoc } from "../../data/searchIndex.js";
 
 describe("executeSearchAll", () => {
@@ -78,3 +83,23 @@ describe("executeGetFullDocument", () => {
     expect(result).toBeNull();
   });
 });
+
+describe("tool specs", () => {
+  it("TOOL_DEFINITIONS is derived 1:1 from TOOL_SPECS (no duplication)", () => {
+    expect(TOOL_DEFINITIONS.map((d) => d.function.name)).toEqual(
+      TOOL_SPECS.map((s) => s.name)
+    );
+    for (let i = 0; i < TOOL_SPECS.length; i++) {
+      expect(TOOL_DEFINITIONS[i].function.description).toBe(
+        TOOL_SPECS[i].description
+      );
+    }
+  });
+
+  it("search_all parameters do not require `limit`", () => {
+    const spec = TOOL_DEFINITIONS.find((d) => d.function.name === "search_all");
+    expect(spec).toBeDefined();
+    const params = spec?.function.parameters as { required?: string[] };
+    expect(params.required).toEqual(["query"]);
+  });
+});

backend/src/__tests__/mcp/handler.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it } from "vitest";
+import { createApp } from "../../api.js";
+import { createSearchDB, insertDoc } from "../../data/searchIndex.js";
+
+function initBody() {
+  return {
+    jsonrpc: "2.0",
+    id: 1,
+    method: "initialize",
+    params: {
+      protocolVersion: "2025-03-26",
+      capabilities: {},
+      clientInfo: { name: "handler-test", version: "0.0.1" },
+    },
+  };
+}
+
+function mcpHeaders() {
+  return {
+    "Content-Type": "application/json",
+    Accept: "application/json, text/event-stream",
+  };
+}
+
+describe("/mcp HTTP handler", () => {
+  it("handles initialize stateless (no CORS, no session id) and returns SSE body", async () => {
+    const db = createSearchDB();
+    insertDoc(db, {
+      type: "graypaper_section",
+      title: "Accumulate",
+      content: "The accumulate function processes work results.",
+    });
+    const app = createApp(db, "./data");
+
+    const res = await app.fetch(
+      new Request("http://x/mcp", {
+        method: "POST",
+        headers: mcpHeaders(),
+        body: JSON.stringify(initBody()),
+      })
+    );
+
+    expect(res.status).toBe(200);
+    // CORS must not be advertised for server-to-server endpoint.
+    expect(res.headers.get("access-control-allow-origin")).toBeNull();
+    // Stateless: transport must not mint a session id.
+    expect(res.headers.get("mcp-session-id")).toBeNull();
+
+    const text = await res.text();
+    expect(text).toContain("protocolVersion");
+    expect(text).toContain("jam-search");
+  });
+
+  it("rejects initialize when Accept header is missing text/event-stream", async () => {
+    const app = createApp(createSearchDB(), "./data");
+    const res = await app.fetch(
+      new Request("http://x/mcp", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(initBody()),
+      })
+    );
+    // MCP transport rejects with 406 when the client didn't advertise SSE.
+    expect(res.status).toBe(406);
+  });
+
+  it("applies CORS to non-MCP routes (sanity check)", async () => {
+    const app = createApp(createSearchDB(), "./data");
+    const res = await app.fetch(
+      new Request("http://x/health", {
+        headers: { Origin: "https://example.com" },
+      })
+    );
+    expect(res.status).toBe(200);
+    expect(res.headers.get("access-control-allow-origin")).not.toBeNull();
+  });
+});

backend/src/__tests__/mcp/server.test.ts

@@ -0,0 +1,104 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { afterEach, describe, expect, it } from "vitest";
+import { createSearchDB, insertDoc } from "../../data/searchIndex.js";
+import { createMcpServer } from "../../mcp/server.js";
+
+describe("mcp server", () => {
+  const opened: Array<() => Promise<void>> = [];
+
+  afterEach(async () => {
+    await Promise.all(opened.map((close) => close()));
+    opened.length = 0;
+  });
+
+  async function connectClient(db = createSearchDB()) {
+    const server = createMcpServer(db, "./data");
+    const [clientTransport, serverTransport] =
+      InMemoryTransport.createLinkedPair();
+    const client = new Client({ name: "test", version: "0.0.0" });
+    await Promise.all([
+      server.connect(serverTransport),
+      client.connect(clientTransport),
+    ]);
+    opened.push(async () => {
+      // close() is idempotent in the MCP SDK; closing both sides cleans up
+      // protocol state, transport handlers, and linked transport pair.
+      await Promise.all([client.close(), server.close()]);
+    });
+    return { client, db };
+  }
+
+  it("lists exactly the two /ask tools", async () => {
+    const { client } = await connectClient();
+    const { tools } = await client.listTools();
+    const names = tools.map((t) => t.name).sort();
+    expect(names).toEqual(["get_full_document", "search_all"]);
+  });
+
+  it("does not mark `limit` as required on search_all", async () => {
+    const { client } = await connectClient();
+    const { tools } = await client.listTools();
+    const searchAll = tools.find((t) => t.name === "search_all");
+    expect(searchAll).toBeDefined();
+    const required = (searchAll?.inputSchema as { required?: string[] })
+      .required;
+    expect(required).toEqual(["query"]);
+  });
+
+  it("calls search_all and returns a text content block", async () => {
+    const db = createSearchDB();
+    insertDoc(db, {
+      type: "graypaper_section",
+      title: "Accumulate",
+      content: "The accumulate function processes work results.",
+    });
+    const { client } = await connectClient(db);
+
+    const result = await client.callTool({
+      name: "search_all",
+      arguments: { query: "accumulate" },
+    });
+
+    expect(result.isError).toBeFalsy();
+    const content = result.content as Array<{ type: string; text: string }>;
+    expect(content[0].type).toBe("text");
+    const parsed = JSON.parse(content[0].text) as Array<{ id: string }>;
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed.length).toBeGreaterThan(0);
+    expect(typeof parsed[0].id).toBe("string");
+  });
+
+  it("calls get_full_document and returns the doc body", async () => {
+    const db = createSearchDB();
+    const id = insertDoc(db, {
+      type: "graypaper_section",
+      title: "Accumulate",
+      content: "Full body of the accumulate section...",
+    });
+    const { client } = await connectClient(db);
+
+    const result = await client.callTool({
+      name: "get_full_document",
+      arguments: { id },
+    });
+
+    expect(result.isError).toBeFalsy();
+    const content = result.content as Array<{ type: string; text: string }>;
+    const parsed = JSON.parse(content[0].text) as {
+      id: string;
+      content: string;
+    };
+    expect(parsed.id).toBe(id);
+    expect(parsed.content).toContain("Full body of the accumulate section");
+  });
+
+  it("returns isError for get_full_document with an unknown id", async () => {
+    const { client } = await connectClient();
+    const result = await client.callTool({
+      name: "get_full_document",
+      arguments: { id: "does-not-exist" },
+    });
+    expect(result.isError).toBe(true);
+  });
+});

backend/src/api.ts

@@ -19,6 +19,7 @@ import {
 import { searchPages, searchPagesRequestSchema } from "./api/searchPages.js";
 import { embeddingCache } from "./cache/embeddingCache.js";
 import type { SearchDB } from "./data/searchIndex.js";
+import { createMcpHandler } from "./mcp/handler.js";
 
 const isDevelopment = process.env.NODE_ENV === "development";
 
@@ -28,14 +29,18 @@ export function createApp(db: SearchDB, dataDir: string) {
   // Middleware
   app.use(logger());
 
-  app.use(
-    cors({
-      origin: isDevelopment
-        ? (origin) =>
-            /^https?:\/\/localhost(:\d+)?$/.test(origin) ? origin : null
-        : "*",
-    })
-  );
+  // CORS is only relevant for browser-origin callers; /mcp is server-to-server
+  // and must not advertise any allowed origin. Skip the middleware for it.
+  const corsMiddleware = cors({
+    origin: isDevelopment
+      ? (origin) =>
+          /^https?:\/\/localhost(:\d+)?$/.test(origin) ? origin : null
+      : "*",
+  });
+  app.use(async (c, next) => {
+    if (c.req.path === "/mcp") return next();
+    return corsMiddleware(c, next);
+  });
 
   // Health check endpoint
   app.get("/health", (c) => {
@@ -80,5 +85,9 @@ export function createApp(db: SearchDB, dataDir: string) {
 
   app.post("/ask", handleAsk(db, dataDir));
 
+  // MCP (Model Context Protocol): stateless Streamable HTTP endpoint exposing
+  // the same two tools the /ask agent uses (search_all + get_full_document).
+  app.all("/mcp", createMcpHandler(db, dataDir));
+
   return app;
 }