From 8af2a973a822e747b3139df3cdd99967eb4f3a95 Mon Sep 17 00:00:00 2001 From: Enrico Becker <90171652+Dewinator@users.noreply.github.com> Date: Tue, 28 Apr 2026 10:07:30 +0200 Subject: [PATCH] =?UTF-8?q?feat(swarm):=20GET=20/.well-known/mycelium-node?= =?UTF-8?q?=20=E2=80=94=20self-signed=20advertisement=20(#87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3c of the Swarm Foundation Plan: serve this node's NodeAdvertisement at the well-known URL per SWARM_SPEC §3.3 / §4.1. First end-to-end wire path — composes Phase 1b (NodeIdentityService.getSelf), Phase 2 (signWithSelfKey), and Phase 3a (WIRE_SPEC_VERSION + NodeAdvertisement type) without modifying any of them. - new mcp-server/src/swarm/endpoints/node-advertisement.ts — pure handler that returns an HTTP-shaped triple (status/headers/body) so the surrounding server is policy-free. - new mcp-server/src/__tests__/node-advertisement-endpoint.test.ts — boots a real http.Server, fetches the endpoint, asserts §4.1 + §4.5 contract: status/content-type/cache-control, multihash(pubkey) === node_id, and signature verifies via Phase 2's verifier against the on-wire pubkey. Plus 503 cases for missing public-url / display-name too long / no self-row. - scripts/dashboard-server.mjs registers the route and a singleton NodeIdentityService — no new server, no new port (issue constraint). - README documents MYCELIUM_PUBLIC_URL (required) and MYCELIUM_DISPLAY_NAME (optional, ≤ 64 chars). Hard constraints honored: unauthenticated, no new web framework, no silent key generation (missing self-row → 503), body is exactly a single NodeAdvertisement with no envelope. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 18 ++ .../node-advertisement-endpoint.test.ts | 223 ++++++++++++++++++ .../src/swarm/endpoints/node-advertisement.ts | 162 +++++++++++++ scripts/dashboard-server.mjs | 29 +++ 4 files changed, 432 insertions(+) create mode 100644 mcp-server/src/__tests__/node-advertisement-endpoint.test.ts create mode 100644 mcp-server/src/swarm/endpoints/node-advertisement.ts diff --git a/README.md b/README.md index ed9f2a4..d96d307 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,24 @@ Full architecture, configuration knobs, observability surface, and failure modes --- +## Swarm endpoints + +When the dashboard server is running, mycelium exposes a discovery endpoint +that lets a peer verify this node's identity without any prior trust: + +| Endpoint | Spec | Purpose | +|---|---|---| +| `GET /.well-known/mycelium-node` | [SWARM_SPEC §4.1](docs/SWARM_SPEC.md) | Returns this node's self-signed `NodeAdvertisement`. Unauthenticated and idempotent. | + +Two environment variables control the response: + +- **`MYCELIUM_PUBLIC_URL`** *(required for swarm participation)* — the absolute `https://` URL where this node's swarm endpoints are reachable from peers (not localhost). Goes into the advertisement's `endpoint_url`. If unset, the endpoint returns **503**; the rest of mycelium is unaffected. +- **`MYCELIUM_DISPLAY_NAME`** *(optional, ≤ 64 characters)* — a human-friendly label for the node, surfaced as the advertisement's `display_name`. Names longer than the spec cap are rejected up front rather than published. + +The signing key (`MYCELIUM_NODE_KEY`, default `~/.mycelium/node.key`) is bootstrapped by `scripts/init-node-identity.mjs` and never leaves the host. The endpoint refuses to mint a key on the fly: a missing self-row surfaces as **503**, never as a silent re-bootstrap. + +--- + ## Roadmap mycelium = **one simulated brain** (memory, neurochemistry, sleep, motivation, curiosity, forgetting, deepening, emergence). Multiple brains = multiple mycelium instances chosen by the user. Phases: diff --git a/mcp-server/src/__tests__/node-advertisement-endpoint.test.ts b/mcp-server/src/__tests__/node-advertisement-endpoint.test.ts new file mode 100644 index 0000000..6a3fc35 --- /dev/null +++ b/mcp-server/src/__tests__/node-advertisement-endpoint.test.ts @@ -0,0 +1,223 @@ +/** + * Integration tests for the /.well-known/mycelium-node endpoint + * (Swarm Phase 3c, issue #87). + * + * Boots a real `node:http` server, registers the handler, and curls it + * via global `fetch`. The point of going end-to-end (rather than + * unit-testing `buildNodeAdvertisementResponse` in isolation) is to + * verify the wiring contract from §4.1 / §4.5: status, content-type, + * cache-control, body shape, multihash invariant, and signature + * verification — all observable to a peer that knows nothing about our + * internals. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { generateKeyPairSync } from "node:crypto"; + +import { buildNodeAdvertisementResponse } from "../swarm/endpoints/node-advertisement.js"; +import { computeNodeId } from "../services/node-identity.js"; +import { signWithSelfKey, verify } from "../services/signature.js"; +import { WIRE_SPEC_VERSION } from "../services/wire-types.js"; + +// --------------------------------------------------------------------------- +// Identity fixture — fresh keypair per test, no shared state +// --------------------------------------------------------------------------- + +interface Identity { + pem: string; + pubkeyRaw: Uint8Array; + pubkeyB64: string; + nodeId: string; +} + +function freshIdentity(): Identity { + const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + // Last 32 bytes of the SPKI DER are the raw Ed25519 pubkey — same + // extraction the wire-validator tests use. + const spki = publicKey.export({ type: "spki", format: "der" }); + const pubkeyRaw = new Uint8Array(spki.subarray(spki.length - 32)); + const pem = privateKey.export({ type: "pkcs8", format: "pem" }) as string; + return { + pem, + pubkeyRaw, + pubkeyB64: Buffer.from(pubkeyRaw).toString("base64"), + nodeId: computeNodeId(Buffer.from(pubkeyRaw)), + }; +} + +/** + * Boot a one-route HTTP server on an ephemeral port, return the base URL + * and a teardown fn. Resolves the port AFTER `listen()` has bound — the + * fetch URL is therefore guaranteed to point at the running server. + */ +async function bootServer( + handler: (req: http.IncomingMessage, res: http.ServerResponse) => Promise +): Promise<{ url: string; close: () => Promise }> { + const server = http.createServer((req, res) => { + if (req.url === "/.well-known/mycelium-node" && req.method === "GET") { + void handler(req, res).catch((e) => { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: String(e) })); + }); + return; + } + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + if (!addr || typeof addr === "string") throw new Error("server bind failed"); + const url = `http://127.0.0.1:${addr.port}`; + const close = () => + new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())) + ); + return { url, close }; +} + +// --------------------------------------------------------------------------- +// Happy path — full §4.1 contract +// --------------------------------------------------------------------------- + +test("GET /.well-known/mycelium-node returns a valid self-signed advertisement", async () => { + const self = freshIdentity(); + + const { url, close } = await bootServer(async (_req, res) => { + const result = await buildNodeAdvertisementResponse({ + loadSelf: async () => ({ + node_id: self.nodeId, + pubkey_b64: self.pubkeyB64, + }), + // Inject the fresh PEM via signWithSelfKey's `loadPem` so the + // signature path is exercised exactly as production runs it, + // minus the disk read. + signRecord: (r) => signWithSelfKey(r, { loadPem: () => self.pem }), + publicUrl: "https://node.example.com", + displayName: "test-node", + }); + res.writeHead(result.status, result.headers); + res.end(result.body); + }); + + try { + const response = await fetch(`${url}/.well-known/mycelium-node`); + + // Status + headers — §4.1 + §4.5 + assert.equal(response.status, 200); + assert.equal( + response.headers.get("content-type"), + "application/json; charset=utf-8" + ); + assert.equal(response.headers.get("cache-control"), "no-store"); + + // Body parses, has every required NodeAdvertisement field. + const body = (await response.json()) as Record; + for (const field of [ + "node_id", + "pubkey", + "endpoint_url", + "spec_version", + "signed_at", + "signature", + ]) { + assert.ok( + Object.prototype.hasOwnProperty.call(body, field), + `missing required field "${field}"` + ); + } + assert.equal(body.spec_version, WIRE_SPEC_VERSION); + assert.equal(body.endpoint_url, "https://node.example.com"); + assert.equal(body.display_name, "test-node"); + + // `signed_at` must be ISO 8601 UTC with millisecond precision and `Z` + // suffix per the §3 type table. + assert.match( + body.signed_at as string, + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ + ); + + // multihash invariant — rule 6 of §5. We hash the on-wire pubkey + // (decoded from unpadded base64url) and assert it round-trips to + // node_id. Same helper Phase 1b introduced — no new crypto here. + const pubkeyOnWire = Buffer.from(body.pubkey as string, "base64url"); + assert.equal(pubkeyOnWire.length, 32); + assert.equal(computeNodeId(pubkeyOnWire), body.node_id); + + // Signature verifies via Phase 2's verifier against the on-wire + // pubkey. `verify` strips `signature` internally and JCS-recomputes + // the bytes, so this is exactly what a peer would do. + const sigOk = verify( + body, + body.signature as string, + new Uint8Array(pubkeyOnWire) + ); + assert.equal(sigOk, true, "signature did not verify against on-wire pubkey"); + } finally { + await close(); + } +}); + +// --------------------------------------------------------------------------- +// 503 cases — the spec is silent on the exact code, but the issue body +// pins MYCELIUM_PUBLIC_URL missing → 503; we extend that pattern to +// every "not yet ready" mode so the operator gets a deterministic +// signal instead of a 200 with an unsigned shell. +// --------------------------------------------------------------------------- + +test("returns 503 when MYCELIUM_PUBLIC_URL is missing", async () => { + const self = freshIdentity(); + const result = await buildNodeAdvertisementResponse({ + loadSelf: async () => ({ node_id: self.nodeId, pubkey_b64: self.pubkeyB64 }), + signRecord: (r) => signWithSelfKey(r, { loadPem: () => self.pem }), + publicUrl: undefined, + }); + assert.equal(result.status, 503); + const body = JSON.parse(result.body) as { error?: string }; + assert.match(body.error ?? "", /MYCELIUM_PUBLIC_URL/); +}); + +test("returns 503 when no self-row exists in `nodes`", async () => { + const result = await buildNodeAdvertisementResponse({ + loadSelf: async () => null, + publicUrl: "https://node.example.com", + }); + assert.equal(result.status, 503); + const body = JSON.parse(result.body) as { error?: string }; + assert.match(body.error ?? "", /not initialized/); +}); + +test("returns 503 when display_name exceeds the §3.3 64-char cap", async () => { + const self = freshIdentity(); + const result = await buildNodeAdvertisementResponse({ + loadSelf: async () => ({ node_id: self.nodeId, pubkey_b64: self.pubkeyB64 }), + signRecord: (r) => signWithSelfKey(r, { loadPem: () => self.pem }), + publicUrl: "https://node.example.com", + displayName: "x".repeat(65), + }); + assert.equal(result.status, 503); + const body = JSON.parse(result.body) as { error?: string }; + assert.match(body.error ?? "", /64/); +}); + +test("omits display_name from the wire body when not configured", async () => { + const self = freshIdentity(); + const result = await buildNodeAdvertisementResponse({ + loadSelf: async () => ({ node_id: self.nodeId, pubkey_b64: self.pubkeyB64 }), + signRecord: (r) => signWithSelfKey(r, { loadPem: () => self.pem }), + publicUrl: "https://node.example.com", + displayName: null, + }); + assert.equal(result.status, 200); + const body = JSON.parse(result.body) as Record; + assert.equal( + Object.prototype.hasOwnProperty.call(body, "display_name"), + false, + "display_name should not appear when not configured (it's optional in §3.3)" + ); + // And the signature still verifies — making sure we did not silently + // include an empty display_name in the canonical bytes. + const pubkeyOnWire = Buffer.from(body.pubkey as string, "base64url"); + const sigOk = verify(body, body.signature as string, new Uint8Array(pubkeyOnWire)); + assert.equal(sigOk, true); +}); diff --git a/mcp-server/src/swarm/endpoints/node-advertisement.ts b/mcp-server/src/swarm/endpoints/node-advertisement.ts new file mode 100644 index 0000000..21e2b63 --- /dev/null +++ b/mcp-server/src/swarm/endpoints/node-advertisement.ts @@ -0,0 +1,162 @@ +/** + * GET /.well-known/mycelium-node — Swarm Phase 3c (issue #87). + * + * Serves THIS node's self-signed `NodeAdvertisement` per SWARM_SPEC §3.3 + * and §4.1. First end-to-end wire-path: a peer can curl this endpoint, + * verify `multihash(pubkey) == node_id`, and verify the Ed25519 signature + * over `JCS(record − signature)` — no out-of-band trust root needed. + * + * Composes Phase 1b (`NodeIdentityService.getSelf`) and Phase 2 + * (`signWithSelfKey` from `services/signature.ts`). Does NOT touch any + * key material itself: the privkey stays in `~/.mycelium/node.key`, + * the DB row stays untouched, and a missing self-row surfaces as 503 + * — never as a silent re-bootstrap (issue §"Hard constraints"). + */ +import { + signWithSelfKey, + type SignedRecord, +} from "../../services/signature.js"; +import { + WIRE_SPEC_VERSION, + type NodeAdvertisement, +} from "../../services/wire-types.js"; + +/** + * What the handler needs from the rest of the system. Injected so the + * tests can exercise the full path without standing up Supabase or + * touching `~/.mycelium/node.key`. + */ +export interface NodeAdvertisementDeps { + /** + * Load the self-row from the `nodes` table (`is_self = true`). Returns + * null when the node has not been bootstrapped yet — the handler + * answers 503 instead of generating a key. + */ + loadSelf: () => Promise<{ node_id: string; pubkey_b64: string } | null>; + /** + * Sign an unsigned record with the node's persistent self-key. + * Defaults to `signWithSelfKey` (reads `MYCELIUM_NODE_KEY` or + * `~/.mycelium/node.key`). + */ + signRecord?: (record: T) => SignedRecord; + /** + * `MYCELIUM_PUBLIC_URL` — the absolute https:// URL where this node's + * swarm endpoints are reachable (§3.3 `endpoint_url`). Required; + * missing value is a 503, not a silent fallback. The handler does NOT + * validate that the URL begins with `https://` — that is rejection + * rule 13 of §5 and is a receiver-side concern; emitting an http URL + * here would still produce a record that fails on verify, which is + * the correct failure mode rather than silently rewriting the value. + */ + publicUrl: string | undefined; + /** + * `MYCELIUM_DISPLAY_NAME` — optional human-friendly label, ≤ 64 chars + * (§3.3 size cap; rule 12 of §5). Names longer than the cap are + * rejected up front so the node never publishes a record the + * validator would drop. + */ + displayName?: string | null; +} + +export interface HttpResponseShape { + status: number; + headers: Record; + body: string; +} + +const RESPONSE_HEADERS: Readonly> = Object.freeze({ + "Content-Type": "application/json; charset=utf-8", + "Cache-Control": "no-store", +}); + +/** + * Build the response for `GET /.well-known/mycelium-node`. Returns an + * HTTP-shaped triple (`status` / `headers` / `body`) so the surrounding + * HTTP server can write it without further policy. The endpoint is + * unauthenticated and idempotent (§4.1) — every error mode is + * operator-readable, not opaque, so a peer trying to debug a discovery + * failure can see exactly which precondition is missing. + */ +export async function buildNodeAdvertisementResponse( + deps: NodeAdvertisementDeps +): Promise { + if (!deps.publicUrl) { + return jsonResponse(503, { error: "MYCELIUM_PUBLIC_URL not configured" }); + } + if (deps.displayName != null && deps.displayName.length > 64) { + return jsonResponse(503, { + error: "MYCELIUM_DISPLAY_NAME exceeds 64 characters (SWARM_SPEC §3.3)", + }); + } + + let self: { node_id: string; pubkey_b64: string } | null; + try { + self = await deps.loadSelf(); + } catch (e) { + return jsonResponse(503, { + error: "loadSelf failed", + detail: e instanceof Error ? e.message : String(e), + }); + } + if (!self) { + return jsonResponse(503, { + error: + "node identity not initialized; run scripts/init-node-identity.mjs first", + }); + } + + // The wire format encodes the pubkey as unpadded base64url (§3.3); the + // DB stores raw bytes that `NodeIdentityService` surfaces as standard + // padded base64. Re-encode here so a peer hashing the on-wire pubkey + // recovers exactly the bytes the DB row was built from — that is the + // invariant rule 6 of §5 enforces on the receiver. + const pubkeyRaw = Buffer.from(self.pubkey_b64, "base64"); + if (pubkeyRaw.length !== 32) { + return jsonResponse(503, { + error: `self pubkey is ${pubkeyRaw.length} bytes; expected 32 (Ed25519)`, + }); + } + const pubkeyB64Url = pubkeyRaw.toString("base64url"); + + // Build the unsigned record. `signed_at` is added by the signer so it + // lands inside the canonical bytes (§2.2 step 1: the signature is + // computed over the record minus the signature field). + const unsigned: Omit & { + display_name?: string; + } = { + node_id: self.node_id, + pubkey: pubkeyB64Url, + endpoint_url: deps.publicUrl, + spec_version: WIRE_SPEC_VERSION, + }; + if (deps.displayName) unsigned.display_name = deps.displayName; + + const signer = deps.signRecord ?? signWithSelfKey; + let signed: SignedRecord; + try { + signed = signer(unsigned); + } catch (e) { + return jsonResponse(503, { + error: "signing failed", + detail: e instanceof Error ? e.message : String(e), + }); + } + + const advertisement: NodeAdvertisement = { + ...signed.record, + signature: signed.signature, + }; + return { + status: 200, + headers: { ...RESPONSE_HEADERS }, + body: JSON.stringify(advertisement), + }; +} + +function jsonResponse(status: number, body: unknown): HttpResponseShape { + return { + status, + headers: { ...RESPONSE_HEADERS }, + body: JSON.stringify(body), + }; +} diff --git a/scripts/dashboard-server.mjs b/scripts/dashboard-server.mjs index 792c1af..053628d 100644 --- a/scripts/dashboard-server.mjs +++ b/scripts/dashboard-server.mjs @@ -39,6 +39,15 @@ const FED_DIST = path.join(ROOT, "mcp-server", "dist", "services"); const { FederationService } = await import(path.join(FED_DIST, "federation.js")); const { GuardService: FedGuardService } = await import(path.join(FED_DIST, "guard.js")); +// Swarm Phase 3c (issue #87): /.well-known/mycelium-node handler + node +// identity adapter both live in the compiled MCP server tree so they share +// signing primitives with the rest of the swarm code path. +const { NodeIdentityService } = await import(path.join(FED_DIST, "node-identity.js")); +const SWARM_ENDPOINTS_DIST = path.join(ROOT, "mcp-server", "dist", "swarm", "endpoints"); +const { buildNodeAdvertisementResponse } = await import( + path.join(SWARM_ENDPOINTS_DIST, "node-advertisement.js") +); + // --- load JWT_SECRET from docker/.env so we can mint a service_role JWT --- async function loadEnv() { try { @@ -1446,11 +1455,31 @@ async function handleTeacherEscalationResolve(req, res, escId) { res.end(JSON.stringify(data)); } +// --- Swarm Phase 3c: GET /.well-known/mycelium-node ---------------------- +// Serves the local node's self-signed NodeAdvertisement (SWARM_SPEC §3.3 + +// §4.1). Unauthenticated and idempotent on purpose — discovery starts here. +// Lazy NodeIdentityService init: a stale process_env update from a restart +// would otherwise be ignored. +const nodeIdentityService = new NodeIdentityService(UPSTREAM, SERVICE_JWT); +async function handleNodeAdvertisement(_req, res) { + const result = await buildNodeAdvertisementResponse({ + loadSelf: () => nodeIdentityService.getSelf(), + publicUrl: process.env.MYCELIUM_PUBLIC_URL || env.MYCELIUM_PUBLIC_URL, + displayName: + process.env.MYCELIUM_DISPLAY_NAME || env.MYCELIUM_DISPLAY_NAME || null, + }); + res.writeHead(result.status, result.headers); + res.end(result.body); +} + const server = http.createServer((req, res) => { // Tiny access log. const t = new Date().toISOString(); res.on("finish", () => console.log(`${t} ${req.socket.remoteAddress} ${req.method} ${req.url} -> ${res.statusCode}`)); + // Swarm discovery — must precede the static fall-through so the + // .well-known path doesn't get rewritten as a file lookup. + if (req.url === "/.well-known/mycelium-node") return handleNodeAdvertisement(req, res); if (req.url.startsWith("/api/")) return proxyApi(req, res); if (req.url.startsWith("/belief")) return proxyBelief(req, res); if (req.url.startsWith("/guard")) return proxyGuard(req, res);