From 6c7338e02c7316974c879628729e9d035c0dc719 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 11:21:00 -0500 Subject: [PATCH] Merge main. Added ability for users to remove views. Added github actions for running tests on pushes and PRs. --- .github/workflows/ci.yml | 17 ++ fangorn-access-worker/README.md | 112 ++++--- fangorn-access-worker/package.json | 2 +- fangorn-access-worker/src/index.ts | 306 +++++++++++-------- fangorn-access-worker/test/index.spec.ts | 256 +++++++++------- fangorn-access-worker/wrangler.toml | 47 ++- pinata-url-provider/src/index.js | 4 + quickbeam-registry/README.md | 60 +++- quickbeam-registry/examples/README.md | 3 +- quickbeam-registry/examples/manage-views.mjs | 13 +- quickbeam-registry/src/index.js | 174 +++++++++-- quickbeam-registry/test/index.test.js | 110 +++++++ quickbeam-registry/wrangler.toml | 10 + 13 files changed, 768 insertions(+), 346 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8f2e98e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,17 @@ +name: CI + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm -r typecheck + - run: pnpm -r test diff --git a/fangorn-access-worker/README.md b/fangorn-access-worker/README.md index a5cb68d..15df3e3 100644 --- a/fangorn-access-worker/README.md +++ b/fangorn-access-worker/README.md @@ -1,5 +1,20 @@ # Fangorn access worker + +``` sh +cd ~/fangorn/webworker/fangorn-access-worker && pnpm deploy +node -e 'import("viem").then(({keccak256,stringToBytes:s})=>console.log(keccak256(s(`sond3r:upload-token:${process.env.K}`))))' # K= + +node -e 'import("viem").then(({keccak256,stringToBytes:s})=>console.log(keccak256(s +(`sond3r:upload-token:0xde0e6c1c331fcd8692463d6ffcf20f9f2e1847264f7a3f578cf54f62f05196cb`))))' +# K= +npx wrangler secret put UPLOAD_HMAC_SECRET # ← that value +openssl rand -hex 32 | npx wrangler secret put WORKER_X25519_SECRET +# then WORKER_URL=https://fangorn-access-worker..workers.dev ./deploy.sh + +Pin WORKER_X25519_SECRET before anyone publishes — unset, the worker mints a key into the bucket it protects. +``` + A Cloudflare Worker that releases decryption keys against on-chain settlement. **Publishers deploy their own** — one worker per R2 bucket — so the content sits in their Cloudflare account, on their bill, under their own terms with @@ -50,14 +65,13 @@ Re-run `npx wrangler types` after changing `wrangler.toml`. There is nothing to configure, because the two things that would normally need configuring configure themselves: -- **The X25519 identity** is minted into your bucket on first request. It is the - key every DEK in your bucket is sealed to, it never leaves your Cloudflare - account, and it is generated once and kept. -- **The upload gate** claims itself. `POST /claim` stores the hash of an upload - token plus the address of the wallet that signed for it, and every later upload - must present that token. SOND3R does this the moment you connect the worker, - and can rotate the token later against the same wallet — so losing the token - never strands the bucket. +- **`UPLOAD_HMAC_SECRET`** — REQUIRED. Without it the worker authorizes nobody + and every upload 401s, deliberately: it would otherwise be an open write + endpoint on a bucket you pay for. It must equal the relay's own + `keccak256(utf8("sond3r:upload-token:" + ETH_PRIVATE_KEY))`. +- **`WORKER_X25519_SECRET`** — 32 bytes of hex, the key every DEK in the bucket + is sealed to. Unset, the worker mints one *into the bucket it protects*, so pin + it and keep a copy: losing it strands every resource ever published here. ## Test @@ -87,18 +101,25 @@ public key and stored beside it. **The worker never sees plaintext** — it hand back a 32-byte key to callers who have paid, and the decryption happens on the buyer's machine. -| route | gated? | does | -|---|---|---| -| `GET /pubkey` | no | the X25519 key publishers seal DEKs to | -| `GET /ct/:id` | no | streams ciphertext, with HTTP Range support | -| `POST /access` | **yes** | checks settlement, unseals the DEK, returns 32 bytes | -| `POST /upload/:id` | **yes** | stores ciphertext + sealed DEK | -| `POST /claim` | **yes** | claims (or rotates) the bucket's upload token | +| route | gated? | does | +| -------------------- | ------- | ---------------------------------------------------- | +| `GET /pubkey` | no | the X25519 key publishers seal DEKs to | +| `GET /ct/:id` | no | streams ciphertext, with HTTP Range support | +| `POST /access` | **yes** | checks settlement, unseals the DEK, returns 32 bytes | +| `GET /upload/:id` | **yes** | reads an object back (a publisher's own manifest) | +| `POST /upload/:id` | **yes** | stores ciphertext + sealed DEK | +| `DELETE /upload/:id` | **yes** | drops an object and its sealed DEK | `/ct/` is deliberately open: ciphertext is safe to hand to anyone, and leaving it ungated is what lets a video stream with ordinary Range requests. Only keys are gated. +**Free tier.** The relay mints an upload token for *any* signed-in address, so +holding a token no longer means anyone vouched for the bearer — this worker is +what bounds the bill. Each owner gets `FREE_BYTES` (a `[vars]` entry, 50 MiB by +default), metered in `usage/` beside the bytes it counts and credited back +on delete. Over the cap, `POST /upload` answers `413 {"reason":"quota"}`. + `/access` releases a DEK when the request is signed, within `TIMESTAMP_WINDOW` seconds, by a stealth address the registry says has settled. In order: @@ -116,34 +137,45 @@ Object keys must be bytes32 — `resourceId` for chunk 0, `keccak256(resourceId ++ uint32 i)` for the rest. Anything else 404s, which is what keeps `/ct/` from serving the bucket's own `.dek` blobs and worker secret. -## Taking back a claimed bucket - -`POST /claim` with a token the bucket doesn't hold answers 401 and a `reason`: +## One bucket, many publishers -| `reason` | means | fix | -|---|---|---| -| `needs-signature` | claimed by another token | sign the claim message with the owning wallet — retry Connect in the publisher portal, which prompts for it | -| `not-owner` | claimed, and owned by a different address | connect with the wallet named in the error | -| `pinned` | the worker has an `UPLOAD_TOKEN` secret, which overrides everything | paste that value into the portal's *Upload token* field, or `npx wrangler secret delete UPLOAD_TOKEN` | +The `/upload/` routes are gated on a token that **names its bearer**: -The claim message is `sond3r storage claim\ntoken: \ntime: -`, valid for 10 minutes. The first claim records the signer as the bucket's -owner; only that address can point the bucket at a different token afterwards. -Nothing else in the bucket is touched — ciphertext, sealed DEKs and the X25519 -identity all survive, so already-published files keep working. - -A bucket claimed before this shipped has no recorded owner, so the first valid -signature adopts it. That is the migration path for buckets stranded by the old -token-only gate. - -## Optional env - -Both are for the shared deployment and neither is needed for your own: +``` +Authorization: Bearer . +``` -| var | effect | -|---|---| -| `WORKER_X25519_SECRET` | pins the identity instead of minting one. **Required** on a worker that already has DEKs sealed to a key — minting a new one strands every published file. `openssl rand -hex 32 \| npx wrangler secret put WORKER_X25519_SECRET` | -| `UPLOAD_TOKEN` | pins the upload token instead of claiming on first use | +The relay derives it from its service key (`uploadTokenFor` in sond3r's +`server/index.js`); this worker recomputes the MAC (`macFor`) and reads the owner +address straight out of the token. No bucket state, no round trip, nothing to +claim — a publisher who has never touched this worker can upload immediately, and +the same wallet derives the same token from any machine. That is what replaced the +old first-upload-claims-the-bucket gate, which could only ever hold one publisher. + +Object keys are already namespaced per publisher (`resourceIdFor(owner, uid)`, +`manifestKey(owner)`), so accidental collisions are impossible. Deliberate ones +are not — uids are public, so any publisher can compute another's `resourceId`. +So the owner is stamped on every object as R2 custom metadata and re-checked on +every write, delete and read-back: + +| status | means | +| ------ | -------------------------------------------------------------------- | +| `401` | no token, or a MAC that does not verify against `UPLOAD_HMAC_SECRET` | +| `403` | a valid token, but this object belongs to another publisher | + +First writer keeps the key. An existing object with **no** owner recorded is +refused rather than adopted — it predates the shared bucket and there is nobody to +attribute it to. + +**Rotating the relay's key** re-derives every token at once; update +`UPLOAD_HMAC_SECRET` here in the same breath and publishers do nothing. Nothing +else in the bucket is touched by any of this: ciphertext, sealed DEKs and the +X25519 identity all survive, so already-published files keep working. + +**Bring your own storage** — a publisher deploying this into their own Cloudflare +account, with the bucket claiming its own token on first upload — is gone for now +and will come back as an option. It is in git, along with sond3r's +`server/cloudflare.js`. ## Develop diff --git a/fangorn-access-worker/package.json b/fangorn-access-worker/package.json index 1f08479..bd80d3e 100644 --- a/fangorn-access-worker/package.json +++ b/fangorn-access-worker/package.json @@ -4,7 +4,7 @@ "private": true, "scripts": { "dev": "wrangler dev --local", - "deploy": "wrangler deploy --env shared", + "deploy": "wrangler deploy", "typecheck": "tsc --noEmit", "test": "vitest run" }, diff --git a/fangorn-access-worker/src/index.ts b/fangorn-access-worker/src/index.ts index 202d1e8..597d5dc 100644 --- a/fangorn-access-worker/src/index.ts +++ b/fangorn-access-worker/src/index.ts @@ -21,6 +21,10 @@ import { gcm } from '@noble/ciphers/aes.js' // ------------------------------------------------------------ // The access worker is a key-release oracle, not a decryptor. // +// One worker, one bucket, EVERY publisher. Keys never collide because they are +// already derived from the publisher's address, and writes are attributed and +// re-checked per object — see `uploadOwner` and `mayTouch`. +// // Envelope model: episodes are AES-encrypted under a random 32-byte DEK. The // big ciphertext lives in R2 keyed by `resourceId`. The DEK is sealed to THIS // worker's static X25519 key (see fangorn `seal()`), and that sealed blob @@ -45,8 +49,10 @@ interface Env { TIMESTAMP_WINDOW: string /** 32-byte X25519 secret (hex). Optional — minted into the bucket if unset. */ WORKER_X25519_SECRET?: string - /** Pins the upload token. Optional — the bucket is claimed on first upload if unset. */ - UPLOAD_TOKEN?: string + /** 32 bytes of hex, shared with the relay that mints upload tokens. REQUIRED to upload. */ + UPLOAD_HMAC_SECRET?: string + /** Free bytes per wallet. Optional — defaults to 50 MiB. */ + FREE_BYTES?: string } /** @@ -135,7 +141,10 @@ const ZERO_ADDRESS = '0x0000000000000000000000000000000000000000' const CORS_HEADERS = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST, DELETE, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + // X-Sealed-Dek is here because publishers encrypt and upload from the BROWSER + // now, not from a relay. Without it every direct upload dies on preflight, and + // the error the publisher sees names CORS rather than anything they can fix. + 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Sealed-Dek', } function withCors(response: Response): Response { @@ -373,141 +382,160 @@ async function handleAccess(request: Request, env: Env): Promise { // Route: POST /upload/:resourceId — store ciphertext + sealed DEK // ------------------------------------------------------------ -/** R2 key holding the SHA-256 of the token allowed to upload. See `authorizeUpload`. */ -const TOKEN_KEY = '.upload-token' +// ------------------------------------------------------------ +// Who is uploading — one shared bucket, many publishers. +// +// "Does the caller hold this bucket's token" stopped being a useful question the +// moment one bucket started serving everybody. So the token NAMES ITS BEARER: +// +// Authorization: Bearer . +// +// `secret` is UPLOAD_HMAC_SECRET, shared with the relay that mints the tokens +// (sond3r's server/index.js, `uploadTokenFor`) and with nothing else. Verifying +// it recovers the owner address with no bucket state and no round trip, which is +// what lets a publisher who has never touched this worker upload immediately — +// and what replaced the first-upload-claims-the-bucket dance, which could only +// ever hold one publisher. +// +// Every key the relay writes is already derived from the owner address +// (`resourceIdFor(owner, uid)`, `manifestKey(owner)` in sond3r's src/envelope.js +// and src/encrypt.js), so two publishers cannot collide by accident. What one +// COULD do on purpose is overwrite: uids are public, so anyone can compute +// anyone's resourceId. Hence the owner is stamped on every object as R2 custom +// metadata and re-checked on every write, delete and read-back. +// ------------------------------------------------------------ + +/** Byte-for-byte with sond3r's `uploadTokenFor` (server/index.js). */ +const macFor = (secret: Hex, owner: Address): string => + keccak256(encodePacked(['bytes32', 'address'], [secret, owner])) -const sha256Hex = async (s: string): Promise => - bytesToHex(new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(s)))) +/** Length-independent compare, so a near-miss MAC leaks no prefix. */ +function sameMac(a: string, b: string): boolean { + if (a.length !== b.length) return false + let diff = 0 + for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i) + return diff === 0 +} /** - * Uploads are gated, and the gate installs itself on first use. + * The publisher this request is authorized as, lowercased, or null. * - * `UPLOAD_TOKEN` wins when set — that is how the shared deployment is pinned. - * Otherwise the FIRST upload to a fresh bucket claims it: whatever token it - * presents is hashed and stored, and every later upload must match. A publisher - * deploying their own worker never sets anything; sond3r mints a token and - * claims the worker the moment they connect it (see POST /api/worker there). - * - * The exposure is the gap between `wrangler deploy` and that first claim, which - * is seconds and ends the first time the real publisher uploads. Being - * permissionless — the previous behaviour — meant anyone who learned the URL - * could overwrite a publisher's ciphertext forever. + * A worker with no (or a malformed) UPLOAD_HMAC_SECRET authorizes NOBODY. Every + * other failure mode here is a misconfiguration that would otherwise open a + * bucket the operator pays for to anyone who learned the URL. */ -async function authorizeUpload(request: Request, env: Env): Promise { - const presented = (request.headers.get('Authorization') ?? '').replace(/^Bearer /i, '').trim() - if (!presented) return false - - const pinned = (env.UPLOAD_TOKEN ?? '').trim() - if (pinned) return presented === pinned - - const digest = await sha256Hex(presented) - const claimed = await env.BUCKET.get(TOKEN_KEY) - if (claimed) return (await claimed.text()) === digest - - // Unclaimed bucket: this token becomes the owner. `etagDoesNotMatch: '*'` - // makes the claim atomic, so two racers cannot both believe they won. - const won = await env.BUCKET.put(TOKEN_KEY, digest, { onlyIf: { etagDoesNotMatch: '*' } }) - if (won) return true - const theirs = await env.BUCKET.get(TOKEN_KEY) - return theirs ? (await theirs.text()) === digest : false +function uploadOwner(request: Request, env: Env): Address | null { + const secret = (env.UPLOAD_HMAC_SECRET ?? '').trim() + if (!/^0x[0-9a-fA-F]{64}$/.test(secret)) return null + const [owner, mac] = (request.headers.get('Authorization') ?? '').replace(/^Bearer /i, '').trim().split('.') + if (!/^0x[0-9a-fA-F]{40}$/.test(owner ?? '') || !mac) return null + const lower = owner.toLowerCase() as Address + return sameMac(mac.toLowerCase(), macFor(secret as Hex, lower)) ? lower : null } -/** R2 key holding the ETH address allowed to rotate the upload token. See `handleClaim`. */ -const OWNER_KEY = '.upload-owner' - -/** The exact string a publisher's wallet signs to claim this bucket. sond3r's - * relay builds the identical string (server/index.js, `claimMessage`) — they - * must agree byte-for-byte or every claim recovers a different address. */ -const claimMessage = (digest: string, timestamp: number): string => - `sond3r storage claim\ntoken: ${digest}\ntime: ${timestamp}` - -/** Wider than TIMESTAMP_WINDOW: a claim waits on a wallet popup, /access doesn't. */ -const CLAIM_WINDOW = 600 +/** R2 custom metadata key: which publisher put these bytes here. */ +const OWNER_META = 'owner' +const ownerMeta = (owner: Address) => ({ customMetadata: { [OWNER_META]: owner } }) /** - * POST /claim — install or ROTATE this bucket's upload token, with no wrangler. + * Whether `owner` may write, delete or read back `key`. * - * The token alone can't authorize rotating itself (a publisher who lost it is - * exactly who needs to rotate), so the authority is the publisher's wallet: the - * first claim records the signing address in `.upload-owner`, and afterwards - * only that address can point the bucket at a different token. Rotating - * ETH_PRIVATE_KEY on the relay — which changes the derived token and used to - * strand the bucket behind `wrangler r2 object delete` — is now a re-Connect. + * A key nobody has written is free to claim; after that it is that publisher's + * for good. First-writer-wins means a publisher COULD squat a rival's future + * resourceId, but only by guessing a uid before it exists — whereas the + * alternative, trusting the derivation, lets anyone overwrite any published file + * in the bucket. * - * Presenting the token that already won needs no signature, so this stays - * idempotent for the ordinary connect-again case. + * An existing object with NO owner metadata predates the shared bucket. There is + * nobody to attribute it to, so it is refused rather than adopted: adopting it + * would hand the first caller who asked whatever the old deployment left behind. * - * ponytail: a bucket claimed BEFORE this shipped has no `.upload-owner`, so the - * first valid signature adopts it. That reopens the same land-grab window a - * fresh bucket already has, once, for legacy buckets only — the alternative is - * leaving them permanently stranded, which is the bug being fixed. + * ponytail: one HEAD per object touched, and last-writer-wins on a genuine race + * for a fresh key. Both are fine at one publisher per key; if concurrent claims + * ever matter, put() with `onlyIf: { etagDoesNotMatch: '*' }` for the first write. */ -async function handleClaim(request: Request, env: Env): Promise { - const presented = (request.headers.get('Authorization') ?? '').replace(/^Bearer /i, '').trim() - if (!presented) return jsonError('missing upload token', 401, 'missing-token') - - const pinned = (env.UPLOAD_TOKEN ?? '').trim() - if (pinned) { - // A pinned worker accepts nothing else, and no signature can override it — - // the secret is Cloudflare-account state, not bucket state. - return presented === pinned - ? new Response(JSON.stringify({ claimed: true }), { headers: { 'Content-Type': 'application/json' } }) - : jsonError('this worker pins an UPLOAD_TOKEN secret, and this is not it', 401, 'pinned') - } +const owned = (head: R2Object | null, owner: Address): boolean => + !head || head.customMetadata?.[OWNER_META] === owner - const digest = await sha256Hex(presented) - const claimed = await env.BUCKET.get(TOKEN_KEY) - if (claimed && (await claimed.text()) === digest) { - return new Response(JSON.stringify({ claimed: true }), { headers: { 'Content-Type': 'application/json' } }) - } +async function mayTouch(env: Env, key: string, owner: Address): Promise { + return owned(await env.BUCKET.head(key), owner) +} - const body = (await request.json().catch(() => ({}))) as { timestamp?: number; signature?: Hex } - const timestamp = Number(body.timestamp) - if (!body.signature || !Number.isFinite(timestamp) || Math.abs(Math.floor(Date.now() / 1000) - timestamp) > CLAIM_WINDOW) { - return jsonError('this bucket is claimed by a different token — sign to take it over', 401, 'needs-signature') - } +// ------------------------------------------------------------ +// The free tier — every wallet, no registration, no bill. +// +// Holding a valid token is no longer a statement that anyone vouched for the +// bearer: the relay mints one for any signed-in wallet, so the only thing left +// standing between a stranger and the operator's R2 bill is this cap. It is +// metered per owner, in the bucket, beside the bytes it meters. +// +// `usage/` is deliberately NOT a bytes32, so `isObjectKey` keeps the +// ungated /ct/ route from serving it. +// ------------------------------------------------------------ - let signer: Address - try { - signer = await recoverMessageAddress({ message: claimMessage(digest, timestamp), signature: body.signature }) - } catch { - return jsonError('invalid claim signature', 401, 'needs-signature') - } +const DEFAULT_FREE_BYTES = 50 * 1024 * 1024 - let owner = await env.BUCKET.get(OWNER_KEY).then((o) => o?.text()) - if (!owner) { - // `etagDoesNotMatch: '*'` makes first-owner atomic, so two racers cannot - // both believe they won. - const won = await env.BUCKET.put(OWNER_KEY, signer, { onlyIf: { etagDoesNotMatch: '*' } }) - owner = won ? signer : await env.BUCKET.get(OWNER_KEY).then((o) => o?.text()) - } - if (owner?.toLowerCase() !== signer.toLowerCase()) { - return jsonError(`this bucket belongs to ${owner} — connect with that wallet`, 401, 'not-owner') - } +const freeBytes = (env: Env): number => Number(env.FREE_BYTES) || DEFAULT_FREE_BYTES + +const usageKey = (owner: Address): string => `usage/${owner}` + +async function usedBytes(env: Env, owner: Address): Promise { + const obj = await env.BUCKET.get(usageKey(owner)) + return obj ? Number(await obj.text()) || 0 : 0 +} - await env.BUCKET.put(TOKEN_KEY, digest) - return new Response(JSON.stringify({ claimed: true, owner }), { headers: { 'Content-Type': 'application/json' } }) +/** + * ponytail: read-modify-write, so two uploads racing from ONE wallet can both + * read the same total and one increment is lost — that wallet ends up + * undercounted by a file, never overcharged, and the next upload reads the + * survivor. R2 has no counter primitive; a Durable Object per owner is the + * upgrade if free storage ever becomes worth gaming. + */ +async function addUsage(env: Env, owner: Address, delta: number): Promise { + const next = Math.max(0, (await usedBytes(env, owner)) + delta) + await env.BUCKET.put(usageKey(owner), String(next)) } async function handleUpload(request: Request, env: Env, resourceId: string): Promise { if (!isObjectKey(resourceId)) return jsonError('resourceId must be 32 bytes of hex', 400) - if (!(await authorizeUpload(request, env))) { - return jsonError('missing or incorrect upload token', 401) - } + const owner = uploadOwner(request, env) + if (!owner) return jsonError('missing or incorrect upload token', 401) + const previous = await env.BUCKET.head(resourceId) + if (!owned(previous, owner)) return jsonError('this object belongs to another publisher', 403) if (!request.body) return jsonError('empty body (expected ciphertext stream)', 400) + // Checked against the DECLARED length before a byte is streamed, so a wallet + // already at its cap is refused up front instead of after paying for the + // transfer. The real size is what gets recorded below, so a client that lies + // here overshoots by at most one file and is then locked out. + const limit = freeBytes(env) + const used = await usedBytes(env, owner) + const budget = limit - used + (previous?.size ?? 0) + const declared = Number(request.headers.get('Content-Length') ?? 0) + if (declared > budget) { + return jsonError(`free storage exhausted: ${used} of ${limit} bytes used`, 413, 'quota') + } + + // Absent DEK = the object is the publisher's OWN state (sond3r keeps its + // per-publisher manifest here), not ciphertext anyone will ever buy. It is + // stored as-is and no DEK object is written, so /access has nothing to + // release and the settlement path cannot hand it out. Anything a buyer pays + // for still arrives with a DEK, because encryptAndUpload always sends one. const sealedHex = request.headers.get('X-Sealed-Dek') - if (!sealedHex) return jsonError('missing X-Sealed-Dek header', 400) - let sealed: Uint8Array - try { - sealed = hexToBytes(sealedHex as Hex) - } catch { - return jsonError('X-Sealed-Dek is not valid hex', 400) + let sealed: Uint8Array | null = null + if (sealedHex) { + try { + sealed = hexToBytes(sealedHex as Hex) + } catch { + return jsonError('X-Sealed-Dek is not valid hex', 400) + } } try { - await env.BUCKET.put(dekKey(resourceId), sealed) // tiny; buffered - await env.BUCKET.put(resourceId, request.body) // big; streamed to R2 + if (sealed) await env.BUCKET.put(dekKey(resourceId), sealed, ownerMeta(owner)) // tiny; buffered + else await env.BUCKET.delete(dekKey(resourceId)) // no stale DEK from a previous life of this key + const written = await env.BUCKET.put(resourceId, request.body, ownerMeta(owner)) // big; streamed to R2 + await addUsage(env, owner, (written?.size ?? declared) - (previous?.size ?? 0)) } catch (e) { console.error('R2 put failed:', e) return jsonError('upload failed', 500) @@ -524,14 +552,16 @@ async function handleUpload(request: Request, env: Env, resourceId: string): Pro // // Same path and same gate as upload, because it is the same authority: whoever // holds the bucket's upload token put these bytes here and is the only one who -// may take them away. An ungated delete would let anyone empty a publisher's -// library over HTTP. +// may take them away — which on a shared bucket means the publisher the token +// names, not merely whoever holds a token. An ungated delete would let anyone +// empty a publisher's library over HTTP; a token-only gate would let any OTHER +// publisher do it. // // Chunked resources are deleted one key at a time by the caller, which knows the // chunk count (sond3r's server/settle.js chunkKey). The worker deliberately does // not walk or guess the chunk list: `isObjectKey` is what keeps this route away -// from `.worker-x25519-secret` and `.upload-token`, and it only holds because -// every key it accepts is a literal bytes32. A "delete all chunks of X" route +// from `.worker-x25519-secret`, and it only holds because every key it accepts +// is a literal bytes32. A "delete all chunks of X" route // would have to synthesize keys, and a bug there deletes the wrong publisher's // objects. // @@ -541,13 +571,19 @@ async function handleUpload(request: Request, env: Env, resourceId: string): Pro async function handleDelete(request: Request, env: Env, resourceId: string): Promise { if (!isObjectKey(resourceId)) return jsonError('resourceId must be 32 bytes of hex', 400) - if (!(await authorizeUpload(request, env))) { - return jsonError('missing or incorrect upload token', 401) - } + const owner = uploadOwner(request, env) + if (!owner) return jsonError('missing or incorrect upload token', 401) + // Without this, one publisher could empty another's library out of the bucket + // they share — the delete gate is now ownership, not merely "holds a token". + const existing = await env.BUCKET.head(resourceId) + if (!owned(existing, owner)) return jsonError('this object belongs to another publisher', 403) try { // The DEK goes with the ciphertext. Leaving it behind would keep releasing // a key for bytes that no longer exist. await env.BUCKET.delete([resourceId, dekKey(resourceId)]) + // Deleting has to give the quota back, or the free tier is a lifetime + // total and a publisher who tidies up gets nothing for it. + if (existing) await addUsage(env, owner, -existing.size) } catch (e) { console.error('R2 delete failed:', e) return jsonError('delete failed', 500) @@ -557,6 +593,35 @@ async function handleDelete(request: Request, env: Env, resourceId: string): Pro }) } +// ------------------------------------------------------------ +// Route: GET /upload/:resourceId — read an object back, upload token required +// +// The mirror of POST /upload, and gated the same way for the same reason: this +// hands back raw stored bytes, so the only caller allowed is whoever put them +// there. GET /ct/ cannot serve this job — it is deliberately ungated because +// everything under it is ciphertext, and the publisher state read through here +// is not. +// +// It exists so a publisher's manifest can live in their OWN bucket instead of on +// the relay's disk. The relay stages nothing and stores nothing per publisher; +// this is how the library comes back on a different machine. +// ------------------------------------------------------------ + +async function handleFetchOwn(request: Request, env: Env, resourceId: string): Promise { + if (!isObjectKey(resourceId)) return jsonError('resourceId must be 32 bytes of hex', 400) + const owner = uploadOwner(request, env) + if (!owner) return jsonError('missing or incorrect upload token', 401) + // A publisher's manifest is read through here, and on a shared bucket that + // makes ownership the gate: holding a valid token proves who you are, not + // that you may read someone else's library. + if (!(await mayTouch(env, resourceId, owner))) return jsonError('this object belongs to another publisher', 403) + const object = await env.BUCKET.get(resourceId) + // 404, not an error: "this publisher has no manifest yet" is the ordinary + // first-run state and the caller starts from an empty one. + if (!object) return jsonError('not found', 404) + return new Response(object.body, { headers: { 'Content-Type': 'application/octet-stream' } }) +} + // ------------------------------------------------------------ // Route: GET /ct/:resourceId — stream the encrypted object (ungated; it's ciphertext) // ------------------------------------------------------------ @@ -617,6 +682,9 @@ export default { return withCors(jsonError('worker misconfigured', 500)) } } + if (pathname.startsWith('/upload/')) { + return withCors(await handleFetchOwn(request, env, decodeURIComponent(pathname.slice(8)))) + } if (pathname.startsWith('/ct/')) { return withCors(await handleCiphertext(request, env, decodeURIComponent(pathname.slice(4)))) } @@ -624,10 +692,6 @@ export default { if (method === 'POST') { if (pathname === '/access') return withCors(await handleAccess(request, env)) - // Claim the bucket without uploading, so a publisher connecting a fresh - // worker closes the unclaimed window at connect time rather than at - // first publish. Idempotent: re-presenting the winning token succeeds. - if (pathname === '/claim') return withCors(await handleClaim(request, env)) if (pathname.startsWith('/upload/')) { return withCors(await handleUpload(request, env, decodeURIComponent(pathname.slice(8)))) } diff --git a/fangorn-access-worker/test/index.spec.ts b/fangorn-access-worker/test/index.spec.ts index 09eaeef..5c9e292 100644 --- a/fangorn-access-worker/test/index.spec.ts +++ b/fangorn-access-worker/test/index.spec.ts @@ -1,18 +1,29 @@ import { env, SELF, createExecutionContext } from "cloudflare:test"; import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { encodePacked, keccak256, toBytes } from "viem"; +import { encodePacked, keccak256, toBytes, type Address, type Hex } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { FangornConfig } from "@fangorn-network/sdk/lib/config.js"; import worker from "../src/index"; -// The worker is a key-release oracle over a bucket it also owns. These cover the -// three things that make a PUBLISHER-OWNED deployment safe to hand someone who -// has never heard of R2: it mints its own identity, it does not serve that -// identity back out, and it will not let a stranger overwrite their ciphertext. +// The worker is a key-release oracle over a bucket it shares between EVERY +// publisher on the relay. These cover what makes that safe: it mints its own +// identity, it does not serve that identity back out, only a relay-minted token +// can write at all, and one publisher cannot touch another's objects. const RID = `0x${"ab".repeat(32)}` as const; const HEX64 = /^0x[0-9a-f]{64}$/; +// The secret the relay and the worker share. Installed on `env` here because +// wrangler.toml deliberately does not carry it — it is a `wrangler secret`. +const SECRET = `0x${"5e".repeat(32)}` as Hex; + +const ALICE = "0x1111111111111111111111111111111111111111" as Address; +const BOB = "0x2222222222222222222222222222222222222222" as Address; + +/** What sond3r's `uploadTokenFor` hands a publisher's browser. */ +const tokenFor = (owner: Address, secret: Hex = SECRET) => + `${owner}.${keccak256(encodePacked(["bytes32", "address"], [secret, owner]))}`; + const upload = (body: string, token?: string, id: string = RID) => SELF.fetch(`https://w/upload/${id}`, { method: "POST", @@ -23,9 +34,9 @@ const upload = (body: string, token?: string, id: string = RID) => body, }); -// Each test starts on a fresh, unclaimed bucket — the state a publisher's worker -// is in the moment the Deploy to Cloudflare button finishes. +// Each test starts on an empty bucket with the shared secret configured. beforeEach(async () => { + (env as { UPLOAD_HMAC_SECRET?: string }).UPLOAD_HMAC_SECRET = SECRET; const { objects } = await env.BUCKET.list(); await Promise.all(objects.map((o) => env.BUCKET.delete(o.key))); }); @@ -54,25 +65,81 @@ describe("worker identity", () => { }); describe("upload gate", () => { + // A cross-repo contract, and the quietest one to get wrong: sond3r's relay + // mints this token independently (server/index.js, `uploadTokenWith`) and its + // --selfcheck asserts the identical literal. If either side drifts, one of the + // two suites goes red instead of every upload silently 401ing. + it("agrees with the relay on the token format", () => { + expect(tokenFor("0x1111111111111111111111111111111111111111", `0x${"5e".repeat(32)}`)).toBe( + "0x1111111111111111111111111111111111111111.0x7908a77e560b9353c8bfc501f7654a7c3ba31939f0b83d123edac190f797c7fd", + ); + }); + it("refuses an unauthenticated upload", async () => { expect((await upload("ciphertext")).status).toBe(401); }); - it("claims an unclaimed bucket to the first token, then rejects others", async () => { - expect((await upload("mine", "token-a")).status).toBe(201); - // Same token again: still the owner. - expect((await upload("mine too", "token-a")).status).toBe(201); - // A stranger who learned the URL cannot overwrite the publisher's bytes. - expect((await upload("theirs", "token-b")).status).toBe(401); + it("accepts a relay-minted token and refuses a forged one", async () => { + expect((await upload("mine", tokenFor(ALICE))).status).toBe(201); + // Same publisher again: overwriting your own object is an ordinary republish. + expect((await upload("mine too", tokenFor(ALICE))).status).toBe(201); + // Right shape, wrong secret — the whole point of the MAC. + expect((await upload("theirs", tokenFor(ALICE, `0x${"99".repeat(32)}`))).status).toBe(401); + // A bare address with no MAC authorizes nobody. + expect((await upload("theirs", ALICE)).status).toBe(401); }); - it("stores only the token's hash, never the token", async () => { - await upload("mine", "token-a"); - expect(await (await env.BUCKET.get(".upload-token"))!.text()).not.toContain("token-a"); + // The bucket is shared, so this is THE isolation property: uids are public, so + // Bob can compute Alice's resourceId — he just cannot write to it. + it("refuses a publisher writing over another publisher's object", async () => { + expect((await upload("alice", tokenFor(ALICE))).status).toBe(201); + expect((await upload("bob", tokenFor(BOB))).status).toBe(403); + expect(await (await env.BUCKET.get(RID))!.text()).toBe("alice"); + }); + + it("stamps the owner on the ciphertext AND its sealed DEK", async () => { + await upload("alice", tokenFor(ALICE)); + expect((await env.BUCKET.head(RID))!.customMetadata?.owner).toBe(ALICE); + // Unstamped, the DEK would be writable by anyone — and swapping a DEK is how + // you make a publisher's file decrypt to something else. + expect((await env.BUCKET.head(`${RID}.dek`))!.customMetadata?.owner).toBe(ALICE); + }); + + // An object left by the pre-shared-bucket deployment has nobody to attribute + // it to. Adopting it would hand it to whoever asked first. + it("refuses an existing object with no owner recorded", async () => { + await env.BUCKET.put(RID, "from the old deployment"); + expect((await upload("mine now", tokenFor(ALICE))).status).toBe(403); + }); + + // Uploads are refused outright rather than falling open — a misconfigured + // worker would otherwise be an open write endpoint on a bucket we pay for. + it("authorizes nobody when the shared secret is missing", async () => { + delete (env as { UPLOAD_HMAC_SECRET?: string }).UPLOAD_HMAC_SECRET; + expect((await upload("mine", tokenFor(ALICE))).status).toBe(401); }); it("rejects a key that is not a bytes32", async () => { - expect((await upload("x", "token-a", ".upload-token")).status).toBe(400); + expect((await upload("x", tokenFor(ALICE), ".worker-x25519-secret")).status).toBe(400); + }); +}); + +// The mirror of upload, and the reason it is gated: a publisher's manifest is +// read back through here, and on a shared bucket that is somebody's library +// index, not ciphertext. +describe("read-back", () => { + const fetchOwn = (token?: string, id: string = RID) => + SELF.fetch(`https://w/upload/${id}`, { headers: token ? { Authorization: `Bearer ${token}` } : {} }); + + it("gives a publisher their own object back", async () => { + await upload("alice", tokenFor(ALICE)); + expect(await (await fetchOwn(tokenFor(ALICE))).text()).toBe("alice"); + }); + + it("refuses another publisher's object, and anyone with no token", async () => { + await upload("alice", tokenFor(ALICE)); + expect((await fetchOwn(tokenFor(BOB))).status).toBe(403); + expect((await fetchOwn()).status).toBe(401); }); }); @@ -87,123 +154,41 @@ describe("delete", () => { }); it("removes the ciphertext and its sealed DEK together", async () => { - await upload("ciphertext", "token-a"); + await upload("ciphertext", tokenFor(ALICE)); expect(await env.BUCKET.get(RID)).not.toBeNull(); expect(await env.BUCKET.get(`${RID}.dek`)).not.toBeNull(); - expect((await del("token-a")).status).toBe(200); + expect((await del(tokenFor(ALICE))).status).toBe(200); expect(await env.BUCKET.get(RID)).toBeNull(); // A DEK left behind would go on releasing a key for bytes that are gone. expect(await env.BUCKET.get(`${RID}.dek`)).toBeNull(); }); it("refuses an unauthenticated delete", async () => { - await upload("ciphertext", "token-a"); + await upload("ciphertext", tokenFor(ALICE)); expect((await del()).status).toBe(401); expect(await env.BUCKET.get(RID)).not.toBeNull(); }); - it("refuses a stranger who learned the URL", async () => { - await upload("ciphertext", "token-a"); - expect((await del("token-b")).status).toBe(401); + // A registered publisher is still a stranger to someone else's library. Without + // this, any one of them could empty the shared bucket. + it("refuses another publisher, token and all", async () => { + await upload("ciphertext", tokenFor(ALICE)); + expect((await del(tokenFor(BOB))).status).toBe(403); expect(await env.BUCKET.get(RID)).not.toBeNull(); }); // The same guard that keeps /ct/ off the private key keeps DELETE off it. - it("cannot delete the worker's own secret or the token record", async () => { - await upload("ciphertext", "token-a"); - expect((await del("token-a", ".worker-x25519-secret")).status).toBe(400); - expect((await del("token-a", ".upload-token")).status).toBe(400); - expect(await env.BUCKET.get(".upload-token")).not.toBeNull(); + it("cannot delete the worker's own secret", async () => { + await SELF.fetch("https://w/pubkey"); // force the mint + expect((await del(tokenFor(ALICE), ".worker-x25519-secret")).status).toBe(400); + expect(await env.BUCKET.get(".worker-x25519-secret")).not.toBeNull(); }); it("is idempotent, so a retry after a half-finished delete is safe", async () => { - await upload("ciphertext", "token-a"); - expect((await del("token-a")).status).toBe(200); - expect((await del("token-a")).status).toBe(200); - }); -}); - -// The publisher's wallet is the authority over the bucket — see handleClaim. -const PUBLISHER = privateKeyToAccount(`0x${"11".repeat(32)}`); -const STRANGER = privateKeyToAccount(`0x${"22".repeat(32)}`); - -const digestOf = async (token: string) => - `0x${[...new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token)))] - .map((b) => b.toString(16).padStart(2, "0")).join("")}`; - -// Duplicated from the worker on purpose: if src/ changes this string, this test -// should fail rather than follow along — sond3r's relay builds it independently -// (server/index.js, `claimMessage`) and both sides must keep producing the same -// bytes. CLAIM_VECTOR below pins the format for the relay's own self-check. -const claimMessage = async (token: string, timestamp: number) => - `sond3r storage claim\ntoken: ${await digestOf(token)}\ntime: ${timestamp}`; - -const claim = async (token: string, signer?: typeof PUBLISHER, timestamp = Math.floor(Date.now() / 1000)) => - SELF.fetch("https://w/claim", { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify({ - timestamp, - signature: signer ? await signer.signMessage({ message: await claimMessage(token, timestamp) }) : undefined, - }), - }); - -describe("/claim", () => { - it("pins the message format the relay signs against", async () => { - // sond3r/server/upload-token.js asserts the same literal. Two repos, one - // string: change it in one place only and the pair of tests catches it. - expect(await claimMessage("token-a", 1700000000)).toBe( - "sond3r storage claim\ntoken: 0xa70bf50e531ce1a817561f2f5d5b6645d4e806becf58ccc5e8cf6b8045a090a8\ntime: 1700000000" - ); - }); - - it("claims without uploading, and is idempotent", async () => { - expect((await claim("token-a", PUBLISHER)).status).toBe(200); - // Re-presenting the winning token is idempotent and needs no signature. - expect((await claim("token-a")).status).toBe(200); - expect((await claim("token-b")).status).toBe(401); - - // The claim really gated uploads, not just itself. - expect((await upload("theirs", "token-b")).status).toBe(401); - expect((await upload("mine", "token-a")).status).toBe(201); - }); - - it("refuses an unsigned claim on a bucket held by another token", async () => { - await claim("token-a", PUBLISHER); - const res = await claim("token-b"); - expect(res.status).toBe(401); - expect((await res.json<{ reason: string }>()).reason).toBe("needs-signature"); - }); - - // The bug this whole path exists for: rotating ETH_PRIVATE_KEY changes the - // relay's derived token, and used to strand the publisher's own bucket behind - // a wrangler command. - it("lets the owning wallet rotate to a new token", async () => { - await claim("old-token", PUBLISHER); - expect((await claim("new-token", PUBLISHER)).status).toBe(200); - - expect((await upload("mine", "new-token")).status).toBe(201); - expect((await upload("stale", "old-token")).status).toBe(401); - }); - - it("refuses a takeover signed by a different wallet", async () => { - await claim("token-a", PUBLISHER); - const res = await claim("token-b", STRANGER); - expect(res.status).toBe(401); - expect((await res.json<{ reason: string }>()).reason).toBe("not-owner"); - }); - - it("says so when the worker pins a token, which no signature can override", async () => { - (env as { UPLOAD_TOKEN?: string }).UPLOAD_TOKEN = "pinned-secret"; - try { - const res = await claim("token-a", PUBLISHER); - expect(res.status).toBe(401); - expect((await res.json<{ reason: string }>()).reason).toBe("pinned"); - expect((await claim("pinned-secret")).status).toBe(200); - } finally { - delete (env as { UPLOAD_TOKEN?: string }).UPLOAD_TOKEN; - } + await upload("ciphertext", tokenFor(ALICE)); + expect((await del(tokenFor(ALICE))).status).toBe(200); + expect((await del(tokenFor(ALICE))).status).toBe(200); }); }); @@ -347,3 +332,46 @@ describe("/access gate", () => { expect(res.status).toBe(403); }); }); + +// The free tier is the only thing left between a stranger's wallet and the +// operator's R2 bill: the relay hands an upload token to ANY signed-in address +// now, registered publisher or not, so the cap has to hold here. +describe("free tier", () => { + const OTHER = `0x${"cd".repeat(32)}` as const; + + beforeEach(() => { (env as { FREE_BYTES?: string }).FREE_BYTES = "10"; }); + afterEach(() => { delete (env as { FREE_BYTES?: string }).FREE_BYTES; }); + + it("refuses an upload that would exceed the wallet's free bytes", async () => { + expect((await upload("12345678", tokenFor(ALICE))).status).toBe(201); + const over = await upload("345", tokenFor(ALICE), OTHER); + expect(over.status).toBe(413); + expect(await over.json<{ reason: string }>()).toMatchObject({ reason: "quota" }); + // Metered per owner, so one wallet filling up cannot lock out another. + expect((await upload("12345678", tokenFor(BOB), OTHER)).status).toBe(201); + }); + + it("gives the bytes back on delete", async () => { + await upload("12345678", tokenFor(ALICE)); + expect((await upload("1234", tokenFor(ALICE), OTHER)).status).toBe(413); + await SELF.fetch(`https://w/upload/${RID}`, { + method: "DELETE", headers: { Authorization: `Bearer ${tokenFor(ALICE)}` }, + }); + expect((await upload("1234", tokenFor(ALICE), OTHER)).status).toBe(201); + }); + + // Re-publishing the same file replaces bytes rather than adding them; counting + // the new copy on top of the old would shrink the tier on every re-upload. + it("counts an overwrite once", async () => { + expect((await upload("12345678", tokenFor(ALICE))).status).toBe(201); + expect((await upload("87654321", tokenFor(ALICE))).status).toBe(201); + expect((await upload("12", tokenFor(ALICE), OTHER)).status).toBe(201); + }); + + // `usage/` is not a bytes32, which is what keeps the ungated ciphertext + // route from serving one wallet's bill to anyone who guesses the address. + it("does not expose the meter through /ct", async () => { + await upload("12345678", tokenFor(ALICE)); + expect((await SELF.fetch(`https://w/ct/usage/${ALICE}`)).status).toBe(404); + }); +}); diff --git a/fangorn-access-worker/wrangler.toml b/fangorn-access-worker/wrangler.toml index f18be19..e4e5353 100644 --- a/fangorn-access-worker/wrangler.toml +++ b/fangorn-access-worker/wrangler.toml @@ -1,34 +1,49 @@ name = "fangorn-access-worker" main = "src/index.ts" compatibility_date = "2025-07-18" +# Pinned: this wrangler login can see two accounts, and `wrangler deploy` with a +# choice to make stops and asks — which fails in CI and, worse, can put the +# storage every publisher depends on in the wrong account. +account_id = "0beaeb0776ca9e8404297afe0da73e5a" # Fangorn@fangorn.network -# This file is also a TEMPLATE: publishers deploy it into their own Cloudflare -# account via the Deploy to Cloudflare button, which reads the binding below and -# creates the bucket for them. Nothing else needs configuring — the worker mints -# its own X25519 key and claims its own upload token on first request. +# ONE deployment, one bucket, every sond3r publisher. Objects are attributed to +# the publisher who wrote them (R2 custom metadata) and keys are already derived +# from the publisher's address, so a shared bucket is not a shared library. # +# Bring-your-own-storage — a publisher deploying this into their own Cloudflare +# account — is gone for now; it lived in sond3r's server/cloudflare.js and the +# bucket-claim path here, and both are in git if it comes back. [[r2_buckets]] binding = "BUCKET" bucket_name = "sond3r" -[env.shared.vars] -SETTLEMENT_REGISTRY_ADDRESS = "0x35b6c6a975933c21442a04041b920f80b381d993" -ARBITRUM_SEPOLIA_RPC = "https://sepolia-rollup.arbitrum.io/rpc" -TIMESTAMP_WINDOW = "60" - [vars] # The live SettlementRegistry on Arbitrum Sepolia. /access authorizes against this # contract, so a worker pointed at the wrong one releases nothing. -SETTLEMENT_REGISTRY_ADDRESS = "0x35b6c6a975933c21442a04041b920f80b381d993" +SETTLEMENT_REGISTRY_ADDRESS = "0x480d54411d77820701fd80f42b81fb6e20176d12" ARBITRUM_SEPOLIA_RPC = "https://sepolia-rollup.arbitrum.io/rpc" TIMESTAMP_WINDOW = "60" -# Optional, and unset on a publisher-owned worker: -# WORKER_X25519_SECRET pins the identity instead of minting one. REQUIRED on a -# worker that already has DEKs sealed to a key. -# UPLOAD_TOKEN pins the upload token instead of claiming on first use -# Both are secrets: `wrangler secret put `. +# The free tier: bytes per wallet, metered in `usage/` in this bucket. +# Every signed-in address on the relay gets an upload token now, registered +# publisher or not, so this cap is the only thing between a stranger's wallet and +# the R2 bill. Raise it here and redeploy; nothing per-publisher is re-issued. +FREE_BYTES = "52428800" # 50 MiB + +# Secrets — `wrangler secret put `: +# +# UPLOAD_HMAC_SECRET REQUIRED. Uploads are refused outright without it. Must +# equal the relay's RELAY_SECRET: +# keccak256(utf8("sond3r:upload-token:" + ETH_PRIVATE_KEY)) +# Rotating the relay's key rotates this; publishers' +# tokens are re-derived and nothing has to be re-claimed. +# +# WORKER_X25519_SECRET 32 bytes of hex. Pins the identity every DEK in the +# bucket is sealed to, instead of minting one into the +# bucket on first use. SET IT on a real deployment: a +# minted key lives in the bucket it protects, and losing +# it strands every resource already published. [observability] enabled = true -head_sampling_rate = 1 \ No newline at end of file +head_sampling_rate = 1 diff --git a/pinata-url-provider/src/index.js b/pinata-url-provider/src/index.js index 00be813..f560e1d 100644 --- a/pinata-url-provider/src/index.js +++ b/pinata-url-provider/src/index.js @@ -64,6 +64,8 @@ export default { // Read the request once (the POST body can only be consumed a single time): // address plus the optional ownership proof (message + signature). const input = await readInput(request); + // TODO: make address an env var + // const address = process.env.STORAGE_SUBSCRIPTION_ADDR; const address = (input.address || '').toLowerCase(); if (!isAddress(address)) { return json(400, { error: 'Provide a valid EVM address via ?address=0x… or JSON body { "address": "0x…" }.' }, cors); @@ -85,6 +87,7 @@ export default { // view returns both registration status (it cross-calls DataRegistry internally) // and the subscription timestamp. STUB_REGISTRATION_CHECK skips the chain // entirely (a valid signature alone suffices — dev/testing without an RPC). + // TODO: is this needed? const stubbed = (env.STUB_REGISTRATION_CHECK ?? 'false') === 'true'; let access = null; if (!stubbed) { @@ -107,6 +110,7 @@ export default { // length). Absent → a back-compat default. Bounded per-request so nobody can // mint a URL for an absurd file. const maxUpload = Number(env.MAX_UPLOAD_SIZE || DEFAULT_MAX_UPLOAD); + // TODO: this seems gratuitious let size; if (input.size == null || input.size === '') { size = Number(env.DEFAULT_UPLOAD_SIZE || DEFAULT_UPLOAD_SIZE); diff --git a/quickbeam-registry/README.md b/quickbeam-registry/README.md index 007ac80..0c781fc 100644 --- a/quickbeam-registry/README.md +++ b/quickbeam-registry/README.md @@ -57,7 +57,7 @@ and nothing here reads an event log. | Route | Auth | Does | |---|---|---| -| `POST /views` | signature + subscription | create or replace a view (idempotent on `{requester, name}`); returns its id, search URL and MCP command. `hostedMcp: true` also provisions a Cloud Run MCP | +| `POST /views` | signature + subscription | create or replace a view (idempotent on `{requester, name}`); returns its id, search URL and MCP command. `hostedMcp: true` also provisions a Cloud Run MCP. `409` if another of your views already covers exactly these sources | | `GET /views?requester=0x…` | none | that wallet's views (omit `requester` for all) | | `GET /views/{id}` | none | one view | | `GET /watchlist` | none | the deduplicated union the instance polls: `{"sources":[{app, owner, namespace}]}`. `owner`/`namespace` may be `*` (that whole app); dedup is on all three, so the same publisher:subspace in two apps stays two entries | @@ -66,7 +66,18 @@ and nothing here reads an event log. | `GET /q/{id}/stream` | none | SSE: which of the view's domains changed, so a client pulls instead of polling | | `GET /q/{id}/cdn/catalog` | none | the instance catalog **filtered** to the view's domains | | `GET /q/{id}/cdn/*` | none | proxied to the CDN (shards, manifests, edges); `domains/{name}/…` outside the view is `404`, not forwarded | -| `POST /admin/remove` | admin signature | delete a view by `{id}` | +| `POST /views/remove` | requester signature | delete one of **your own** views by `{id}` | +| `POST /admin/remove` | admin signature | delete any view by `{id}` | + +A wallet cannot end up with the same view twice. Sending a view under a name it +already uses **replaces** it — the id is `(requester, name)`, so that is how you change +what a view watches without changing its URLs. Sending the same source set under a +*second* name is refused with `409` naming the view that already covers it: a view is a +filter over the shared collection, so a duplicate would be the same search under two +URLs, two catalogs and two hosted MCPs to keep in step. Sources are compared as a set of +canonical `app:owner:namespace` triples, so order, a repeat, or an app spelled as a name +on one side and its id on the other make no difference. Two *different* wallets asking +for the same namespaces is not a duplicate — that is the whole design. The search **and export** proxies both **strip any caller-supplied `scope`, `owner` or `namespace`** before injecting the view's own pairs, so a view URL always means that @@ -175,8 +186,8 @@ catalog, and returns its URL. An MCP is stateless, holds no connections and need disk, so it scales to zero — an idle user's MCP costs nothing. That is the opposite of the watcher, which is why the watcher lives on a VM. -Unticking the box on a later `POST /views` deletes the service; so does -`POST /admin/remove`. If Cloud Run is unconfigured (`GCP_*` unset), the request returns +Unticking the box on a later `POST /views` deletes the service; so does removing the +view (`POST /views/remove`, `POST /admin/remove`). If Cloud Run is unconfigured (`GCP_*` unset), the request returns an `mcpError` and the view still gets its search endpoint — the feature degrades, it does not fail the view. @@ -196,21 +207,32 @@ created against, and match the watcher's `APP`, or their catalogs come back empt ## Teardown -Nothing expires on its own — a lapsed subscription keeps running until someone removes -its view. That is deliberate: teardown is a founder action. +Nothing expires on its own — a view keeps being watched until somebody removes it. +Two routes, because they answer to different people: + +**`POST /views/remove` — the requester's own.** Ownership only: the signing wallet must +be the view's `requester`, and an admin wallet gets **no** override here (that is what +`/admin/remove` is for). There is no subscription check either — a wallet whose +subscription has lapsed must still be able to stop being watched. ```sh -# Any wallet in ADMIN_WALLETS. Two steps: collect the challenge, sign, resend. -curl -s -X POST "$WORKER/admin/remove" -H 'content-type: application/json' \ - -d '{"address":"0xYOURADMIN","id":"qb_147c24c5_music"}' +# Two steps, like every write here: collect the challenge, sign it, resend. +curl -s -X POST "$WORKER/views/remove" -H 'content-type: application/json' \ + -d '{"address":"0xYOU","id":"qb_147c24c5_music"}' # → { "challenge": "..." } sign it, then: -curl -s -X POST "$WORKER/admin/remove" -H 'content-type: application/json' \ - -d '{"address":"0xYOURADMIN","id":"qb_147c24c5_music","message":"","signature":"0x…"}' +curl -s -X POST "$WORKER/views/remove" -H 'content-type: application/json' \ + -d '{"address":"0xYOU","id":"qb_147c24c5_music","message":"","signature":"0x…"}' ``` -Removing a view drops its sources from the watchlist **only if no other view wants -them**. Add a founder by appending to `ADMIN_WALLETS` in `wrangler.toml` and -redeploying. +The website exposes this as **Stop watching** inside each view on the dashboard. + +**`POST /admin/remove` — any view.** Same body, same handshake, but the signer must be +in `ADMIN_WALLETS`. Add a founder by appending to it in `wrangler.toml` and redeploying. + +Either route deletes the view's row, its `dns:` label and its hosted MCP (a Cloud Run +failure comes back as `207` + `mcpError`; the row is gone regardless). Its sources leave +the watchlist **only if no other view wants them** — the instance cancels its stream +from that on-chain head once the last view referencing it goes. ## Configuration @@ -236,6 +258,7 @@ A `[build]` guard aborts the deploy if the installed SDK carries no valid addres | `ADMIN_WALLETS` | comma-separated wallets allowed to tear down | | `MAX_VIEWS_PER_WALLET` | abuse cap, not an entitlement; `"0"` disables | | `SEARCH_URL`, `CDN_URL` | the instance, plain HTTP | +| `INSTANCE_TIMEOUT_MS` | how long the instance may take to *answer* a proxied request before it is a `502` (default `15000`). Bounds time-to-headers only — `/stream` and `/export` still stream for as long as they like | | `VIEW_DOMAIN_SUFFIX` | per-view subdomains live under this; empty serves views at `/q/{id}/…` | | `GCP_PROJECT`, `GCP_REGION`, `GCP_SA_EMAIL`, `QUICKBEAM_IMAGE` | hosted MCP; unset disables the feature | | `GCP_SA_KEY` (**secret**) | the service-account JSON's `private_key`; needs `roles/run.admin` + `roles/iam.serviceAccountUser` | @@ -273,7 +296,12 @@ production. ## Known ceilings -- `GET /watchlist` and `GET /views` read every row. KV lists page at 1000 keys; fine - for a prototype, and the first thing to change if this becomes the product. +- `GET /watchlist` and `GET /views` read every row, cached in one `snapshot:views` key + (10 min TTL, dropped on every write) so a poll costs a `get` and not a KV **list** — + the op capped at 1000/day on the free plan, which an instance polling every 60s + exceeds on its own. Editing a `view:` row directly with `wrangler kv key put` does + NOT invalidate it: delete `snapshot:views` too, or wait out the TTL. +- The rebuilt snapshot is one value, and KV lists page at 1000 keys; fine for a + prototype, and the first thing to change if this becomes the product. - KV is eventually consistent, so a just-created view can take a moment to appear in `/watchlist`. The instance converges on the next poll either way. diff --git a/quickbeam-registry/examples/README.md b/quickbeam-registry/examples/README.md index e88026a..7f34c0a 100644 --- a/quickbeam-registry/examples/README.md +++ b/quickbeam-registry/examples/README.md @@ -56,7 +56,8 @@ The script exits non-zero on a refusal and names it: |---|---| | `202` / `200` | created / replaced, or removed | | `402` | that wallet has no active subscription | -| `403` | not a registered publisher, or not an admin for `/admin/remove` | +| `403` | not a registered publisher; not an admin for `/admin/remove`; not the requester for `/views/remove` | +| `409` | another of your views already covers exactly these sources | | `401` | the signature did not verify — wrong key for that address | | `429` | per-wallet view cap (`MAX_VIEWS_PER_WALLET`) | diff --git a/quickbeam-registry/examples/manage-views.mjs b/quickbeam-registry/examples/manage-views.mjs index 17308db..532ee84 100644 --- a/quickbeam-registry/examples/manage-views.mjs +++ b/quickbeam-registry/examples/manage-views.mjs @@ -13,8 +13,11 @@ * node examples/manage-views.mjs --worker https://…workers.dev --key 0xKEY \ * --name my-view --source 0xOWNER:namespace [--source 0xOWNER:other] [--hosted-mcp] * - * node examples/manage-views.mjs --worker https://…workers.dev --key 0xADMINKEY \ - * --remove qb_147c24c5_my-view + * node examples/manage-views.mjs --worker https://…workers.dev --key 0xKEY \ + * --remove qb_147c24c5_my-view [--admin] + * + * --remove deletes YOUR OWN view (the wallet that created it). --admin sends it to + * the founder route instead, which is the only way to remove somebody else's. * * node examples/manage-views.mjs --worker https://…workers.dev --watchlist * @@ -33,6 +36,7 @@ function parseArgs(argv) { else if (arg === '--name') out.name = next(); else if (arg === '--source') out.sources.push(next()); else if (arg === '--remove') out.remove = next(); + else if (arg === '--admin') out.admin = true; else if (arg === '--hosted-mcp') out.hostedMcp = true; else if (arg === '--watchlist') out.watchlist = true; else throw new Error(`Unknown argument: ${arg}`); @@ -104,8 +108,9 @@ async function main() { console.log(`address: ${account.address}`); if (args.remove) { - report(`POST /admin/remove ${args.remove}`, - await signedPost(worker, '/admin/remove', account, { id: args.remove })); + const route = args.admin ? '/admin/remove' : '/views/remove'; + report(`POST ${route} ${args.remove}`, + await signedPost(worker, route, account, { id: args.remove })); return; } diff --git a/quickbeam-registry/src/index.js b/quickbeam-registry/src/index.js index eb1a562..7425ad2 100644 --- a/quickbeam-registry/src/index.js +++ b/quickbeam-registry/src/index.js @@ -18,7 +18,8 @@ * it embedded. And there is no access control on reads: the source graphs are public * on-chain data, so a view id is a convenience, not a secret. Writes are gated * (signature + active subscription) because creating a view is what spends embedding - * work. + * work. Removing one spends nothing, so it needs only the signature — and only the + * requester's own: unwatching is theirs to do, not an admin's. * * Dependencies are `viem` (signature recovery + selector encoding) and the Fangorn * SDK, which supplies the deployment addresses. @@ -107,10 +108,12 @@ export default { // falling through to "provide a valid EVM address" for a mistyped *path* sends the // reader hunting for an auth problem that does not exist — which is exactly what a // watcher pointed at the old `/sources` route used to see. - if (url.pathname !== '/views' && url.pathname !== '/admin/remove') { + if (url.pathname !== '/views' && url.pathname !== '/views/remove' + && url.pathname !== '/admin/remove') { return json(404, { error: `Unknown route ${url.pathname}`, routes: ['GET /watchlist', 'GET /views', 'GET /views/{id}', 'POST /views', + 'POST /views/remove', 'GET /q/{viewId}/search', 'GET /q/{viewId}/export', 'GET /q/{viewId}/stream', 'GET /q/{viewId}/cdn/*', 'POST /admin/remove'], @@ -131,25 +134,24 @@ export default { return json(401, { ok: false, address, error: ownership.error, challenge: ownership.challenge }, cors); } - // ── Founder-only teardown ─────────────────────────────────────────────── - if (url.pathname === '/admin/remove') { - if (!isAdmin(env, address)) return json(403, { error: 'Not an admin wallet.' }, cors); + // ── Teardown ──────────────────────────────────────────────────────────── + // Two routes, same deletion, different people. `/views/remove` is how a requester + // stops watching what they asked for: OWNERSHIP ONLY — an admin wallet gets no + // pass here, and there is no subscription check, because unwatching has to keep + // working after the subscription that created the view lapses. Dropping the last + // view over a namespace is what takes it off /watchlist and cancels the + // instance's stream from that on-chain head. + if (url.pathname === '/views/remove' || url.pathname === '/admin/remove') { + const asAdmin = url.pathname === '/admin/remove'; const id = (input.id || '').trim(); const view = await readView(env, id); if (!view) return json(404, { error: `No view ${id}` }, cors); - - // Delete the hosted MCP before the row, so a failure here leaves the view - // visible rather than orphaning a Cloud Run service nothing points at. - let mcpError = null; - if (view.mcp) { - try { - await deleteMcpService(env, id); - } catch (err) { - mcpError = err.message; - } + if (asAdmin ? !isAdmin(env, address) : view.requester !== address) { + return json(403, { + error: asAdmin ? 'Not an admin wallet.' : `View ${id} belongs to another wallet.`, + }, cors); } - await env.QUICKBEAM_KV.delete(viewKey(id)); - await env.QUICKBEAM_KV.delete(dnsKey(dnsLabel(id))); + const mcpError = await deleteView(env, id, view); return json(mcpError ? 207 : 200, { ok: !mcpError, removed: id, ...(mcpError ? { mcpError } : {}) }, cors); } @@ -187,18 +189,30 @@ export default { const id = viewId(address, name); const existing = await readView(env, id); + const mine = (await listViews(env)).filter((v) => v.requester === address); + + // The same sources under a SECOND name is an accident, not a feature: a view is a + // filter over the shared collection, so two of them over one source set are the + // same search twice — two URLs, two catalogs and (hosted) two Cloud Run services + // to keep in step. Re-sending a view under its own name is the intended way to + // change one; that lands on the same `id` and replaces, which is why the twin + // search skips it. + const twin = mine.find((v) => v.id !== id && sameSources(v.sources, sources, env)); + if (twin) { + return json(409, { + ok: false, + error: `You already have a view over these sources, called "${twin.name}". ` + + `Use it, or name this one "${twin.name}" to replace it.`, + existing: withUrls(twin, url, env), + }, cors); + } - if (!existing) { - const cap = Number(env.MAX_VIEWS_PER_WALLET || 0); - if (cap > 0) { - const mine = (await listViews(env)).filter((v) => v.requester === address); - if (mine.length >= cap) { - return json(429, { - ok: false, - error: `This wallet already has ${mine.length} view(s), the current limit.`, - }, cors); - } - } + const cap = Number(env.MAX_VIEWS_PER_WALLET || 0); + if (!existing && cap > 0 && mine.length >= cap) { + return json(429, { + ok: false, + error: `This wallet already has ${mine.length} view(s), the current limit.`, + }, cors); } const view = { @@ -228,6 +242,7 @@ export default { await env.QUICKBEAM_KV.put(viewKey(id), JSON.stringify(view)); await env.QUICKBEAM_KV.put(dnsKey(dnsLabel(id)), id); + await invalidateViews(env); return json(existing ? 200 : 202, { ok: true, ...withUrls(view, url, env), @@ -300,6 +315,19 @@ function toAppId(nameOrId) { const withApp = (s, env) => ({ app: toAppId(s.app || env.DEFAULT_APP), owner: s.owner || '*', namespace: s.namespace || '*' }); +/** + * Do two source lists cover exactly the same thing? Compared as a set of canonical + * triples, so order, a repeated source, or an app spelled as a name on one side and as + * its id on the other cannot make a duplicate look new. + */ +function sameSources(a, b, env) { + const key = (sources) => [...new Set(sources.map((s) => { + const { app, owner, namespace } = withApp(s, env); + return `${app}:${owner}:${namespace}`; + }))].sort().join('|'); + return key(a) === key(b); +} + /** * CDN domain for one source. MUST match `_domain_for` in quickbeam/watcher.py — the * watcher names the directory, this names it back to filter a view's catalog. Byte @@ -405,6 +433,7 @@ async function withMcpStatus(env, view) { const updated = { ...view, mcp: { ...view.mcp, status: live.status, url: live.url } }; const { searchUrl, cdnUrl, mcpCommand, ...row } = updated; await env.QUICKBEAM_KV.put(viewKey(view.id), JSON.stringify(row)); + await invalidateViews(env); // the cached set still holds the pre-URL row return updated; } } catch { @@ -425,17 +454,60 @@ async function ensureMcp(env, view, url) { return { service: serviceName(view.id), status: live.status, url: live.url ?? null }; } +/** + * Delete a view and everything provisioned for it, returning the Cloud Run error if + * the hosted MCP could not be torn down. The MCP goes first, so a failure there is + * reported (207) rather than silently orphaning a service nothing points at; the row + * goes either way, because a view whose delete half-failed is worse than a stray + * service an admin can sweep. + */ +async function deleteView(env, id, view) { + let mcpError = null; + if (view.mcp) { + try { + await deleteMcpService(env, id); + } catch (err) { + mcpError = err.message; + } + } + await env.QUICKBEAM_KV.delete(viewKey(id)); + await env.QUICKBEAM_KV.delete(dnsKey(dnsLabel(id))); + await invalidateViews(env); + return mcpError; +} + async function readView(env, id) { if (!id) return null; const raw = await env.QUICKBEAM_KV.get(viewKey(id)); return raw ? JSON.parse(raw) : null; } +/** One key holding every view, so the common read is a `get` and not a `list`. */ +const SNAPSHOT_KEY = 'snapshot:views'; +/** Rebuild at most this often. A missed invalidation self-heals within one TTL. */ +const SNAPSHOT_TTL = 600; + /** - * Every view. KV list pages at 1000 keys — fine for a prototype, and the ceiling to - * remember before this becomes the product. + * Every view, from one cached key. + * + * The uncached form — `list({prefix:'view:'})` plus a `get` per row — was a LIST + * OPERATION ON EVERY REQUEST, and `/watchlist` is polled by each instance forever: + * at --sources-refresh=60 that is 1440/day against a free-plan ceiling of 1000, before + * the website's own /views calls. Cached, a poll is one `get` (100k/day) and the list + * runs at most once per SNAPSHOT_TTL — 144/day at the current value. + * + * Writers call `invalidateViews`, so the TTL is a safety net for the paths that do not: + * a `wrangler kv key put` straight into the namespace shows up within one TTL, or + * immediately if you delete this key too. + * + * ponytail: one key for the whole set. KV list pages at 1000 keys and a value caps at + * 25 MB, so the ceiling is the same prototype ceiling as before — per-app snapshot keys + * if a deployment ever outgrows it. */ async function listViews(env) { + const cached = await env.QUICKBEAM_KV.get(SNAPSHOT_KEY); + if (cached) return JSON.parse(cached); + const out = []; let cursor; do { @@ -446,9 +518,18 @@ async function listViews(env) { } cursor = page.list_complete ? undefined : page.cursor; } while (cursor); + await env.QUICKBEAM_KV.put(SNAPSHOT_KEY, JSON.stringify(out), { expirationTtl: SNAPSHOT_TTL }); return out; } +/** + * Drop the cached set. Called after every write, BEFORE the response goes out, so the + * caller's next read cannot see the set it just changed as stale — KV's own eventual + * consistency (up to 60s) is the remaining window, and it applied to the uncached form + * just the same. + */ +const invalidateViews = (env) => env.QUICKBEAM_KV.delete(SNAPSHOT_KEY); + /* ───────────────────────────── entitlement ──────────────────────────────── */ /** @@ -630,7 +711,7 @@ async function proxy(request, env, url, cors, resolved = null) { headers.delete('host'); let upstream; try { - upstream = await fetch(target, { + upstream = await instanceFetch(target, env, { method: request.method, headers, body: request.method === 'GET' || request.method === 'HEAD' ? undefined : request.body, @@ -658,7 +739,7 @@ async function proxy(request, env, url, cors, resolved = null) { async function filteredCatalog(env, view, cors) { let catalog; try { - const res = await fetch(`${trimSlash(env.CDN_URL)}/catalog`); + const res = await instanceFetch(`${trimSlash(env.CDN_URL)}/catalog`, env); if (!res.ok) return json(502, { error: `Catalog HTTP ${res.status}` }, cors); catalog = await res.json(); } catch (err) { @@ -683,7 +764,7 @@ async function resolveDomains(env, view) { } const match = domainMatcher(view, env); try { - const res = await fetch(`${trimSlash(env.CDN_URL)}/catalog`); + const res = await instanceFetch(`${trimSlash(env.CDN_URL)}/catalog`, env); if (!res.ok) return []; const catalog = await res.json(); return (catalog.domains || []).map((d) => d.name).filter(match); @@ -692,6 +773,33 @@ async function resolveDomains(env, view) { } } +/** + * A fetch at the shared instance, bounded on how long it may take to ANSWER — not on + * how long it may stream. + * + * `SEARCH_URL`/`CDN_URL` are a grey-cloud A record straight at a VM, so a box that is + * off, or a firewall that DROPs instead of refusing, black-holes the SYN. A plain + * fetch then hangs until Cloudflare gives up around 100s and the `catch` below never + * runs — which reads to a caller as "the endpoint is timing out" when what happened is + * "the instance is unreachable", the single most misleading failure this worker has. + * + * The timer is cleared the moment headers arrive, so `/stream` (SSE, open for hours) + * and `/export` (a whole corpus) stream for as long as they like — only the wait for + * a first answer is capped. + */ +async function instanceFetch(url, env, init) { + const ms = Number(env.INSTANCE_TIMEOUT_MS || 15000); + const abort = new AbortController(); + const timer = setTimeout(() => abort.abort(), ms); + try { + return await fetch(url, { ...init, signal: abort.signal }); + } catch (err) { + throw abort.signal.aborted ? new Error(`no answer within ${ms}ms`) : err; + } finally { + clearTimeout(timer); + } +} + const trimSlash = (s) => (s || '').replace(/\/$/, ''); /** diff --git a/quickbeam-registry/test/index.test.js b/quickbeam-registry/test/index.test.js index fbc88ad..f3f9051 100644 --- a/quickbeam-registry/test/index.test.js +++ b/quickbeam-registry/test/index.test.js @@ -302,6 +302,30 @@ test('re-creating a view replaces it and keeps createdAt', async () => { assert.equal(viewCount(), 1); }); +test('the same sources under a second name is refused, not duplicated', async () => { + const first = await (await createView(alice, 'music', [src(PUB, 'x')])).json(); + + const dupe = await createView(alice, 'tunes', [src(PUB, 'x')]); + assert.equal(dupe.status, 409); + const body = await dupe.json(); + assert.match(body.error, /music/); + assert.equal(body.existing.id, first.id); + assert.equal(viewCount(), 1); + + // Order, a repeat, and the app spelled as a name rather than its id are all the same + // source set — none of them is a way in. + const shuffled = await createView(alice, 'tunes', [ + src(PUB2, 'y'), src(PUB, 'x'), src(PUB, 'x', APP_NAME), + ]); + assert.equal(shuffled.status, 202, 'a genuinely different set still creates'); + assert.equal((await createView(alice, 'more', [src(PUB, 'x'), src(PUB2, 'y')])).status, 409); + assert.equal(viewCount(), 2); + + // Another wallet asking for the same namespaces is not a duplicate — that is the + // whole point of embedding once and filtering per requester. + assert.equal((await createView(bob, 'music', [src(PUB, 'x')])).status, 202); +}); + test('an unregistered wallet is told to register (403)', async () => { rpcResponse = () => accessResult(false, 0); const res = await createView(alice, 'v', [src(PUB, 'ns')]); @@ -487,6 +511,21 @@ test('cdn passthrough refuses a domain outside the view', async () => { assert.equal(lastInstanceUrl, null, 'must not reach the instance at all'); }); +test('a box that never answers is a 502, not a hang', async () => { + const v = await (await createView(alice, 'mine', [src(PUB, 'a')])).json(); + env.INSTANCE_TIMEOUT_MS = '50'; + // A black hole, not a refusal: an off box or a DROP firewall never completes the + // handshake, so without the bound this request runs until Cloudflare kills it ~100s + // later and the caller is told "timed out" about the wrong hop. + instanceResponse = (href, init) => new Promise((_, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }); + + const res = await get(`/q/${v.id}/stream`); + assert.equal(res.status, 502); + assert.match((await res.json()).error, /no answer within 50ms/); +}); + test('an unrecognised proxy path is 404 rather than forwarded somewhere', async () => { const v = await (await createView(alice, 'mine', [src(PUB, 'a')])).json(); const res = await get(`/q/${v.id}/admin/secrets`); @@ -526,6 +565,35 @@ test('views are readable without a signature', async () => { assert.equal(theirs.views.length, 0); }); +test('a requester can remove their own view, and only their own', async () => { + const v = await (await createView(alice, 'mine', [src(PUB, 'a')])).json(); + + // Neither a stranger nor an admin wallet can remove somebody else's view here — + // /views/remove answers to the requester alone. + assert.equal((await signedPost('/views/remove', bob, { id: v.id })).status, 403); + assert.equal((await signedPost('/views/remove', admin, { id: v.id })).status, 403); + assert.equal(viewCount(), 1); + + const ok = await signedPost('/views/remove', alice, { id: v.id }); + assert.equal(ok.status, 200); + assert.equal(viewCount(), 0); + // Off the watch list: the instance stops following that on-chain head. + assert.deepEqual((await (await get('/watchlist')).json()).sources, []); +}); + +test('removing a view needs a signature, and works without a subscription', async () => { + const v = await (await createView(alice, 'mine', [src(PUB, 'a')])).json(); + + const unsigned = await post('/views/remove', { address: alice.address, id: v.id }); + assert.equal(unsigned.status, 401); + assert.equal(viewCount(), 1); + + // A lapsed subscription must not trap a requester into being watched forever. + rpcResponse = () => accessResult(true, 0); + assert.equal((await signedPost('/views/remove', alice, { id: v.id })).status, 200); + assert.equal(viewCount(), 0); +}); + test('teardown is admin-only', async () => { const v = await (await createView(alice, 'mine', [src(PUB, 'a')])).json(); const denied = await signedPost('/admin/remove', alice, { id: v.id }); @@ -652,3 +720,45 @@ test('an unknown subdomain is 404, not a proxy to nowhere', async () => { const res = await worker.fetch(new Request('https://qb-dead-beef.qb.sond3r.com/search?q=x'), env); assert.equal(res.status, 404); }); + +/* ── watch-list caching ──────────────────────────────────────────────────── */ + +/** Count `list` operations from here on — the KV op with a 1000/day free-plan cap. */ +function countLists(kv) { + const real = kv.list.bind(kv); + let n = 0; + kv.list = async (opts) => { n += 1; return real(opts); }; + return () => n; +} + +test('POLLING IS A GET, NOT A LIST: repeated /watchlist lists KV once', async () => { + // Every instance polls /watchlist forever. Uncached this was one list operation per + // poll — 1440/day at --sources-refresh=60 against a free-plan ceiling of 1000, which + // is what took the real deployment over its quota. + await createView(alice, 'music', [src(PUB, 'tracks')]); + const lists = countLists(env.QUICKBEAM_KV); + + for (let i = 0; i < 5; i += 1) await get('/watchlist'); + + assert.equal(lists(), 1, 'the set must be rebuilt once, then served from the snapshot'); +}); + +test('a view created after the cache was warmed still reaches the watch list', async () => { + await createView(alice, 'music', [src(PUB, 'tracks')]); + await get('/watchlist'); // warm it + + await createView(bob, 'more', [src(PUB2, 'other')]); + + const { sources } = await (await get('/watchlist')).json(); + assert.equal(sources.length, 2, 'a stale snapshot would leave the new namespace unwatched'); +}); + +test('removing a view drops it from the watch list immediately', async () => { + const v = await (await createView(alice, 'music', [src(PUB, 'tracks')])).json(); + await get('/watchlist'); // warm it + + await signedPost('/admin/remove', admin, { id: v.id }); + + const { sources } = await (await get('/watchlist')).json(); + assert.deepEqual(sources, [], 'teardown must not wait out the snapshot TTL'); +}); diff --git a/quickbeam-registry/wrangler.toml b/quickbeam-registry/wrangler.toml index 459e73b..ecdfa1a 100644 --- a/quickbeam-registry/wrangler.toml +++ b/quickbeam-registry/wrangler.toml @@ -97,6 +97,16 @@ POLL_HINT_SECONDS = "60" SEARCH_URL = "http://qb.sond3r.com:8080" CDN_URL = "http://qb.sond3r.com:8090" +# How long the instance may take to ANSWER before a proxied request is a 502. Bounds +# the wait for headers only — /stream (SSE) and /export keep streaming afterwards for +# as long as they like. +# +# A box that is off, or a firewall that DROPs rather than refuses, black-holes the SYN: +# without this the request hangs until Cloudflare kills it near 100s, and the caller is +# told "the endpoint timed out" about the wrong hop entirely. 502 "Instance unreachable" +# in 15s names the real one. +INSTANCE_TIMEOUT_MS = "15000" + # ── App scoping ────────────────────────────────────────────────────────────── # A source is the whole app:publisher:subspace triple. Apps are stored canonically as # their 32-byte id (keccak256 of the name, the SDK's toAppId) — the website only ever