diff --git a/.changeset/media-endpoint-extensions.md b/.changeset/media-endpoint-extensions.md new file mode 100644 index 00000000..cf225dfe --- /dev/null +++ b/.changeset/media-endpoint-extensions.md @@ -0,0 +1,13 @@ +--- +"@dwk/micropub": minor +--- + +Implement the proposed media-endpoint extensions (#363, roadmap #354), gated +behind `extensions.proposed`: media `q=source` (newest-first listing and +by-URL lookup, `media` scope required), the `{ "url": ... }` upload response +body, and recoverable `action=delete`/`action=undelete` via an R2 `.trash/` +prefix with scope-pair enforcement and strict URL ownership validation. +Upload metadata is now always recorded in a new `micropub_media` D1 table +(best-effort while the group is off, fail-closed when on); the new +`mediaTrashRetentionDays` config (default 30) drives trash-row pruning, with +blob purge delegated to an R2 lifecycle rule. diff --git a/packages/micropub/CLAUDE.md b/packages/micropub/CLAUDE.md index 159d3fce..4e76313c 100644 --- a/packages/micropub/CLAUDE.md +++ b/packages/micropub/CLAUDE.md @@ -37,6 +37,11 @@ toggled by maturity group via the `extensions` config search over an injected `venues` D1 store, independent from post storage. `geo`'s reverse-geocoded suggestion is a placeholder (echoes the query coordinates) until a real lookup is wired in. +- **Media-endpoint extensions** (#363) — proposed-only media `q=source` + (listing + by-URL, `media` scope), `{ url }` upload response body, and + recoverable `action=delete`/`undelete` via an R2 `.trash/` prefix with + scope-pair enforcement. Upload metadata is always recorded in the + `micropub_media` D1 table (fail-closed only when the group is on). ## Spec @@ -80,6 +85,7 @@ src/mf2.ts # mf2 body parsing (form + JSON), update operations, source v src/pagination.ts # offset-based pagination parsing/validation (pure, reusable) src/source-filters.ts # proposed source-list filter and cursor parsing (pure) src/venues.ts # proposed q=geo venue store (D1) + query parsing +src/media.ts # proposed media-endpoint extensions: metadata store (D1), URL ownership validation, q=source parsing src/auth.ts # token extraction, scope checking, DPoP enforcement src/event.ts # h-event post type: markup rendering, h-event → CalendarEvent src/fediverse.ts # h-entry → PostInput adapter + syndication to @dwk/activitypub's /publish (#278; wire-format contract, no AP import) diff --git a/packages/micropub/README.md b/packages/micropub/README.md index 44087b3c..b54c0fa9 100644 --- a/packages/micropub/README.md +++ b/packages/micropub/README.md @@ -68,6 +68,13 @@ The handler fails loudly at startup if any of these are missing: filtered lists use deterministic keyset cursors. - **Opt-in Location/Venue** (`q=geo`): a read-only proximity search over an injected venue store, independent from post storage. See below. +- **Opt-in media-endpoint extensions**: with `extensions.proposed` on, the + media endpoint gains a `q=source` listing (newest-first, `media` scope + required) and by-URL lookup, a `{ "url": ... }` JSON body on upload, and + recoverable `action=delete`/`action=undelete` (requiring both the action + scope and `media`). Deleted blobs move to an R2 `.trash/` prefix retained + for `mediaTrashRetentionDays` (default 30); configure an R2 lifecycle rule + on that prefix to purge the bytes. ### Location/Venue (`q=geo`) extension diff --git a/packages/micropub/src/config.ts b/packages/micropub/src/config.ts index ba6bc541..937765a7 100644 --- a/packages/micropub/src/config.ts +++ b/packages/micropub/src/config.ts @@ -169,6 +169,14 @@ export interface MicropubConfig { readonly fediverse?: FediverseSyndicationConfig; /** Maximum accepted media upload size in bytes. Defaults to 25 MiB. */ readonly maxMediaBytes?: number; + /** + * Days soft-deleted media stays recoverable under the R2 `.trash/` prefix + * before `undelete` permanently fails. Defaults to 30. Purging the trash + * *bytes* is delegated to an R2 lifecycle rule the composed deployment + * configures on the prefix; this window only drives the opportunistic + * pruning of expired metadata rows (proposed media-endpoint extensions). + */ + readonly mediaTrashRetentionDays?: number; /** * Whether to check each token against the issued-token store (revocation). * Defaults to `true` — staleness here is a security bug, so the check hits the @@ -224,6 +232,7 @@ export interface ResolvedConfig { readonly syndicateTo: () => Promise; readonly fediverse?: FediverseSyndicationConfig; readonly maxMediaBytes: number; + readonly mediaTrashRetentionDays: number; readonly checkRevocation: boolean; readonly checkDpopReplay: boolean; readonly generatePostUrl: GeneratePostUrl; @@ -232,6 +241,7 @@ export interface ResolvedConfig { } const DEFAULT_MAX_MEDIA_BYTES = 25 * 1024 * 1024; +const DEFAULT_MEDIA_TRASH_RETENTION_DAYS = 30; /** Lowercase, dash-separated slug derived from arbitrary text (max 80 chars). */ function slugify(text: string): string { @@ -358,6 +368,8 @@ export function resolveConfig(config: MicropubConfig): ResolvedConfig { syndicateTo: normalizeSyndicateTo(config.syndicateTo), ...(config.fediverse ? { fediverse: config.fediverse } : {}), maxMediaBytes: config.maxMediaBytes ?? DEFAULT_MAX_MEDIA_BYTES, + mediaTrashRetentionDays: + config.mediaTrashRetentionDays ?? DEFAULT_MEDIA_TRASH_RETENTION_DAYS, checkRevocation: config.checkRevocation ?? true, checkDpopReplay: config.checkDpopReplay ?? true, generatePostUrl: diff --git a/packages/micropub/src/handler.ts b/packages/micropub/src/handler.ts index 20754b33..37d11a27 100644 --- a/packages/micropub/src/handler.ts +++ b/packages/micropub/src/handler.ts @@ -57,7 +57,16 @@ import { type MicropubVenueStore, type Venue, } from "./venues.js"; -import { authorize, tokenFromHeader, type AuthEnv } from "./auth.js"; +import { authorize, hasScope, tokenFromHeader, type AuthEnv } from "./auth.js"; +import { + createMicropubMediaStore, + MEDIA_EXTENSIONS, + MediaValidationError, + mediaKeyFromUrl, + mediaTrashKey, + parseMediaSourceParams, + type MediaRecord, +} from "./media.js"; import { syndicateEntry } from "./fediverse.js"; /** Cloudflare bindings required by the Micropub handler. */ @@ -351,50 +360,103 @@ async function foldUploadedMedia( for (const [key, values] of Object.entries(parsed.mf2.properties)) { properties[key] = [...values]; } - for (const [field, file] of files) { - if (file.size > config.maxMediaBytes) { - throw new Mf2ParseError( - `file "${file.name}" exceeds the ${config.maxMediaBytes}-byte limit`, - ); + const storedUrls: string[] = []; + try { + for (const [field, file] of files) { + if (file.size > config.maxMediaBytes) { + throw new Mf2ParseError( + `file "${file.name}" exceeds the ${config.maxMediaBytes}-byte limit`, + ); + } + const url = await storeMedia(file, env, config); + storedUrls.push(url); + const prop = field.endsWith("[]") ? field.slice(0, -2) : field; + (properties[prop] ??= []).push(url); } - const url = await storeMedia(file, env, config); - const prop = field.endsWith("[]") ? field.slice(0, -2) : field; - (properties[prop] ??= []).push(url); + } catch (err) { + // A later file failed after earlier ones committed. Their URLs were never + // returned to the client, so roll them back — otherwise a servable (and, + // with the proposed extensions on, `q=source`-listed) orphan blob would + // outlive a create that never happened. Best-effort: a blob that survives + // a failed rollback is at worst an unreferenced legacy blob. + const store = createMicropubMediaStore(env); + for (const url of storedUrls) { + const key = url.slice(config.mediaEndpoint.length + 1); + try { + await env.MEDIA.delete(key); + } catch { + // Best-effort. + } + try { + await store.remove(key); + } catch { + // Best-effort. + } + } + throw err; } return { ...parsed, mf2: { type: parsed.mf2.type, properties } }; } // --- Media ------------------------------------------------------------------ -const EXTENSIONS: Record = { - "image/jpeg": ".jpg", - "image/png": ".png", - "image/gif": ".gif", - "image/webp": ".webp", - "image/avif": ".avif", - "video/mp4": ".mp4", - "audio/mpeg": ".mp3", -}; - /** * Content types safe to serve inline with their declared type — the media this * endpoint exists for. Anything else (notably `text/html`, `image/svg+xml`, …) * is served as an opaque `application/octet-stream` attachment so a `media`-scope * client cannot upload active content that renders as stored XSS on this origin. */ -const SAFE_INLINE_TYPES = new Set(Object.keys(EXTENSIONS)); +const SAFE_INLINE_TYPES = new Set(Object.keys(MEDIA_EXTENSIONS)); + +/** + * The `micropub_media` metadata insert failed after a successful R2 write + * while the proposed media extensions are on — the row is load-bearing for + * `q=source`, so the fresh blob was rolled back and the request must fail. + */ +class MediaMetadataError extends Error {} -/** Stream a file to R2 under a random key and return its public media URL. */ +/** + * Stream a file to R2 under a random key and return its public media URL. + * + * Every stored blob also gets a `micropub_media` metadata row, regardless of + * `extensions.proposed` (invisible to clients; avoids a listing history gap + * when a deployment later opts in). Failure handling splits by enablement: + * enabled, the row is load-bearing for `q=source`, so an insert failure rolls + * the blob back and throws {@link MediaMetadataError}; disabled, the insert + * is best-effort — the upload keeps today's R2-write-succeeds guarantee and + * the unrecorded blob simply behaves as a legacy blob later. + */ async function storeMedia( file: File, env: MicropubEnv, config: ResolvedConfig, ): Promise { - const ext = EXTENSIONS[file.type] ?? ""; + const ext = MEDIA_EXTENSIONS[file.type] ?? ""; const key = `${crypto.randomUUID()}${ext}`; + const contentType = file.type || "application/octet-stream"; await env.MEDIA.put(key, file.stream(), { - httpMetadata: { contentType: file.type || "application/octet-stream" }, + httpMetadata: { contentType }, }); + try { + await createMicropubMediaStore(env).record({ + key, + contentType, + sizeBytes: file.size, + now: Math.floor(Date.now() / 1000), + }); + } catch (err) { + if (config.extensions.proposed) { + try { + await env.MEDIA.delete(key); + } catch { + // Best-effort rollback; the orphaned blob is unlisted either way. + } + throw new MediaMetadataError( + `failed to record media metadata: ${err instanceof Error ? err.message : String(err)}`, + ); + } + emit(config, "warn", MicropubLogEvent.MediaMetadataFailed, {}); + } return `${config.mediaEndpoint}/${key}`; } @@ -466,21 +528,284 @@ async function handleMediaUpload( 413, ); } - const url = await storeMedia(file, env, config); + let url: string; + try { + url = await storeMedia(file, env, config); + } catch (err) { + if (err instanceof MediaMetadataError) { + return error("server_error", err.message, 500); + } + throw err; + } emit(config, "info", MicropubLogEvent.MediaStored, { contentType: file.type || "application/octet-stream", }); + if (config.extensions.proposed) { + await pruneExpiredMediaRows(env, config); + // Upstream Response-from-Media-Endpoint minimum: mirror the authoritative + // `Location` header in a JSON body so clients need not read headers. + return new Response(JSON.stringify({ url }), { + status: 201, + headers: { + location: url, + "content-type": "application/json", + ...CORS_HEADERS, + }, + }); + } return new Response(null, { status: 201, headers: { location: url, ...CORS_HEADERS }, }); } +/** + * Opportunistically drop metadata rows whose trash retention has passed. The + * blob bytes are purged by the deployment's R2 lifecycle rule on the trash + * prefix; expired rows are excluded from every response either way, so this + * is hygiene, never correctness — failures are swallowed. + */ +async function pruneExpiredMediaRows( + env: MicropubEnv, + config: ResolvedConfig, +): Promise { + const cutoff = + Math.floor(Date.now() / 1000) - config.mediaTrashRetentionDays * 86400; + try { + await createMicropubMediaStore(env).pruneExpired(cutoff); + } catch { + // Hygiene only. + } +} + +/** The interop-consensus media `q=source` item shape. */ +function mediaView( + record: MediaRecord, + config: ResolvedConfig, +): Record { + return { + url: `${config.mediaEndpoint}/${record.key}`, + published: new Date(record.uploadedAt * 1000) + .toISOString() + .replace(/\.\d{3}Z$/, "Z"), + mime_type: record.contentType, + }; +} + +/** + * Handle `GET` to the media endpoint (proposed extensions only): `q=source` + * as a newest-first listing or a single-file lookup by `url`. Requires the + * `media` scope — deliberately unlike the post endpoint's scope-less + * `q=source`, because the listing enumerates every upload, including media + * attached to draft, unlisted, or private posts. + */ +async function handleMediaQuery( + request: Request, + env: MicropubEnv, + config: ResolvedConfig, +): Promise { + const auth = await authorize( + request, + env, + config, + tokenFromHeader(request), + ["media"], + config.mediaEndpoint, + ); + if (!auth.ok) { + emit(config, "warn", MicropubLogEvent.AuthRejected, { + reason: auth.error, + status: auth.status, + }); + return error(auth.error, auth.description, auth.status); + } + + const params = new URL(request.url).searchParams; + const q = params.get("q"); + if (q !== "source") { + emit(config, "warn", MicropubLogEvent.RequestRejected, { + reason: "query_unsupported", + }); + return error("invalid_request", `unsupported query \`q=${q ?? ""}\``, 400); + } + + let query; + try { + query = parseMediaSourceParams(params); + } catch (err) { + if (err instanceof MediaValidationError) { + return error("invalid_request", err.message, 400); + } + throw err; + } + + const store = createMicropubMediaStore(env); + if ("url" in query) { + const key = mediaKeyFromUrl(query.url, config.mediaEndpoint); + if (!key) { + return error( + "invalid_request", + "`url` does not name media owned by this endpoint", + 400, + ); + } + const record = await store.get(key); + if (!record || record.deletedAt !== null) { + // Body-code/status divergence per the missing-post convention in + // spec/packages/micropub.md "Error responses". + return error("invalid_request", "no media exists at that URL", 404); + } + return json(mediaView(record, config)); + } + const records = await store.list(query.page); + return json({ items: records.map((record) => mediaView(record, config)) }); +} + +/** + * Handle a non-multipart `POST` to the media endpoint (proposed extensions + * only): `action=delete` — a recoverable soft delete via the R2 trash prefix + * — and the package-defined symmetric `action=undelete`. Each action requires + * **both** its action scope and `media`: a `media`-only uploader token cannot + * destroy media, and a `delete`-only post token cannot touch the media + * endpoint. + */ +async function handleMediaAction( + request: Request, + env: MicropubEnv, + config: ResolvedConfig, +): Promise { + const auth = await authorize( + request, + env, + config, + tokenFromHeader(request), + ["media"], + config.mediaEndpoint, + ); + if (!auth.ok) { + emit(config, "warn", MicropubLogEvent.AuthRejected, { + reason: auth.error, + status: auth.status, + }); + return error(auth.error, auth.description, auth.status); + } + + const contentType = request.headers.get("content-type") ?? ""; + let action: string | undefined; + let url: string | undefined; + if (contentType.includes("application/json")) { + let body: unknown; + try { + body = await request.json(); + } catch { + return error("invalid_request", "request body is not valid JSON", 400); + } + if (body && typeof body === "object") { + const record = body as Record; + if (typeof record.action === "string") action = record.action; + if (typeof record.url === "string") url = record.url; + } + } else { + for (const [key, value] of await readForm(request)) { + if (key === "action") action ??= value; + if (key === "url") url ??= value; + } + } + + if (action !== "delete" && action !== "undelete") { + emit(config, "warn", MicropubLogEvent.RequestRejected, { + reason: "action_unsupported", + }); + return error( + "invalid_request", + `unsupported media action \`${action ?? ""}\``, + 400, + ); + } + if (!hasScope(auth.claims.scope, [action])) { + emit(config, "warn", MicropubLogEvent.AuthRejected, { + reason: "insufficient_scope", + status: 403, + }); + return error( + "insufficient_scope", + `media \`${action}\` requires both the \`${action}\` and \`media\` scopes`, + 403, + ); + } + if (!url) { + return error( + "invalid_request", + `\`url\` is required for \`${action}\``, + 400, + ); + } + // Load-bearing ownership validation: reject anything that is not exactly + // this endpoint's single-segment generator-format key — before any storage + // access. + const key = mediaKeyFromUrl(url, config.mediaEndpoint); + if (!key) { + return error( + "invalid_request", + "`url` does not name media owned by this endpoint", + 400, + ); + } + + const store = createMicropubMediaStore(env); + const now = Math.floor(Date.now() / 1000); + await pruneExpiredMediaRows(env, config); + const trashKey = mediaTrashKey(key); + + if (action === "delete") { + const live = await env.MEDIA.head(key); + if (!live) { + // Already deleted (recoverable) or never stored: both are 404. + return error("invalid_request", "no media exists at that URL", 404); + } + // The live blob is never removed before the trash copy is durable, so no + // partial failure can lose the bytes; a resumed delete (both blobs + // present) skips straight to the removal. + if (!(await env.MEDIA.head(trashKey))) { + const body = await env.MEDIA.get(key); + if (body) { + await env.MEDIA.put(trashKey, body.body, { + httpMetadata: body.httpMetadata, + }); + } + } + await env.MEDIA.delete(key); + await store.setDeleted(key, now); + emit(config, "info", MicropubLogEvent.MediaDeleted, {}); + return noContent(); + } + + const trash = await env.MEDIA.get(trashKey); + if (!trash) { + // After the retention purge the deletion is permanent. + return error( + "invalid_request", + "no recoverable media exists at that URL", + 404, + ); + } + await env.MEDIA.put(key, trash.body, { httpMetadata: trash.httpMetadata }); + await env.MEDIA.delete(trashKey); + await store.setDeleted(key, null); + emit(config, "info", MicropubLogEvent.MediaUndeleted, {}); + return noContent(); +} + /** Serve a previously uploaded media blob from R2 (public, unauthenticated). */ async function handleMediaGet( key: string, env: MicropubEnv, ): Promise { + // Only single-segment generator-format keys are public; in particular the + // recoverable-delete trash prefix (`.trash/`) must never be servable. + if (!key || key.includes("/")) { + return new Response("Not Found", { status: 404 }); + } const object = await env.MEDIA.get(key); if (!object) return new Response("Not Found", { status: 404 }); const headers = new Headers(CORS_HEADERS); @@ -606,6 +931,12 @@ async function handleQuery( ? { properties: ["audience", "location-visibility"], audiences: config.audiences, + // Package-defined advertisement (the upstream media proposals + // define none) so clients can feature-detect without probing. + "media-endpoint-extensions": { + q: ["source"], + actions: ["delete", "undelete"], + }, "source-filters": { after: "whole-second RFC3339 exclusive creation-time lower bound", before: @@ -978,6 +1309,9 @@ async function handleAction( }); return error("invalid_request", err.message, 400); } + if (err instanceof MediaMetadataError) { + return error("server_error", err.message, 500); + } throw err; } } @@ -1283,9 +1617,25 @@ export function createMicropub(config: MicropubConfig): MicropubHandler { return new Response(null, { status: 204, headers: CORS_HEADERS }); } - // Media endpoint: POST uploads, GET serves a blob under `${mediaPath}/`. + // Media endpoint: POST uploads (and, with the proposed extensions on, + // form/JSON `action=delete`/`undelete` plus GET `q=source`), GET serves a + // blob under `${mediaPath}/`. if (pathname === resolved.mediaPath) { - if (method !== "POST") return methodNotAllowed("POST, OPTIONS"); + if (method === "GET" && resolved.extensions.proposed) { + return handleMediaQuery(request, env, resolved); + } + if (method !== "POST") { + return methodNotAllowed( + resolved.extensions.proposed ? "GET, POST, OPTIONS" : "POST, OPTIONS", + ); + } + const contentType = request.headers.get("content-type") ?? ""; + if ( + resolved.extensions.proposed && + !contentType.toLowerCase().includes("multipart/form-data") + ) { + return handleMediaAction(request, env, resolved); + } return handleMediaUpload(request, env, resolved); } if (pathname.startsWith(`${resolved.mediaPath}/`)) { diff --git a/packages/micropub/src/index.test.ts b/packages/micropub/src/index.test.ts index 98bf7cbc..66db90ff 100644 --- a/packages/micropub/src/index.test.ts +++ b/packages/micropub/src/index.test.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it } from "vitest"; import { createMicropub, createMicropubContactStore, + createMicropubMediaStore, createMicropubVenueStore, } from "./index.js"; import type { MicropubEnv } from "./index.js"; @@ -2325,3 +2326,503 @@ describe("@dwk/micropub proposed venues (q=geo)", () => { expect(res.status).toBe(400); }); }); + +// --- Proposed media-endpoint extensions --------------------------------------- + +describe("@dwk/micropub media-endpoint extensions", () => { + const mediaExt = createMicropub({ + baseUrl: BASE, + me: ME, + extensions: { proposed: true }, + }); + const mediaStore = createMicropubMediaStore(harness); + + beforeEach(async () => { + await mediaStore.init(); + await harness.MICROPUB_DB.prepare("DELETE FROM micropub_media").run(); + // Empty the R2 bucket so listing/trash assertions are isolated. + const listed = await harness.MEDIA.list(); + for (const object of listed.objects) await harness.MEDIA.delete(object.key); + }); + + async function upload( + handlerFn: typeof handler, + bytes: number[] = [1, 2, 3], + type = "image/png", + ): Promise { + const minted = await mintToken("media"); + const form = new FormData(); + form.set("file", new File([new Uint8Array(bytes)], "f", { type })); + return handlerFn( + new Request(MEDIA, { + method: "POST", + headers: await authHeaders(minted, "POST", MEDIA), + body: form, + }), + harness, + ctx, + ); + } + + async function mediaAction( + scope: string, + action: string, + url: string, + json = false, + ): Promise { + const minted = await mintToken(scope); + const headers = await authHeaders(minted, "POST", MEDIA); + const body = json + ? JSON.stringify({ action, url }) + : new URLSearchParams({ action, url }).toString(); + return mediaExt( + new Request(MEDIA, { + method: "POST", + headers: { + ...headers, + "content-type": json + ? "application/json" + : "application/x-www-form-urlencoded", + }, + body, + }), + harness, + ctx, + ); + } + + async function mediaQuery( + scope: string, + query: string, + handlerFn: typeof handler = mediaExt, + ): Promise { + const minted = await mintToken(scope); + return handlerFn( + new Request(`${MEDIA}?${query}`, { + headers: await authHeaders(minted, "GET", MEDIA), + }), + harness, + ctx, + ); + } + + // --- Disabled path stays byte-identical ------------------------------------ + + it("keeps action=delete uninterpreted when the proposed group is off", async () => { + const uploaded = await upload(handler); + const url = uploaded.headers.get("location")!; + const minted = await mintToken("delete media"); + const res = await handler( + new Request(MEDIA, { + method: "POST", + headers: { + ...(await authHeaders(minted, "POST", MEDIA)), + "content-type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ action: "delete", url }).toString(), + }), + harness, + ctx, + ); + // Falls through to the existing missing-file 400; the blob survives. + expect(res.status).toBe(400); + expect((await handler(new Request(url), harness, ctx)).status).toBe(200); + }); + + it("keeps GET of the media endpoint a 405 when the proposed group is off", async () => { + const res = await mediaQuery("media", "q=source", handler); + expect(res.status).toBe(405); + }); + + it("keeps the upload response body empty when the proposed group is off", async () => { + const res = await upload(handler); + expect(res.status).toBe(201); + expect(await res.text()).toBe(""); + }); + + it("omits the media-endpoint-extensions member from q=config by default", async () => { + const minted = await mintToken("create"); + const res = await handler( + new Request(`${MICROPUB}?q=config`, { + headers: await authHeaders(minted, "GET", MICROPUB), + }), + harness, + ctx, + ); + const body = (await res.json()) as Record; + expect(body["media-endpoint-extensions"]).toBeUndefined(); + }); + + it("records upload metadata even when the proposed group is off", async () => { + const res = await upload(handler); + const key = res.headers.get("location")!.slice(`${MEDIA}/`.length); + const record = await mediaStore.get(key); + expect(record?.contentType).toBe("image/png"); + expect(record?.sizeBytes).toBe(3); + }); + + it("still 201s a disabled-path upload when the metadata insert fails", async () => { + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + await harness.MICROPUB_DB.prepare( + `CREATE TABLE micropub_media ( + key TEXT PRIMARY KEY CHECK (0), + content_type TEXT, size_bytes INTEGER, uploaded_at INTEGER, + deleted_at INTEGER + )`, + ).run(); + try { + const res = await upload(handler); + // Today's guarantee: a successful R2 write means a successful upload. + expect(res.status).toBe(201); + const url = res.headers.get("location")!; + expect((await handler(new Request(url), harness, ctx)).status).toBe(200); + } finally { + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + // A fresh store instance: the shared one memoizes its schema-ready + // promise, so its init() would no-op against the dropped table. + await createMicropubMediaStore(harness).init(); + } + }); + + it("500s an enabled-path upload and removes the blob when the metadata insert fails", async () => { + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + await harness.MICROPUB_DB.prepare( + `CREATE TABLE micropub_media ( + key TEXT PRIMARY KEY CHECK (0), + content_type TEXT, size_bytes INTEGER, uploaded_at INTEGER, + deleted_at INTEGER + )`, + ).run(); + try { + const res = await upload(mediaExt); + // A URL handed to a client must always be both servable and listed. + expect(res.status).toBe(500); + expect((await harness.MEDIA.list()).objects.length).toBe(0); + } finally { + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + // A fresh store instance: the shared one memoizes its schema-ready + // promise, so its init() would no-op against the dropped table. + await createMicropubMediaStore(harness).init(); + } + }); + + // --- Enabled: advertisement and upload body -------------------------------- + + it("advertises the media-endpoint extensions in q=config when enabled", async () => { + const minted = await mintToken("create"); + const res = await mediaExt( + new Request(`${MICROPUB}?q=config`, { + headers: await authHeaders(minted, "GET", MICROPUB), + }), + harness, + ctx, + ); + const body = (await res.json()) as Record; + expect(body["media-endpoint-extensions"]).toEqual({ + q: ["source"], + actions: ["delete", "undelete"], + }); + }); + + it("adds the { url } JSON body to an enabled upload", async () => { + const res = await upload(mediaExt); + expect(res.status).toBe(201); + const location = res.headers.get("location")!; + expect((await res.json()) as { url: string }).toEqual({ url: location }); + }); + + // --- Enabled: q=source at the media endpoint -------------------------------- + + it("lists uploads newest-first with url, published, and mime_type", async () => { + const first = await upload(mediaExt, [1], "image/png"); + const second = await upload(mediaExt, [2, 2], "image/jpeg"); + const firstUrl = first.headers.get("location")!; + const secondUrl = second.headers.get("location")!; + // Force distinct upload times for a deterministic order. + await harness.MICROPUB_DB.prepare( + "UPDATE micropub_media SET uploaded_at = uploaded_at + 10 WHERE key = ?", + ) + .bind(secondUrl.slice(`${MEDIA}/`.length)) + .run(); + + const res = await mediaQuery("media", "q=source"); + expect(res.status).toBe(200); + const body = (await res.json()) as { + items: { url: string; published: string; mime_type: string }[]; + }; + expect(body.items.map((item) => item.url)).toEqual([secondUrl, firstUrl]); + expect(body.items[0]!.mime_type).toBe("image/jpeg"); + // RFC 3339 with no fractional seconds. + expect(body.items[0]!.published).toMatch( + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/, + ); + }); + + it("paginates the media listing with limit and offset", async () => { + await upload(mediaExt, [1]); + await upload(mediaExt, [2]); + const res = await mediaQuery("media", "q=source&limit=1&offset=1"); + const body = (await res.json()) as { items: unknown[] }; + expect(body.items.length).toBe(1); + }); + + it("requires the media scope for the media listing", async () => { + const res = await mediaQuery("create", "q=source"); + expect(res.status).toBe(403); + expect(((await res.json()) as { error: string }).error).toBe( + "insufficient_scope", + ); + }); + + it("rejects unknown media q=source parameters", async () => { + const res = await mediaQuery("media", "q=source&nope=1"); + expect(res.status).toBe(400); + }); + + it("400s an unknown q at the enabled media endpoint", async () => { + const res = await mediaQuery("media", "q=bogus"); + expect(res.status).toBe(400); + }); + + it("returns a single media object by url", async () => { + const uploaded = await upload(mediaExt, [7, 7], "image/png"); + const url = uploaded.headers.get("location")!; + const res = await mediaQuery( + "media", + `q=source&url=${encodeURIComponent(url)}`, + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { url: string; mime_type: string }; + expect(body.url).toBe(url); + expect(body.mime_type).toBe("image/png"); + }); + + it("400s a by-url query for a foreign URL and 404s an unknown one", async () => { + const foreign = await mediaQuery( + "media", + `q=source&url=${encodeURIComponent("https://evil.example.net/media/x")}`, + ); + expect(foreign.status).toBe(400); + const unknown = await mediaQuery( + "media", + `q=source&url=${encodeURIComponent( + `${MEDIA}/6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d.jpg`, + )}`, + ); + expect(unknown.status).toBe(404); + }); + + // --- Enabled: action=delete / action=undelete ------------------------------- + + it("requires both delete and media scopes for action=delete", async () => { + const uploaded = await upload(mediaExt); + const url = uploaded.headers.get("location")!; + for (const scope of ["media", "delete"]) { + const res = await mediaAction(scope, "delete", url); + expect(res.status).toBe(403); + expect(((await res.json()) as { error: string }).error).toBe( + "insufficient_scope", + ); + } + }); + + it("soft-deletes media into the trash prefix and undeletes it back", async () => { + const uploaded = await upload(mediaExt, [5, 5, 5], "image/png"); + const url = uploaded.headers.get("location")!; + const key = url.slice(`${MEDIA}/`.length); + + const del = await mediaAction("delete media", "delete", url); + expect(del.status).toBe(204); + // Live blob gone, trash copy durable, listing empty, public GET 404. + expect(await harness.MEDIA.get(key)).toBeNull(); + expect(await harness.MEDIA.get(`.trash/${key}`)).not.toBeNull(); + expect((await mediaExt(new Request(url), harness, ctx)).status).toBe(404); + const listing = (await (await mediaQuery("media", "q=source")).json()) as { + items: unknown[]; + }; + expect(listing.items.length).toBe(0); + + // Deleting already-deleted media is 404 (recoverable ≠ addressable). + expect((await mediaAction("delete media", "delete", url)).status).toBe(404); + + const undel = await mediaAction("undelete media", "undelete", url); + expect(undel.status).toBe(204); + expect(await harness.MEDIA.get(key)).not.toBeNull(); + expect(await harness.MEDIA.get(`.trash/${key}`)).toBeNull(); + const relisted = (await (await mediaQuery("media", "q=source")).json()) as { + items: { url: string }[]; + }; + expect(relisted.items.map((item) => item.url)).toEqual([url]); + }); + + it("accepts a JSON action=delete body", async () => { + const uploaded = await upload(mediaExt); + const url = uploaded.headers.get("location")!; + const res = await mediaAction("delete media", "delete", url, true); + expect(res.status).toBe(204); + }); + + it("404s undelete after the trash copy is gone (post-purge permanence)", async () => { + const uploaded = await upload(mediaExt); + const url = uploaded.headers.get("location")!; + const key = url.slice(`${MEDIA}/`.length); + await mediaAction("delete media", "delete", url); + await harness.MEDIA.delete(`.trash/${key}`); + const res = await mediaAction("undelete media", "undelete", url); + expect(res.status).toBe(404); + }); + + it("resumes a mid-failure delete where the trash copy already exists", async () => { + const uploaded = await upload(mediaExt, [9], "image/png"); + const url = uploaded.headers.get("location")!; + const key = url.slice(`${MEDIA}/`.length); + // Simulate a crash between the trash copy and the live delete. + await harness.MEDIA.put(`.trash/${key}`, new Uint8Array([9])); + const res = await mediaAction("delete media", "delete", url); + expect(res.status).toBe(204); + expect(await harness.MEDIA.get(key)).toBeNull(); + expect(await harness.MEDIA.get(`.trash/${key}`)).not.toBeNull(); + }); + + it("rejects delete URLs outside the media endpoint before touching storage", async () => { + for (const url of [ + "https://evil.example.net/media/6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d.jpg", + `${MEDIA}/.trash/6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d.jpg`, + `${MEDIA}/../secret`, + "not-a-url", + ]) { + const res = await mediaAction("delete media", "delete", url); + expect(res.status).toBe(400); + } + }); + + it("404s a delete of never-uploaded media", async () => { + const res = await mediaAction( + "delete media", + "delete", + `${MEDIA}/6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d.jpg`, + ); + expect(res.status).toBe(404); + }); + + it("rejects an unknown media action", async () => { + const res = await mediaAction("delete media", "destroy", `${MEDIA}/x`); + expect(res.status).toBe(400); + }); + + it("never serves the trash prefix from the public GET route", async () => { + const uploaded = await upload(mediaExt, [3], "image/png"); + const url = uploaded.headers.get("location")!; + const key = url.slice(`${MEDIA}/`.length); + await mediaAction("delete media", "delete", url); + const res = await mediaExt( + new Request(`${MEDIA}/.trash/${key}`), + harness, + ctx, + ); + expect(res.status).toBe(404); + }); + + it("records metadata for files folded out of a multipart create", async () => { + const minted = await mintToken("create"); + const form = new FormData(); + form.set("h", "entry"); + form.set("content", "with a photo"); + form.set( + "photo", + new File([new Uint8Array([9, 9])], "p.jpg", { type: "image/jpeg" }), + ); + const res = await mediaExt( + new Request(MICROPUB, { + method: "POST", + headers: await authHeaders(minted, "POST", MICROPUB), + body: form, + }), + harness, + ctx, + ); + expect(res.status).toBe(201); + const listing = (await (await mediaQuery("media", "q=source")).json()) as { + items: { mime_type: string }[]; + }; + expect(listing.items.map((item) => item.mime_type)).toEqual(["image/jpeg"]); + }); + + it("prunes expired trash rows during media-endpoint writes", async () => { + const uploaded = await upload(mediaExt); + const url = uploaded.headers.get("location")!; + const key = url.slice(`${MEDIA}/`.length); + await mediaAction("delete media", "delete", url); + // Age the deletion far past the 30-day retention default. + await harness.MICROPUB_DB.prepare( + "UPDATE micropub_media SET deleted_at = 1 WHERE key = ?", + ) + .bind(key) + .run(); + await upload(mediaExt); + expect(await mediaStore.get(key)).toBeNull(); + }); + + it("requires both undelete and media scopes for action=undelete", async () => { + const uploaded = await upload(mediaExt); + const url = uploaded.headers.get("location")!; + await mediaAction("delete media", "delete", url); + for (const scope of ["media", "undelete"]) { + const res = await mediaAction(scope, "undelete", url); + expect(res.status).toBe(403); + expect(((await res.json()) as { error: string }).error).toBe( + "insufficient_scope", + ); + } + }); + + it("rolls back earlier files when a later fold's metadata insert fails", async () => { + // Sabotage: the `.png` key (first file) inserts fine, the `.jpg` key + // (second file) trips the CHECK — so the failure hits mid-loop with one + // blob and row already committed. + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + await harness.MICROPUB_DB.prepare( + `CREATE TABLE micropub_media ( + key TEXT PRIMARY KEY CHECK (key NOT LIKE '%.jpg'), + content_type TEXT, size_bytes INTEGER, uploaded_at INTEGER, + deleted_at INTEGER + )`, + ).run(); + try { + const minted = await mintToken("create"); + const form = new FormData(); + form.set("h", "entry"); + form.set("content", "two photos"); + form.append( + "photo[]", + new File([new Uint8Array([1])], "a.png", { type: "image/png" }), + ); + form.append( + "photo[]", + new File([new Uint8Array([2])], "b.jpg", { type: "image/jpeg" }), + ); + const res = await mediaExt( + new Request(MICROPUB, { + method: "POST", + headers: await authHeaders(minted, "POST", MICROPUB), + body: form, + }), + harness, + ctx, + ); + // The create fails closed — and no blob or row from the failed request + // survives, so nothing servable/listed outlives a post never created. + expect(res.status).toBe(500); + expect((await harness.MEDIA.list()).objects.length).toBe(0); + const rows = await harness.MICROPUB_DB.prepare( + "SELECT COUNT(*) AS n FROM micropub_media", + ).first<{ n: number }>(); + expect(rows?.n).toBe(0); + } finally { + await harness.MICROPUB_DB.prepare("DROP TABLE micropub_media").run(); + // A fresh store instance: the shared one memoizes its schema-ready + // promise, so its init() would no-op against the dropped table. + await createMicropubMediaStore(harness).init(); + } + }); +}); diff --git a/packages/micropub/src/index.ts b/packages/micropub/src/index.ts index 65de94ff..9c25eb11 100644 --- a/packages/micropub/src/index.ts +++ b/packages/micropub/src/index.ts @@ -149,3 +149,14 @@ export { type VenueSearchQuery, type VenueStoreEnv, } from "./venues.js"; + +export { + createMicropubMediaStore, + mediaKeyFromUrl, + parseMediaSourceParams, + MediaValidationError, + type MediaRecord, + type MediaSourceQuery, + type MediaStoreEnv, + type MicropubMediaStore, +} from "./media.js"; diff --git a/packages/micropub/src/log.ts b/packages/micropub/src/log.ts index dd4b5135..a03a0f2f 100644 --- a/packages/micropub/src/log.ts +++ b/packages/micropub/src/log.ts @@ -34,6 +34,20 @@ export const MicropubLogEvent = { ActionCompleted: "micropub.action.completed", /** A media upload was streamed to R2. */ MediaStored: "micropub.media.stored", + /** + * A media blob was soft-deleted into the trash prefix + * (proposed media-endpoint extensions). + */ + MediaDeleted: "micropub.media.deleted", + /** A soft-deleted media blob was restored from the trash prefix. */ + MediaUndeleted: "micropub.media.undeleted", + /** + * The `micropub_media` metadata insert failed after a successful R2 write + * while the proposed group is off, so the upload still returned `201` and + * the blob is unrecorded (it behaves as a legacy blob if the group is later + * enabled). + */ + MediaMetadataFailed: "micropub.media.metadata.failed", } as const; /** Union of the event-name string literals in {@link MicropubLogEvent}. */ diff --git a/packages/micropub/src/media.test.ts b/packages/micropub/src/media.test.ts new file mode 100644 index 00000000..fa22a0ae --- /dev/null +++ b/packages/micropub/src/media.test.ts @@ -0,0 +1,205 @@ +import { env } from "cloudflare:test"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { + MediaValidationError, + createMicropubMediaStore, + mediaKeyFromUrl, + mediaTrashKey, + parseMediaSourceParams, +} from "./media.js"; +import type { MediaStoreEnv } from "./media.js"; + +const harness = env as unknown as MediaStoreEnv; + +const MEDIA = "https://example.com/media"; +const KEY = "6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d.jpg"; +const BARE_KEY = "6d9f2c3a-1b4e-4f5a-8c7d-0e1f2a3b4c5d"; + +// --- URL ownership validation ------------------------------------------------ + +describe("mediaKeyFromUrl", () => { + it("accepts a canonical media URL with a known extension", () => { + expect(mediaKeyFromUrl(`${MEDIA}/${KEY}`, MEDIA)).toBe(KEY); + }); + + it("accepts a canonical media URL with no extension", () => { + expect(mediaKeyFromUrl(`${MEDIA}/${BARE_KEY}`, MEDIA)).toBe(BARE_KEY); + }); + + it("canonicalizes a default port to the same origin", () => { + expect(mediaKeyFromUrl(`https://example.com:443/media/${KEY}`, MEDIA)).toBe( + KEY, + ); + }); + + it.each([ + ["a foreign origin", `https://evil.example.net/media/${KEY}`], + ["a different path prefix", `https://example.com/other/${KEY}`], + ["the media endpoint itself", MEDIA], + ["a trailing extra segment", `${MEDIA}/${KEY}/extra`], + ["a traversal segment", `${MEDIA}/../secret/${KEY}`], + ["an encoded slash in the key", `${MEDIA}/${BARE_KEY}%2Fescape`], + ["an encoded traversal", `${MEDIA}/%2E%2E/${KEY}`], + ["the trash prefix", `${MEDIA}/.trash/${KEY}`], + ["a query string", `${MEDIA}/${KEY}?x=1`], + ["a fragment", `${MEDIA}/${KEY}#frag`], + ["an uppercase UUID", `${MEDIA}/${BARE_KEY.toUpperCase()}`], + ["an unknown extension", `${MEDIA}/${BARE_KEY}.html`], + ["a non-UUID key", `${MEDIA}/notauuid.jpg`], + ["a relative URL", `/media/${KEY}`], + ["a non-http scheme", `ftp://example.com/media/${KEY}`], + ])("rejects %s", (_label, url) => { + expect(mediaKeyFromUrl(url, MEDIA)).toBeNull(); + }); +}); + +describe("mediaTrashKey", () => { + it("prefixes the key with the trash namespace", () => { + expect(mediaTrashKey(KEY)).toBe(`.trash/${KEY}`); + }); +}); + +// --- `q=source` (media) parameter parsing ------------------------------------- + +function params(query: string): URLSearchParams { + return new URL(`${MEDIA}?${query}`).searchParams; +} + +describe("parseMediaSourceParams", () => { + it("defaults to the first page of ten", () => { + expect(parseMediaSourceParams(params("q=source"))).toEqual({ + page: { limit: 10, offset: 0 }, + }); + }); + + it("parses explicit limit and offset", () => { + expect( + parseMediaSourceParams(params("q=source&limit=5&offset=20")), + ).toEqual({ page: { limit: 5, offset: 20 } }); + }); + + it("caps limit at 100", () => { + expect(parseMediaSourceParams(params("q=source&limit=500"))).toEqual({ + page: { limit: 100, offset: 0 }, + }); + }); + + it("returns the url form when `url` is present", () => { + expect( + parseMediaSourceParams(params(`q=source&url=${MEDIA}/${KEY}`)), + ).toEqual({ url: `${MEDIA}/${KEY}` }); + }); + + it.each([ + ["an unknown parameter", "q=source&nope=1"], + ["a repeated url", `q=source&url=${MEDIA}/a&url=${MEDIA}/b`], + ["a repeated limit", "q=source&limit=1&limit=2"], + ["url combined with limit", `q=source&url=${MEDIA}/${KEY}&limit=5`], + ["url combined with offset", `q=source&url=${MEDIA}/${KEY}&offset=5`], + ["a non-numeric limit", "q=source&limit=abc"], + ["a zero limit", "q=source&limit=0"], + ["a negative offset", "q=source&offset=-1"], + ["a fractional limit", "q=source&limit=1.5"], + ])("rejects %s", (_label, query) => { + expect(() => parseMediaSourceParams(params(query))).toThrow( + MediaValidationError, + ); + }); +}); + +// --- D1 media metadata store -------------------------------------------------- + +describe("createMicropubMediaStore", () => { + const store = createMicropubMediaStore(harness); + + beforeEach(async () => { + await store.init(); + await harness.MICROPUB_DB.prepare("DELETE FROM micropub_media").run(); + }); + + it("records an upload and reads it back", async () => { + await store.record({ + key: KEY, + contentType: "image/jpeg", + sizeBytes: 123, + now: 1000, + }); + expect(await store.get(KEY)).toEqual({ + key: KEY, + contentType: "image/jpeg", + sizeBytes: 123, + uploadedAt: 1000, + deletedAt: null, + }); + }); + + it("returns null for an unknown key", async () => { + expect(await store.get("missing")).toBeNull(); + }); + + it("lists live media newest-first with key as the tie-breaker", async () => { + await store.record({ key: "b", contentType: "t", sizeBytes: 1, now: 100 }); + await store.record({ key: "a", contentType: "t", sizeBytes: 1, now: 100 }); + await store.record({ key: "c", contentType: "t", sizeBytes: 1, now: 200 }); + const keys = (await store.list({ limit: 10, offset: 0 })).map((r) => r.key); + expect(keys).toEqual(["c", "b", "a"]); + }); + + it("paginates with limit and offset", async () => { + for (const [key, now] of [ + ["a", 1], + ["b", 2], + ["c", 3], + ] as const) { + await store.record({ key, contentType: "t", sizeBytes: 1, now }); + } + const page = await store.list({ limit: 1, offset: 1 }); + expect(page.map((r) => r.key)).toEqual(["b"]); + }); + + it("excludes soft-deleted media from listings", async () => { + await store.record({ key: "a", contentType: "t", sizeBytes: 1, now: 1 }); + await store.record({ key: "b", contentType: "t", sizeBytes: 1, now: 2 }); + expect(await store.setDeleted("b", 3)).toBe(true); + const keys = (await store.list({ limit: 10, offset: 0 })).map((r) => r.key); + expect(keys).toEqual(["a"]); + expect((await store.get("b"))?.deletedAt).toBe(3); + }); + + it("clears the soft-delete flag on undelete", async () => { + await store.record({ key: "a", contentType: "t", sizeBytes: 1, now: 1 }); + await store.setDeleted("a", 2); + expect(await store.setDeleted("a", null)).toBe(true); + expect((await store.get("a"))?.deletedAt).toBeNull(); + }); + + it("reports an unknown key on setDeleted", async () => { + expect(await store.setDeleted("missing", 1)).toBe(false); + }); + + it("removes a row outright for upload rollback", async () => { + await store.record({ key: "a", contentType: "t", sizeBytes: 1, now: 1 }); + await store.remove("a"); + expect(await store.get("a")).toBeNull(); + // Idempotent: removing an unknown key is a no-op. + await store.remove("a"); + }); + + it("prunes only rows deleted before the cutoff", async () => { + await store.record({ key: "old", contentType: "t", sizeBytes: 1, now: 1 }); + await store.record({ + key: "recent", + contentType: "t", + sizeBytes: 1, + now: 2, + }); + await store.record({ key: "live", contentType: "t", sizeBytes: 1, now: 3 }); + await store.setDeleted("old", 10); + await store.setDeleted("recent", 100); + await store.pruneExpired(50); + expect(await store.get("old")).toBeNull(); + expect((await store.get("recent"))?.deletedAt).toBe(100); + expect((await store.get("live"))?.deletedAt).toBeNull(); + }); +}); diff --git a/packages/micropub/src/media.ts b/packages/micropub/src/media.ts new file mode 100644 index 00000000..ca6e333d --- /dev/null +++ b/packages/micropub/src/media.ts @@ -0,0 +1,334 @@ +/** + * Media metadata store, URL ownership validation, and `q=source` query + * parsing for the proposed Micropub media-endpoint extensions (issue #363 + * design): media listing, upload response body, and recoverable + * `action=delete`/`undelete`. + * + * R2 keys are random UUIDs, so R2 listing alone cannot produce the + * newest-first ordering `q=source` needs; every stored blob gets a row in the + * strongly-consistent `micropub_media` D1 table (never KV). R2 remains the + * blob authority — the row is ordering/metadata bookkeeping. + * + * @see spec/packages/micropub.md#proposed-media-endpoint-extensions + */ + +import type { PageRequest } from "./pagination.js"; + +/** Cloudflare binding required by the media metadata store. */ +export interface MediaStoreEnv { + /** D1 database holding media metadata rows (shared with the post store). */ + readonly MICROPUB_DB: D1Database; +} + +/** + * Upload content types mapped to the R2 key extension the generator appends. + * Doubles as the allowlist of types safe to serve inline (see the handler's + * blob `GET` route): anything else is served as an opaque + * `application/octet-stream` attachment. + */ +export const MEDIA_EXTENSIONS: Record = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/gif": ".gif", + "image/webp": ".webp", + "image/avif": ".avif", + "video/mp4": ".mp4", + "audio/mpeg": ".mp3", +}; + +/** R2 key prefix holding recoverable (soft-deleted) media blobs. */ +export const MEDIA_TRASH_PREFIX = ".trash/"; + +/** The trash-namespace R2 key for a live media key. */ +export function mediaTrashKey(key: string): string { + return `${MEDIA_TRASH_PREFIX}${key}`; +} + +/** + * The generator format for media keys: a lowercase UUID plus an optional + * known extension (the values of {@link MEDIA_EXTENSIONS}). Anchored so a + * key is always exactly one path segment. + */ +const KEY_PATTERN = new RegExp( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:${Object.values( + MEDIA_EXTENSIONS, + ) + .map((ext) => ext.replace(".", "\\.")) + .join("|")})?$`, +); + +/** + * Resolve a client-supplied `url` to the media key it names, or `null` if it + * is not owned by this media endpoint. This is the load-bearing ownership + * validation for the delete/undelete actions and the by-URL query: the URL + * must be absolute and canonicalize to exactly `${mediaEndpoint}/` with + * `` a single generator-format path segment — no other origins or path + * prefixes, no traversal or encoded-slash key confusion, no query or + * fragment, and never the trash prefix. + */ +export function mediaKeyFromUrl( + url: string, + mediaEndpoint: string, +): string | null { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + if (parsed.search !== "" || parsed.hash !== "") return null; + const endpoint = new URL(mediaEndpoint); + if (parsed.protocol !== endpoint.protocol) return null; + if (parsed.origin !== endpoint.origin) return null; + const prefix = `${endpoint.pathname}/`; + if (!parsed.pathname.startsWith(prefix)) return null; + const key = parsed.pathname.slice(prefix.length); + // Percent-encoded bytes could smuggle a slash (or dot segments) past the + // single-segment check once decoded downstream; the generator never + // percent-encodes, so any `%` is foreign. + if (key.includes("%")) return null; + if (!KEY_PATTERN.test(key)) return null; + return key; +} + +/** Raised when a media query or action carries an invalid parameter. */ +export class MediaValidationError extends Error {} + +/** A validated media `q=source` request: by URL, or a listing page. */ +export type MediaSourceQuery = { url: string } | { page: PageRequest }; + +const DEFAULT_LIMIT = 10; +const MAX_LIMIT = 100; + +function requireAtMostOne( + params: URLSearchParams, + name: string, +): string | null { + const values = params.getAll(name); + if (values.length > 1) { + throw new MediaValidationError(`\`${name}\` must not be repeated`); + } + return values[0] ?? null; +} + +function parseNonNegativeInteger(raw: string, name: string): number { + if (!/^\d+$/.test(raw)) { + throw new MediaValidationError( + `\`${name}\` must be a non-negative integer, got "${raw}"`, + ); + } + return Number(raw); +} + +/** + * Parse and validate a media-endpoint `q=source` query string: either a + * single `url`, or an optional `limit` (default 10, max 100) and `offset` + * (default 0) page. Every duplicated, unknown, or malformed parameter is + * rejected rather than ignored, matching the other proposed extension + * queries. + */ +export function parseMediaSourceParams( + params: URLSearchParams, +): MediaSourceQuery { + const allowed = new Set(["q", "url", "limit", "offset"]); + for (const key of params.keys()) { + if (!allowed.has(key)) { + throw new MediaValidationError( + `unsupported media \`q=source\` query parameter \`${key}\``, + ); + } + } + + const url = requireAtMostOne(params, "url"); + const limitRaw = requireAtMostOne(params, "limit"); + const offsetRaw = requireAtMostOne(params, "offset"); + + if (url !== null) { + if (limitRaw !== null || offsetRaw !== null) { + throw new MediaValidationError( + "`limit`/`offset` do not apply to a single-file `url` query", + ); + } + return { url }; + } + + let limit = DEFAULT_LIMIT; + if (limitRaw !== null) { + limit = parseNonNegativeInteger(limitRaw, "limit"); + if (limit < 1) { + throw new MediaValidationError("`limit` must be at least 1"); + } + limit = Math.min(limit, MAX_LIMIT); + } + const offset = + offsetRaw === null ? 0 : parseNonNegativeInteger(offsetRaw, "offset"); + return { page: { limit, offset } }; +} + +// --- D1 media metadata store -------------------------------------------------- + +/** A stored media blob's metadata row. */ +export interface MediaRecord { + /** R2 key (primary key); the public URL is `${mediaEndpoint}/${key}`. */ + readonly key: string; + /** Stored content type, as declared at upload. */ + readonly contentType: string; + /** Blob size in bytes. */ + readonly sizeBytes: number; + /** Upload time (seconds since the epoch). */ + readonly uploadedAt: number; + /** Soft-delete time (seconds since the epoch), or `null` while live. */ + readonly deletedAt: number | null; +} + +/** Storage interface over media metadata rows. */ +export interface MicropubMediaStore { + /** Create the schema if absent. Idempotent. */ + init(): Promise; + /** Insert a new upload's row. */ + record(record: { + key: string; + contentType: string; + sizeBytes: number; + now: number; + }): Promise; + /** Read a row (including soft-deleted ones), or `null` if unknown. */ + get(key: string): Promise; + /** + * List live (non-deleted) media newest-first by upload time with `key` as + * the deterministic tie-breaker. The caller owns limit/offset validation. + */ + list(page: PageRequest): Promise; + /** + * Set or clear a row's soft-delete time. Returns `false` if the key is + * unknown (a legacy blob with no row). + */ + setDeleted(key: string, deletedAt: number | null): Promise; + /** + * Hard-delete a row, for rolling an upload back before its URL was ever + * handed out (multi-file fold failure). Idempotent. + */ + remove(key: string): Promise; + /** + * Remove rows soft-deleted before `cutoff` (seconds since the epoch). The + * blob bytes under the trash prefix are purged by an R2 lifecycle rule, so + * this is opportunistic row hygiene, not correctness. + */ + pruneExpired(cutoff: number): Promise; +} + +const SCHEMA = `CREATE TABLE IF NOT EXISTS micropub_media ( + key TEXT PRIMARY KEY, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + uploaded_at INTEGER NOT NULL, + deleted_at INTEGER +)`; + +const INDEXES = [ + "CREATE INDEX IF NOT EXISTS micropub_media_live ON micropub_media (deleted_at, uploaded_at, key)", +]; + +interface MediaRow { + readonly key: string; + readonly content_type: string; + readonly size_bytes: number; + readonly uploaded_at: number; + readonly deleted_at: number | null; +} + +function rowToRecord(row: MediaRow): MediaRecord { + return { + key: row.key, + contentType: row.content_type, + sizeBytes: row.size_bytes, + uploadedAt: row.uploaded_at, + deletedAt: row.deleted_at, + }; +} + +/** Create the built-in D1 media metadata store. */ +export function createMicropubMediaStore( + env: MediaStoreEnv, +): MicropubMediaStore { + const db = env.MICROPUB_DB; + let ready: Promise | null = null; + const ensureSchema = (): Promise => { + ready ??= db + .prepare(SCHEMA) + .run() + .then(async () => { + await db.batch(INDEXES.map((sql) => db.prepare(sql))); + }) + .catch((err: unknown) => { + ready = null; + throw err; + }); + return ready; + }; + + return { + async init() { + await ensureSchema(); + }, + async record({ key, contentType, sizeBytes, now }) { + await ensureSchema(); + await db + .prepare( + `INSERT INTO micropub_media (key, content_type, size_bytes, uploaded_at, deleted_at) + VALUES (?, ?, ?, ?, NULL)`, + ) + .bind(key, contentType, sizeBytes, now) + .run(); + }, + async get(key) { + await ensureSchema(); + const row = await db + .prepare( + `SELECT key, content_type, size_bytes, uploaded_at, deleted_at + FROM micropub_media WHERE key = ?`, + ) + .bind(key) + .first(); + return row ? rowToRecord(row) : null; + }, + async list(page) { + await ensureSchema(); + const { results } = await db + .prepare( + `SELECT key, content_type, size_bytes, uploaded_at, deleted_at + FROM micropub_media + WHERE deleted_at IS NULL + ORDER BY uploaded_at DESC, key DESC + LIMIT ? OFFSET ?`, + ) + .bind(page.limit, page.offset) + .all(); + return (results ?? []).map(rowToRecord); + }, + async setDeleted(key, deletedAt) { + await ensureSchema(); + const result = await db + .prepare("UPDATE micropub_media SET deleted_at = ? WHERE key = ?") + .bind(deletedAt, key) + .run(); + return (result.meta.changes ?? 0) > 0; + }, + async remove(key) { + await ensureSchema(); + await db + .prepare("DELETE FROM micropub_media WHERE key = ?") + .bind(key) + .run(); + }, + async pruneExpired(cutoff) { + await ensureSchema(); + await db + .prepare( + "DELETE FROM micropub_media WHERE deleted_at IS NOT NULL AND deleted_at < ?", + ) + .bind(cutoff) + .run(); + }, + }; +} diff --git a/spec/packages/micropub.md b/spec/packages/micropub.md index 4c80b6b0..491adb28 100644 --- a/spec/packages/micropub.md +++ b/spec/packages/micropub.md @@ -385,8 +385,9 @@ failures. ### Proposed media-endpoint extensions -This is the design for issue #363 (roadmap #354). It is **design only — not -yet implemented**. It adopts three upstream proposals as one gated feature: +This is the design for issue #363 (roadmap #354), now **implemented** (media +metadata in `src/media.ts`, endpoint wiring in `src/handler.ts`). It adopts +three upstream proposals as one gated feature: [Delete from Media Endpoint][mp-ext-media-delete], [Response from Media Endpoint][mp-ext-media-response], and [Query for Media from Media Endpoint][mp-ext-media-source] together with its