From f903fab129a44b1b1fe758e1d998a413995c84b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 21:22:38 +0000 Subject: [PATCH 1/2] fix(activitypub,safe-fetch,webfinger): cap outbox scan and dedupe timeout signal Address two non-blocking follow-ups from #364's review: bound the owner-outbox merge loop in the Mastodon timeline so a like/announce-dominated outbox can't force a near-full-table scan per request, and consolidate the three near-identical createTimeoutSignal implementations into one shared export from @dwk/safe-fetch. --- .changeset/activitypub-outbox-scan-cap.md | 12 +++++ packages/activitypub/src/delivery.ts | 9 ++-- packages/activitypub/src/discovery.ts | 4 +- packages/activitypub/src/mastodon-api.test.ts | 47 +++++++++++++++++++ packages/activitypub/src/object.ts | 20 +++++++- packages/activitypub/src/timeout.ts | 16 ------- packages/safe-fetch/src/index.ts | 1 + packages/safe-fetch/src/safe-fetch.ts | 11 +++-- packages/webfinger/package.json | 3 +- packages/webfinger/src/lookup.ts | 17 ++----- pnpm-lock.yaml | 3 ++ 11 files changed, 102 insertions(+), 41 deletions(-) create mode 100644 .changeset/activitypub-outbox-scan-cap.md delete mode 100644 packages/activitypub/src/timeout.ts diff --git a/.changeset/activitypub-outbox-scan-cap.md b/.changeset/activitypub-outbox-scan-cap.md new file mode 100644 index 00000000..126ec045 --- /dev/null +++ b/.changeset/activitypub-outbox-scan-cap.md @@ -0,0 +1,12 @@ +--- +"@dwk/activitypub": patch +"@dwk/safe-fetch": patch +"@dwk/webfinger": patch +--- + +Cap the number of outbox batches scanned when merging owner posts into a +Mastodon timeline page, so a like/announce-dominated outbox can no longer +force a near-full-table scan per request. Also de-duplicate the cancellable +timeout-signal helper: `@dwk/safe-fetch` now exports `createTimeoutSignal`, +reused by `@dwk/activitypub` and `@dwk/webfinger` instead of each carrying its +own copy. diff --git a/packages/activitypub/src/delivery.ts b/packages/activitypub/src/delivery.ts index d8edf9bf..ad2ccc47 100644 --- a/packages/activitypub/src/delivery.ts +++ b/packages/activitypub/src/delivery.ts @@ -12,10 +12,13 @@ * primitive; this module wires it to the network. */ -import { assertPublicUrl, SsrfError } from "@dwk/safe-fetch"; +import { + assertPublicUrl, + createTimeoutSignal, + SsrfError, +} from "@dwk/safe-fetch"; import { signRequest, type SignerKey } from "./signature.js"; -import { createTimeoutSignal } from "./timeout.js"; /** Machine-readable cause of a blocked delivery target. */ export type BlockedReason = @@ -88,7 +91,7 @@ export async function deliverActivity( const signed = await signRequest(inboxUrl, body, signer, { now }); let response: Response; - const timeout = createTimeoutSignal(timeoutMs); + const timeout = createTimeoutSignal(undefined, timeoutMs); try { response = await fetchImpl(inboxUrl, { method: "POST", diff --git a/packages/activitypub/src/discovery.ts b/packages/activitypub/src/discovery.ts index 6cd8063a..87f27eb5 100644 --- a/packages/activitypub/src/discovery.ts +++ b/packages/activitypub/src/discovery.ts @@ -9,10 +9,10 @@ * @see spec/fediverse-interop.md §2.4 */ +import { createTimeoutSignal } from "@dwk/safe-fetch"; import { resolveHandle } from "@dwk/webfinger"; import { assertPublicHttpsTarget } from "./delivery.js"; -import { createTimeoutSignal } from "./timeout.js"; /** Whether a string looks like a handle rather than an IRI. */ export function isHandleShaped(value: string): boolean { @@ -112,7 +112,7 @@ export async function fetchActorGuarded( fetchImpl: typeof fetch, ): Promise { let response: Response; - const timeout = createTimeoutSignal(10_000); + const timeout = createTimeoutSignal(undefined, 10_000); try { response = await guardedFetch(fetchImpl)(iri, { headers: { accept: "application/activity+json" }, diff --git a/packages/activitypub/src/mastodon-api.test.ts b/packages/activitypub/src/mastodon-api.test.ts index 11896068..ad8e7430 100644 --- a/packages/activitypub/src/mastodon-api.test.ts +++ b/packages/activitypub/src/mastodon-api.test.ts @@ -420,6 +420,53 @@ describe("buildMastodonBackend", () => { ]); }); + it("caps the outbox scan instead of exhausting a like-dominated table", async () => { + const config = freshConfig(); + const timestamp = Date.now(); + // limit:1 => BATCH = max(1*4, 40) = 40; MAX_OUTBOX_SCAN_BATCHES = 25, so + // the scan gives up after 1000 rows. Bury a real owner post behind 1000 + // newer non-post rows so it falls just past that cap. + const buriedPost = { + id: `${config.iris.outbox}/buried-owner-post`, + type: "Create", + actor: config.iris.id, + object: { + id: `${config.iris.outbox}/buried-owner-post/object`, + type: "Note", + content: "buried owner post", + }, + }; + const stub = testEnv.ACTOR.get(testEnv.ACTOR.idFromName(config.iris.id)); + await runInDurableObject(stub, async (_instance, state) => { + state.storage.sql.exec( + `INSERT INTO outbox (id, json, published_at) VALUES (?, ?, ?)`, + buriedPost.id, + JSON.stringify(buriedPost), + timestamp, + ); + for (let i = 0; i < 1000; i++) { + const like = { + id: `${config.iris.outbox}/scan-cap-like-${i}`, + type: "Like", + actor: config.iris.id, + object: `https://remote.example/objects/${i}`, + }; + state.storage.sql.exec( + `INSERT INTO outbox (id, json, published_at) VALUES (?, ?, ?)`, + like.id, + JSON.stringify(like), + timestamp + i + 1, + ); + } + }); + const backend = buildMastodonBackend({ config, actor: testEnv.ACTOR }); + + const page = await backend.timeline({ limit: 1 }); + expect( + page.entries.some((entry) => entry.activity["id"] === buriedPost.id), + ).toBe(false); + }); + it("serves cached actor profile fields without an outbound request", async () => { const config = freshConfig(); const stub = testEnv.ACTOR.get(testEnv.ACTOR.idFromName(config.iris.id)); diff --git a/packages/activitypub/src/object.ts b/packages/activitypub/src/object.ts index 62af49a4..49fe8fe5 100644 --- a/packages/activitypub/src/object.ts +++ b/packages/activitypub/src/object.ts @@ -71,6 +71,14 @@ const VOTE_SWEEP_MS = 10 * 60_000; const ACTOR_PROFILE_DEBOUNCE_MS = 1_000; /** Actor documents are optional display metadata; keep their cache well below a DO SQLite cell. */ const ACTOR_PROFILE_MAX_BODY_BYTES = 128 * 1024; +/** + * Hard cap on outbox batches scanned per timeline page when merging in owner + * posts (`#serveClientList`). Without it, an owner outbox dominated by + * non-post activities (Like/Announce/etc.) forces a near-full-table scan per + * request; past this cap the page simply returns fewer than `limit` owner + * posts rather than exhausting the table. + */ +const MAX_OUTBOX_SCAN_BATCHES = 25; /** * Cardinality cap on the pending-metrics table: at most this many distinct * `(event, fields)` keys accumulate between drains. Delivery fields include @@ -1448,7 +1456,17 @@ export class ActivityPubObject extends DurableObject { let outboxCursorSeq = tieSeq !== null ? Number(tieSeq) : null; let outboxExhausted = false; let isFirstOutboxBatch = true; - while (outboxMatches.length < limit && !outboxExhausted) { + // An owner outbox dominated by non-post activities (Like/Announce/etc.) + // would otherwise force a near-full-table scan per timeline request; + // cap the number of batches so a sparse outbox degrades to "found + // fewer than `limit` owner posts this page" instead of an unbounded scan. + let outboxBatches = 0; + while ( + outboxMatches.length < limit && + !outboxExhausted && + outboxBatches < MAX_OUTBOX_SCAN_BATCHES + ) { + outboxBatches++; let outboxWhere = initialOutbox.where; const outboxParams = [...initialOutbox.params]; if (outboxCursorReceivedAt !== null && !isFirstOutboxBatch) { diff --git a/packages/activitypub/src/timeout.ts b/packages/activitypub/src/timeout.ts deleted file mode 100644 index 8f902e2f..00000000 --- a/packages/activitypub/src/timeout.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Create a timeout signal that callers can release when their operation - * settles. `AbortSignal.timeout()` leaves its timer pending after success, - * which keeps a Durable Object's event loop alive unnecessarily. - */ -export function createTimeoutSignal(timeoutMs: number): { - readonly signal: AbortSignal; - readonly cancel: () => void; -} { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - return { - signal: controller.signal, - cancel: () => clearTimeout(timeout), - }; -} diff --git a/packages/safe-fetch/src/index.ts b/packages/safe-fetch/src/index.ts index 134642c3..668f168d 100644 --- a/packages/safe-fetch/src/index.ts +++ b/packages/safe-fetch/src/index.ts @@ -15,6 +15,7 @@ export { isPrivateOrReservedHost, assertPublicUrl, safeFetch, + createTimeoutSignal, SsrfError, DEFAULT_MAX_REDIRECTS, DEFAULT_TIMEOUT_MS, diff --git a/packages/safe-fetch/src/safe-fetch.ts b/packages/safe-fetch/src/safe-fetch.ts index 53d0e97f..0a1bc0d4 100644 --- a/packages/safe-fetch/src/safe-fetch.ts +++ b/packages/safe-fetch/src/safe-fetch.ts @@ -38,11 +38,14 @@ const DEFAULT_ALLOWED_SCHEMES = ["http:", "https:"] as const; const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); /** - * Create a request signal that is cancelled by either the caller or an - * overall timeout. Unlike `AbortSignal.timeout()`, its timer can be released - * as soon as the request completes. + * Create a request signal that is cancelled by either an optional caller + * signal or an overall timeout. Unlike `AbortSignal.timeout()`, its timer can + * be released as soon as the request completes, so it doesn't keep a Durable + * Object's event loop alive after a successful call. The shared primitive for + * every `@dwk` package that bounds an outbound request with a cancellable + * timeout, rather than each re-deriving its own copy. */ -function createTimeoutSignal( +export function createTimeoutSignal( callerSignal: AbortSignal | null | undefined, timeoutMs: number, ): { readonly signal: AbortSignal; readonly cancel: () => void } { diff --git a/packages/webfinger/package.json b/packages/webfinger/package.json index f839e5e5..679b1e76 100644 --- a/packages/webfinger/package.json +++ b/packages/webfinger/package.json @@ -41,6 +41,7 @@ "access": "public" }, "dependencies": { - "@dwk/log": "workspace:*" + "@dwk/log": "workspace:*", + "@dwk/safe-fetch": "workspace:*" } } diff --git a/packages/webfinger/src/lookup.ts b/packages/webfinger/src/lookup.ts index 204fe3a2..bbc5eefc 100644 --- a/packages/webfinger/src/lookup.ts +++ b/packages/webfinger/src/lookup.ts @@ -14,6 +14,8 @@ * @see spec/fediverse-interop.md §2.4 (community discovery) */ +import { createTimeoutSignal } from "@dwk/safe-fetch"; + /** A parsed `user@host` handle. `user` keeps its case; `host` is lowercased. */ export interface ParsedHandle { readonly user: string; @@ -113,19 +115,6 @@ export interface ResolveHandleOptions { readonly timeoutMs?: number; } -/** Make a timeout signal whose timer is released when the lookup completes. */ -function createTimeoutSignal(timeoutMs: number): { - readonly signal: AbortSignal; - readonly cancel: () => void; -} { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - return { - signal: controller.signal, - cancel: () => clearTimeout(timeout), - }; -} - /** * Resolve a handle to its ActivityPub actor IRI: parse, query * `/.well-known/webfinger` on the handle's host, and select the `self` actor @@ -139,7 +128,7 @@ export async function resolveHandle( const parsed = parseHandle(handle); if (!parsed) return null; let response: Response; - const timeout = createTimeoutSignal(options.timeoutMs ?? 10_000); + const timeout = createTimeoutSignal(undefined, options.timeoutMs ?? 10_000); try { response = await options.fetch(webfingerQueryUrl(parsed), { headers: { accept: "application/jrd+json, application/json" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63787cbb..bbb4817d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -406,6 +406,9 @@ importers: '@dwk/log': specifier: workspace:* version: link:../log + '@dwk/safe-fetch': + specifier: workspace:* + version: link:../safe-fetch packages/webmention: dependencies: From a6436f76d908b0e0bcc22deca3d9ef65611f6788 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 21:32:22 +0000 Subject: [PATCH 2/2] fix(activitypub,host-meta,remotestorage,webfinger): fix typecheck-before-build and cap inbox scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the missing @dwk/safe-fetch path mapping to every tsconfig.json that transitively pulls in @dwk/webfinger's source (host-meta, remotestorage, webfinger itself) — CI runs typecheck before build, so these packages could not resolve webfinger's new @dwk/safe-fetch import from source. Also cap the inbox notifications scan loop with the same MAX_SCAN_BATCHES bound already applied to the outbox merge, per review: a notifications page over an inbox dominated by plain Create/Update rows had the identical unbounded-scan shape. --- .changeset/activitypub-outbox-scan-cap.md | 13 ++++--- packages/activitypub/src/mastodon-api.test.ts | 37 +++++++++++++++++++ packages/activitypub/src/object.ts | 27 ++++++++++---- packages/host-meta/tsconfig.json | 6 ++- packages/remotestorage/tsconfig.json | 6 ++- packages/webfinger/tsconfig.json | 3 +- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/.changeset/activitypub-outbox-scan-cap.md b/.changeset/activitypub-outbox-scan-cap.md index 126ec045..1b78b106 100644 --- a/.changeset/activitypub-outbox-scan-cap.md +++ b/.changeset/activitypub-outbox-scan-cap.md @@ -4,9 +4,10 @@ "@dwk/webfinger": patch --- -Cap the number of outbox batches scanned when merging owner posts into a -Mastodon timeline page, so a like/announce-dominated outbox can no longer -force a near-full-table scan per request. Also de-duplicate the cancellable -timeout-signal helper: `@dwk/safe-fetch` now exports `createTimeoutSignal`, -reused by `@dwk/activitypub` and `@dwk/webfinger` instead of each carrying its -own copy. +Cap the number of batches scanned per client-list page — both the outbox +owner-post merge into a Mastodon timeline and the inbox notifications scan — +so a like/announce-dominated outbox or a plain-post-dominated inbox can no +longer force a near-full-table scan per request. Also de-duplicate the +cancellable timeout-signal helper: `@dwk/safe-fetch` now exports +`createTimeoutSignal`, reused by `@dwk/activitypub` and `@dwk/webfinger` +instead of each carrying its own copy. diff --git a/packages/activitypub/src/mastodon-api.test.ts b/packages/activitypub/src/mastodon-api.test.ts index ad8e7430..19d1ca91 100644 --- a/packages/activitypub/src/mastodon-api.test.ts +++ b/packages/activitypub/src/mastodon-api.test.ts @@ -557,6 +557,43 @@ describe("buildMastodonBackend", () => { ); }); + it("caps the inbox scan instead of exhausting a plain-post-dominated table", async () => { + const config = freshConfig(); + const timestamp = Date.now(); + // limit:1 => BATCH = max(1*4, 40) = 40; MAX_SCAN_BATCHES = 25, so the + // scan gives up after 1000 rows. Bury a real mention behind 1000 newer + // plain (non-notification) rows so it falls just past that cap. + const buriedMention = createMention(config); + const stub = testEnv.ACTOR.get(testEnv.ACTOR.idFromName(config.iris.id)); + await runInDurableObject(stub, async (_instance, state) => { + state.storage.sql.exec( + `INSERT INTO inbox (id, json, received_at, object_type) + VALUES (?, ?, ?, 'Note')`, + buriedMention["id"] as string, + JSON.stringify(buriedMention), + timestamp, + ); + for (let i = 0; i < 1000; i++) { + const note = createNote(config); + state.storage.sql.exec( + `INSERT INTO inbox (id, json, received_at, object_type) + VALUES (?, ?, ?, 'Note')`, + note["id"] as string, + JSON.stringify(note), + timestamp + i + 1, + ); + } + }); + const backend = buildMastodonBackend({ config, actor: testEnv.ACTOR }); + + const page = await backend.notifications({ limit: 1 }); + expect( + page.entries.some( + (entry) => entry.activity["id"] === buriedMention["id"], + ), + ).toBe(false); + }); + it("timeline() maxId cursor translates to max_received_at/tie_seq and excludes newer rows", async () => { const config = freshConfig(); const first = createNote(config); diff --git a/packages/activitypub/src/object.ts b/packages/activitypub/src/object.ts index 49fe8fe5..8ad3d954 100644 --- a/packages/activitypub/src/object.ts +++ b/packages/activitypub/src/object.ts @@ -72,13 +72,15 @@ const ACTOR_PROFILE_DEBOUNCE_MS = 1_000; /** Actor documents are optional display metadata; keep their cache well below a DO SQLite cell. */ const ACTOR_PROFILE_MAX_BODY_BYTES = 128 * 1024; /** - * Hard cap on outbox batches scanned per timeline page when merging in owner - * posts (`#serveClientList`). Without it, an owner outbox dominated by - * non-post activities (Like/Announce/etc.) forces a near-full-table scan per - * request; past this cap the page simply returns fewer than `limit` owner - * posts rather than exhausting the table. + * Hard cap on batches scanned per client-list page in `#serveClientList`, + * both for the inbox (notifications: favourite/reblog/mention) and for the + * outbox owner-post merge into the timeline. Without it, an inbox or outbox + * dominated by unwanted activity types (plain Create/Update rows for + * notifications; Like/Announce/etc. for the timeline merge) forces a + * near-full-table scan per request; past this cap the page simply returns + * fewer than `limit` matches rather than exhausting the table. */ -const MAX_OUTBOX_SCAN_BATCHES = 25; +const MAX_SCAN_BATCHES = 25; /** * Cardinality cap on the pending-metrics table: at most this many distinct * `(event, fields)` keys accumulate between drains. Delivery fields include @@ -1381,7 +1383,16 @@ export class ActivityPubObject extends DurableObject { // Gating it on `matches.length > 0` instead (as a prior version did) lets // a zero-match first batch re-issue the exact same query forever. let isFirstBatch = true; - while (matches.length < limit && !exhausted) { + // Bounded for the same reason as the outbox merge below: a notifications + // page (favourite/reblog/mention) over an inbox dominated by plain + // Create/Update rows would otherwise scan the whole table. + let inboxBatches = 0; + while ( + matches.length < limit && + !exhausted && + inboxBatches < MAX_SCAN_BATCHES + ) { + inboxBatches++; let batchWhere = where; const batchParams = [...params]; if (cursorReceivedAt !== null && !isFirstBatch) { @@ -1464,7 +1475,7 @@ export class ActivityPubObject extends DurableObject { while ( outboxMatches.length < limit && !outboxExhausted && - outboxBatches < MAX_OUTBOX_SCAN_BATCHES + outboxBatches < MAX_SCAN_BATCHES ) { outboxBatches++; let outboxWhere = initialOutbox.where; diff --git a/packages/host-meta/tsconfig.json b/packages/host-meta/tsconfig.json index 1e27ad38..7d0cc53a 100644 --- a/packages/host-meta/tsconfig.json +++ b/packages/host-meta/tsconfig.json @@ -7,7 +7,11 @@ // rootDir-containment check. This config is `noEmit`, so rootDir is moot. "paths": { "@dwk/log": ["../log/src/index.ts"], - "@dwk/webfinger": ["../webfinger/src/index.ts"] + "@dwk/webfinger": ["../webfinger/src/index.ts"], + // `@dwk/webfinger`'s own source (path-mapped above) imports + // `@dwk/safe-fetch` for its shared timeout-signal helper, and that + // import resolves under THIS program's paths too. + "@dwk/safe-fetch": ["../safe-fetch/src/index.ts"] }, "types": ["@cloudflare/workers-types"], "noEmit": true diff --git a/packages/remotestorage/tsconfig.json b/packages/remotestorage/tsconfig.json index e5fbd631..f12bb04b 100644 --- a/packages/remotestorage/tsconfig.json +++ b/packages/remotestorage/tsconfig.json @@ -12,7 +12,11 @@ // `@dwk/store`'s `Store` interface references `@dwk/rdf`'s `StoredQuad` // (the `readQuads`/`writeQuads` RDF tier we don't call); map it so // `typecheck` resolves the transitive type before the workspace is built. - "@dwk/rdf": ["../rdf/src/index.ts"] + "@dwk/rdf": ["../rdf/src/index.ts"], + // `@dwk/webfinger`'s own source (path-mapped above) imports + // `@dwk/safe-fetch` for its shared timeout-signal helper, and that + // import resolves under THIS program's paths too. + "@dwk/safe-fetch": ["../safe-fetch/src/index.ts"] }, "types": [ "@cloudflare/workers-types", diff --git a/packages/webfinger/tsconfig.json b/packages/webfinger/tsconfig.json index 667333c2..4dfab0bb 100644 --- a/packages/webfinger/tsconfig.json +++ b/packages/webfinger/tsconfig.json @@ -6,7 +6,8 @@ // typecheck ahead of build), which would otherwise trip the // rootDir-containment check. This config is `noEmit`, so rootDir is moot. "paths": { - "@dwk/log": ["../log/src/index.ts"] + "@dwk/log": ["../log/src/index.ts"], + "@dwk/safe-fetch": ["../safe-fetch/src/index.ts"] }, "types": ["@cloudflare/workers-types"], "noEmit": true