diff --git a/.changeset/fast-media-usage-backfill.md b/.changeset/fast-media-usage-backfill.md new file mode 100644 index 0000000000..c13dcad992 --- /dev/null +++ b/.changeset/fast-media-usage-backfill.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Speeds up the initial scan when administrators turn on media usage tracking on Cloudflare Workers and Node.js. diff --git a/.changeset/media-usage-activation-ui.md b/.changeset/media-usage-activation-ui.md new file mode 100644 index 0000000000..d57490e224 --- /dev/null +++ b/.changeset/media-usage-activation-ui.md @@ -0,0 +1,6 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +--- + +Adds a media usage tracking setting. Tracking is enabled during initial setup. Existing sites enable it from Settings, keep the page open while EmDash scans existing content, and can return later to continue from saved progress. diff --git a/.changeset/remove-media-usage-scheduled-recovery.md b/.changeset/remove-media-usage-scheduled-recovery.md new file mode 100644 index 0000000000..d45d3f441f --- /dev/null +++ b/.changeset/remove-media-usage-scheduled-recovery.md @@ -0,0 +1,14 @@ +--- +"emdash": minor +"@emdash-cms/cloudflare": minor +--- + +Removes scheduled media usage recovery. Media usage tracking now advances only while an administrator keeps its settings page visible. This breaks Cloudflare deployments that configure `mediaUsageCron` and Node.js integrations that provide a custom `CronScheduler`. + +#### What should I do? + +On Cloudflare, remove the dedicated media usage Cron Trigger and the `mediaUsageCron` option. Keep the general Cron Trigger unchanged; no replacement trigger is required. + +If you provide a custom Node.js scheduler, remove `setMediaUsageMaintenance()`. A custom `CronScheduler` now implements only `start()`, `stop()`, `reschedule()`, and `setSystemCleanup()`. + +Keep **Settings → Media usage tracking** open until it shows **Ready**. If the page closes, return to continue from saved progress. diff --git a/demos/cloudflare/wrangler.jsonc b/demos/cloudflare/wrangler.jsonc index 71bbfb825c..3e14e658f0 100644 --- a/demos/cloudflare/wrangler.jsonc +++ b/demos/cloudflare/wrangler.jsonc @@ -34,9 +34,9 @@ "bucket_name": "emdash-media", }, ], - // Cron trigger drives the AI Search reindex queue flush. + // Cron triggers drive general maintenance. "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, // Observability "observability": { diff --git a/docs/src/content/docs/deployment/cloudflare.mdx b/docs/src/content/docs/deployment/cloudflare.mdx index 8e52d3650f..e1e85f1469 100644 --- a/docs/src/content/docs/deployment/cloudflare.mdx +++ b/docs/src/content/docs/deployment/cloudflare.mdx @@ -94,9 +94,11 @@ If the database is empty (no collections) and the setup wizard hasn't been compl To change the schema or content model of a site that is already deployed, see [Evolving a Deployed Site](/deployment/schema-evolution/). -## Scheduled Publishing +## Scheduled tasks -On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from Worker Cron Triggers. New Cloudflare templates include both required schedules automatically. When updating an existing project, configure distinct general and Media Usage lanes: +Cloudflare runs scheduled publishing, plugin tasks, and general maintenance from one Cron Trigger. + +Use the standard Worker entry point: ```ts title="src/worker.ts" import handler, { @@ -112,28 +114,18 @@ export default { } satisfies ExportedHandler; ``` -By default, `*/2 * * * *` runs Media Usage maintenance and every other expression runs general maintenance. Then add both Cron Triggers to `wrangler.jsonc`: +Configure one Cron Trigger for general maintenance in `wrangler.jsonc`: ```jsonc title="wrangler.jsonc" { "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, } ``` -To use different schedules, set the corresponding `generalCron` or `mediaUsageCron` option in `createScheduledHandler()` and use the same expression in `wrangler.jsonc`. - - - -### Enable automatic media usage indexing - -Keep `mediaUsageCron` running while you enable automatic media usage indexing. Pause all application -and direct database writes, follow [Enable automatic media usage -indexing](/reference/rest-api/#enable-automatic-media-usage-indexing), then resume writes when the -endpoint returns `active`. Existing content is indexed in the background. +To use a different general maintenance schedule, set `generalCron` in `createScheduledHandler()` and +use the same expression in `wrangler.jsonc`. ## Deploy diff --git a/docs/src/content/docs/deployment/nodejs.mdx b/docs/src/content/docs/deployment/nodejs.mdx index 89b8196815..f2aaa3ecdb 100644 --- a/docs/src/content/docs/deployment/nodejs.mdx +++ b/docs/src/content/docs/deployment/nodejs.mdx @@ -53,20 +53,11 @@ export default defineConfig({ The server runs on `http://localhost:4321` by default. Migrations are applied on the first request. If the database is empty and setup hasn't been completed, your seed file (or the built-in default if you don't have one) is also applied on that first request. -## Scheduled Tasks +## Scheduled tasks -The built-in scheduler runs only while a Node process is running. It handles scheduled publishing, -plugin tasks, and background media indexing. +The built-in scheduler runs only while a Node.js process is running. It handles scheduled publishing, plugin tasks, and general maintenance. -Keep at least one Node process running continuously in production. If all processes stop or sleep, -scheduled tasks pause. - -### Enable automatic media usage indexing - -Keep at least one Node process running while you enable automatic media usage indexing. Pause all -application and direct database writes, follow [Enable automatic media usage -indexing](/reference/rest-api/#enable-automatic-media-usage-indexing), then resume writes when the -endpoint returns `active`. Existing content is indexed in the background. +Keep at least one Node.js process running continuously in production. Scheduled tasks pause when every process stops or sleeps. ## Production Storage diff --git a/docs/src/content/docs/guides/media-library.mdx b/docs/src/content/docs/guides/media-library.mdx index 61b87ee5c4..e40ec3182f 100644 --- a/docs/src/content/docs/guides/media-library.mdx +++ b/docs/src/content/docs/guides/media-library.mdx @@ -15,6 +15,26 @@ and files that are not assigned to a folder. Open a folder to see its files. EmDash media library showing image grid with upload button +## Used in + +Open a file in EmDash's media library to see the content entries that reference it. While EmDash scans existing content, the list includes the references found so far and may be incomplete. + +### Turn on media usage tracking + +If media usage tracking is off, an administrator can turn it on: + +1. Finish any content edits. If another application writes directly to the content database, pause it and wait for any writes in progress to finish. +2. Open **Settings → Media usage tracking**, select **Enable tracking**, then confirm. +3. When the page shows **Indexing existing content**, editing and other database writes can resume. +4. Keep the page open until it shows **Ready**. If you leave, return to continue from saved progress. + +Once media usage tracking is on, it cannot be turned off. + + + ## Uploading Files ### From the Media Library diff --git a/docs/src/content/docs/reference/rest-api.mdx b/docs/src/content/docs/reference/rest-api.mdx index 5ed1abe849..4f9d239d57 100644 --- a/docs/src/content/docs/reference/rest-api.mdx +++ b/docs/src/content/docs/reference/rest-api.mdx @@ -261,12 +261,12 @@ collections: | `stale` | Indexed coverage is outdated | | `unknown` | Stored coverage contains a state this version does not recognize | -Only `complete` supports a scoped complete-zero statement within the EmDash-managed fields +Only `complete` supports a scoped complete-zero statement within the supported fields described below. Counts with any other status are indexed projections and may over-report or under-report. Even complete results are advisory during concurrent writes; usage reads are not a transactional lock and must not be used as a deletion guarantee. -### Get Media Usage Details +### Get media usage details ```http GET /_emdash/api/media/:id/usage?limit=50&cursor=... @@ -320,7 +320,7 @@ trashed entry. Sources are `columns` or `draft_overlay`; occurrences identify th and path without exposing internal index metadata. Media usage covers local media references in top-level image and file fields, repeater image -fields, and Portable Text image blocks managed by EmDash content collections. It does not scan +fields, and Portable Text image blocks stored in EmDash content collections. It does not scan custom code, rendered HTML, settings, menus, widgets, plugin-private data, external sites, or provider-only assets. @@ -606,13 +606,14 @@ IDs or URLs, or change media usage records. DELETE /_emdash/api/media/:id ``` -### Enable automatic media usage indexing +### Enable media usage tracking -Automatic media usage indexing must be enabled once for each production site. Writes must be -paused while EmDash prepares each collection so that no changes are missed. +Sites with media usage tracking off must turn it on once. Pause direct database writes while EmDash prepares each collection. EmDash temporarily blocks its own content and schema writes during this step. Both endpoints require `schema:manage`. Bearer tokens also require the `admin` scope. +For the admin procedure, see [Turn on media usage tracking](/guides/media-library/#turn-on-media-usage-tracking). The endpoints below provide the same procedure for API operators. + #### Check the current state ```http @@ -621,13 +622,13 @@ GET /_emdash/api/admin/media-usage/activation This request does not change anything. It returns one of these states: -- `expanded`: automatic indexing is not enabled. +- `expanded`: media usage tracking is off. - `activating`: EmDash is preparing the site's collections. - `active`: EmDash tracks changes to media references in content. Status responses do not include internal lock data or raw database errors. -#### Prepare the next collection +#### Start activation ```http POST /_emdash/api/admin/media-usage/activation @@ -635,43 +636,83 @@ Content-Type: application/json X-EmDash-Request: 1 { - "writersDrained": true, - "maintenanceReady": true + "writersDrained": true } ``` -Each request prepares one collection. Send one request at a time until the state becomes `active`. - -Set both fields to `true`: +The request prepares at most one collection. After it succeeds, advance setup and historical indexing through the progress endpoint below. -- `writersDrained`: Application and direct database writes have stopped, and any writes already in - progress have finished. -- `maintenanceReady`: Scheduled background tasks are running through `mediaUsageCron` on Cloudflare - or the built-in scheduler on Node.js. +Set `writersDrained` to `true` after application and direct database writes have stopped and any writes already in progress have finished. -#### Enable indexing in production +#### Enable tracking with the API -1. Confirm that scheduled background tasks are running: `mediaUsageCron` on Cloudflare, or the - built-in scheduler on Node.js. -2. Stop all application and direct database writes. Wait for writes already in progress to finish. -3. Call `GET` to check the current state. -4. Call `POST` one request at a time until the state becomes `active`. -5. Resume writes. -6. Keep scheduled background tasks running while EmDash indexes existing content. +1. Stop all direct database writers. Wait for writes already in progress to finish. EmDash fences its own writes during setup. +2. Call the activation `GET` endpoint to check the current state. +3. Call the activation `POST` endpoint once with `writersDrained: true`. +4. Call the progress `POST` endpoint serially, following `nextRequestInMs`, until activation becomes `active`. +5. Resume direct database writes after activation becomes `active`. +6. Continue progress requests until historical indexing reports `ready` and `nextRequestInMs` is `null`. -If `POST` times out or returns `409` or `500`, call `GET` before sending another request. If -`lastErrorCode` is set, keep writes stopped, check the application logs, fix the problem, and try -again. Do not edit EmDash's internal database tables. +If `POST` times out or returns `409` or `500`, call `GET` before deciding what to do. If the state is +still `activating` without `lastErrorCode`, another request may own the current batch. If `lastErrorCode` is set, keep writes stopped, check the application logs, fix the +problem, and send one confirmed POST to retry. Do not edit EmDash's internal database tables. -When the state is `active`, EmDash tracks changes to media references in content. Existing content -may still be indexing in the background. +When the state is `active`, EmDash tracks changes to media references in content. Existing content may still need progress requests before historical indexing is ready. + +#### Check historical indexing progress + +```http +GET /_emdash/api/admin/media-usage/progress +``` + +After activation is active, this returns `indexing`, `ready`, or `needs_attention` together with the +number of ready and total current content types. The endpoint does not inspect content rows or return +work-item details. It requires `schema:manage`; bearer tokens also require the `admin` scope. + +#### Advance setup and historical indexing + +```http +POST /_emdash/api/admin/media-usage/progress +X-EmDash-Request: 1 +``` + +The request has no body. It runs one bounded maintenance step and returns the stored activation and progress state after that step. + +```json +{ + "success": true, + "data": { + "activation": { + "state": "active", + "collectionCursor": null, + "attemptCount": 2, + "drainConfirmedAt": "2026-08-24T12:00:00.000Z", + "lastAttemptedAt": "2026-08-24T12:00:01.000Z", + "lastErrorCode": null, + "leaseExpiresAt": null, + "activatedAt": "2026-08-24T12:00:01.000Z", + "updatedAt": "2026-08-24T12:00:02.000Z" + }, + "progress": { + "status": "indexing", + "readyCollections": 1, + "totalCollections": 2 + }, + "nextRequestInMs": 0 + } +} +``` + +`progress` is `null` until activation is active. `nextRequestInMs` is `0` for an immediate successor, `30000` for a delayed retry, or `null` when the server knows of no successor. Send only one progress request at a time and wait for the returned delay. + +Closing the client does not discard completed work, but it stops future requests. To resume, read activation first, read progress when activation is active, then continue progress requests. After an ambiguous response, perform the same reads before retrying. -### List Media Usage Work +### List media usage work ```http GET /_emdash/api/admin/media-usage/work?collection=posts&state=failed&limit=50&cursor=... @@ -710,7 +751,7 @@ page's `nextCursor`. The endpoint does not calculate an exact backlog count. Responses omit work versions, lease tokens, raw database errors, indexed content, media references, and exact counts. -### Retry Media Usage Work +### Retry media usage work ```http POST /_emdash/api/admin/media-usage/work/retry @@ -735,9 +776,8 @@ replaces newer work or exposes its lease token. The list returns only known durable work. Retry can create work for the supplied identity in an active collection even when no work row exists, but it does not scan for historical gaps. Use -collection-scoped Media Usage repair after imports or direct database writes. -When scheduled maintenance is disabled, failed jobs remain visible and manually retryable, but no -automatic freshness deadline is promised. +collection-scoped media usage repair after imports or direct database writes. +Failed jobs remain visible and manually retryable. A `needs_attention` progress state stops automatic requests from the media usage tracking settings page until the underlying failure is resolved. ### Recover Collection Deletion @@ -763,7 +803,7 @@ returns `409 WORK_LEASE_ACTIVE`; a concurrent state change returns `409 WORK_CHA require `schema:manage`, and bearer tokens also require the `admin` scope. They recover internal index cleanup only and never delete media assets. -### Repair Media Usage +### Repair media usage ```http POST /_emdash/api/admin/media-usage/repair diff --git a/e2e/tests/setup-wizard.spec.ts b/e2e/tests/setup-wizard.spec.ts index 733cb0c772..f98e36d743 100644 --- a/e2e/tests/setup-wizard.spec.ts +++ b/e2e/tests/setup-wizard.spec.ts @@ -58,6 +58,7 @@ test.describe("Setup Wizard", () => { test("shows validation error when title is empty", async ({ admin }) => { await admin.goToSetup(); + await admin.page.getByLabel("Site Title").fill(""); await admin.page.getByRole("button", { name: "Continue" }).click(); await expect(admin.page.locator("text=Site title is required")).toBeVisible(); diff --git a/packages/admin/src/components/ConfirmDialog.tsx b/packages/admin/src/components/ConfirmDialog.tsx index 25d370b6f9..2b65c56b40 100644 --- a/packages/admin/src/components/ConfirmDialog.tsx +++ b/packages/admin/src/components/ConfirmDialog.tsx @@ -23,6 +23,12 @@ export interface ConfirmDialogProps { pendingLabel: string; /** Button variant — defaults to "destructive" */ variant?: "destructive" | "primary"; + /** Use tighter Kumo spacing for short, focused confirmations. */ + compact?: boolean; + /** Prevent dismissing an irreversible request after it has started. */ + preventCloseWhilePending?: boolean; + /** Disable confirmation until required input in the dialog is complete. */ + confirmDisabled?: boolean; isPending: boolean; /** Error from a mutation — pass mutation.error directly */ error: unknown; @@ -39,24 +45,44 @@ export function ConfirmDialog({ confirmLabel, pendingLabel, variant = "destructive", + compact = false, + preventCloseWhilePending = false, + confirmDisabled = false, isPending, error, onConfirm, children, }: ConfirmDialogProps) { const { t } = useLingui(); + const closeLocked = preventCloseWhilePending && isPending; return ( - !o && onClose()} disablePointerDismissal> - - {title} - {description} + !nextOpen && !closeLocked && onClose()} + disablePointerDismissal + > + +
+ + {title} + + + {description} + +
{children} -
- -
diff --git a/packages/admin/src/components/MediaLibrary.tsx b/packages/admin/src/components/MediaLibrary.tsx index 6c881cb59b..fc100754d1 100644 --- a/packages/admin/src/components/MediaLibrary.tsx +++ b/packages/admin/src/components/MediaLibrary.tsx @@ -1,4 +1,5 @@ import { + Banner, Breadcrumbs, Button, Grid, @@ -56,6 +57,13 @@ import { fetchProviderMedia, uploadToProvider, } from "../lib/api"; +import { useCurrentUser } from "../lib/api/current-user.js"; +import { + MEDIA_USAGE_ACTIVATION_QUERY_KEY, + MEDIA_USAGE_PROGRESS_QUERY_KEY, + fetchMediaUsageActivationStatus, + fetchMediaUsageProgress, +} from "../lib/api/media-usage-activation.js"; import { useDebouncedValue } from "../lib/hooks.js"; import { providerItemToMediaItem, @@ -217,11 +225,43 @@ export function MediaLibrary({ onMoveMedia, }: MediaLibraryProps) { const { t } = useLingui(); + const isAdmin = (useCurrentUser().data?.role ?? 0) >= 50; + const [activeProvider, setActiveProvider] = React.useState("local"); + const activationQuery = useQuery({ + queryKey: MEDIA_USAGE_ACTIVATION_QUERY_KEY, + queryFn: fetchMediaUsageActivationStatus, + enabled: isAdmin && activeProvider === "local", + retry: false, + staleTime: 60_000, + refetchOnMount: "always", + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }); + const setupStatus = isAdmin && !activationQuery.isError ? activationQuery.data : undefined; + const progressQuery = useQuery({ + queryKey: MEDIA_USAGE_PROGRESS_QUERY_KEY, + queryFn: fetchMediaUsageProgress, + enabled: + isAdmin && + activeProvider === "local" && + !activationQuery.isError && + setupStatus?.state === "active", + retry: false, + staleTime: 60_000, + refetchOnMount: "always", + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }); + const setupProgress = progressQuery.data; + const setupIncomplete = + setupStatus && + (setupStatus.state !== "active" || + progressQuery.isError || + (progressQuery.isSuccess && setupProgress?.status !== "ready")); const [toastManager] = React.useState(createKumoToastManager); const [viewMode, setViewMode] = React.useState<"grid" | "list">("grid"); const [detailItem, setDetailItem] = React.useState(null); const [isDetailOpen, setIsDetailOpen] = React.useState(false); - const [activeProvider, setActiveProvider] = React.useState("local"); const [searchQuery, setSearchQuery] = React.useState(""); const [localTypeFilter, setLocalTypeFilter] = React.useState("all"); const mediaHeadingRef = React.useRef(null); @@ -836,6 +876,26 @@ export function MediaLibrary({ )} + {activeProvider === "local" && setupIncomplete ? ( + + {setupStatus.state === "expanded" ? t`Open setup` : t`View setup`} + + } + /> + ) : null} {/* Provider tabs (only when an external provider is configured) */} {providerTabs.length > 1 && ( diff --git a/packages/admin/src/components/Settings.tsx b/packages/admin/src/components/Settings.tsx index 872614c2b7..0372892c61 100644 --- a/packages/admin/src/components/Settings.tsx +++ b/packages/admin/src/components/Settings.tsx @@ -11,11 +11,13 @@ import { Envelope, DownloadSimple, CaretDown, + Images, } from "@phosphor-icons/react"; import { useQuery } from "@tanstack/react-query"; import * as React from "react"; import { fetchManifest } from "../lib/api"; +import { useCurrentUser } from "../lib/api/current-user.js"; import { SUPPORTED_LOCALES } from "../locales/index.js"; import { useLocale } from "../locales/useLocale.js"; import { SettingsNavRow, SettingsSection } from "./settings/SettingsLayout.js"; @@ -24,6 +26,7 @@ import { SettingsNavRow, SettingsSection } from "./settings/SettingsLayout.js"; * Settings hub page — links to all settings sub-pages. */ export function Settings() { + const { data: currentUser } = useCurrentUser(); const { data: manifest } = useQuery({ queryKey: ["manifest"], queryFn: fetchManifest, @@ -32,6 +35,7 @@ export function Settings() { const { t } = useLingui(); const { locale, setLocale } = useLocale(); const showSecuritySettings = manifest?.authMode === "passkey"; + const showMediaUsageSettings = (currentUser?.role ?? 0) >= 50; const selectedLocale = SUPPORTED_LOCALES.find((option) => option.code === locale) ?? null; return ( @@ -62,6 +66,17 @@ export function Settings() { /> + {showMediaUsageSettings ? ( + + } + title={t`Media usage tracking`} + description={t`Track where media is used across your content`} + /> + + ) : null} + {showSecuritySettings && ( = ROLE_ADMIN; + const [dialogOpen, setDialogOpen] = React.useState(false); + const [notice, setNotice] = React.useState(null); + const [pageVisible, setPageVisible] = React.useState( + () => typeof document === "undefined" || document.visibilityState !== "hidden", + ); + const [progress, setProgress] = React.useState(() => + queryClient.getQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY), + ); + const [progressRequestError, setProgressRequestError] = React.useState(false); + const [resumeToken, setResumeToken] = React.useState(0); + const submittingRef = React.useRef(false); + const progressRequestRef = React.useRef | null>(null); + const mountedRef = React.useRef(true); + const pageVisibleRef = React.useRef(pageVisible); + const focusAfterActionRef = React.useRef(false); + const stateHeadingRef = React.useRef(null); + const wasHiddenRef = React.useRef(false); + React.useEffect(() => { + mountedRef.current = true; + pageVisibleRef.current = document.visibilityState !== "hidden"; + return () => { + mountedRef.current = false; + pageVisibleRef.current = false; + }; + }, []); + + const activationQuery = useQuery({ + queryKey: MEDIA_USAGE_ACTIVATION_QUERY_KEY, + queryFn: fetchMediaUsageActivationStatus, + enabled: isAdmin, + retry: false, + refetchOnMount: "always", + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }); + + const closeDialog = React.useCallback(() => { + setDialogOpen(false); + }, []); + const refreshStatus = React.useCallback( + async (uncertain = false) => { + if (submittingRef.current) return; + if (uncertain) { + setNotice("unconfirmed"); + closeDialog(); + } + const result = await activationQuery.refetch({ cancelRefetch: false }); + if (!result.isSuccess) return; + setResumeToken((current) => current + 1); + setNotice((current) => (current === "validation" || current === "version" ? current : null)); + }, + [activationQuery, closeDialog], + ); + const handleProgressAccessError = React.useCallback((caught: unknown): boolean => { + if (!(caught instanceof MediaUsageActivationRequestError)) return false; + const nextNotice = caught.kind === "version_mismatch" ? "version" : caught.kind; + if (nextNotice !== "denied" && nextNotice !== "validation" && nextNotice !== "version") + return false; + setNotice(nextNotice); + return true; + }, []); + const readProgress = React.useCallback( + () => + queryClient.fetchQuery({ + queryKey: MEDIA_USAGE_PROGRESS_QUERY_KEY, + queryFn: fetchMediaUsageProgress, + retry: false, + staleTime: 0, + }), + [queryClient], + ); + const reconcileProgressFailure = React.useCallback( + async (caught: unknown) => { + if (handleProgressAccessError(caught)) return; + try { + const activation = await fetchMediaUsageActivationStatus(); + if (!mountedRef.current) return; + queryClient.setQueryData(MEDIA_USAGE_ACTIVATION_QUERY_KEY, activation); + let storedProgress: MediaUsageProgress | undefined; + if (activation.state === "active") { + storedProgress = await readProgress(); + if (!mountedRef.current) return; + } + setProgress(storedProgress); + const storedFailure = + activation.state === "activating" && activation.lastErrorCode !== null; + setProgressRequestError( + !storedFailure && + (activation.state === "activating" || storedProgress?.status === "indexing"), + ); + } catch (error) { + if (!mountedRef.current || handleProgressAccessError(error)) return; + setProgressRequestError(true); + } + }, + [handleProgressAccessError, queryClient, readProgress], + ); + const requestProgress = React.useCallback(() => { + if (progressRequestRef.current) return progressRequestRef.current; + const request = advanceMediaUsageProgress().finally(() => { + if (progressRequestRef.current === request) progressRequestRef.current = null; + }); + progressRequestRef.current = request; + return request; + }, []); + const canDriveProgress = + activationQuery.data !== undefined && + activationQuery.data.state !== "expanded" && + activationQuery.data.lastErrorCode === null; + + React.useEffect(() => { + if ( + !isAdmin || + !pageVisible || + activationQuery.isFetching || + activationQuery.isError || + !canDriveProgress + ) + return; + let cancelled = false; + let timer: number | undefined; + let finishWait: (() => void) | undefined; + const wait = (delayMs: 30_000) => + new Promise((resolve) => { + finishWait = () => { + timer = undefined; + finishWait = undefined; + resolve(); + }; + timer = window.setTimeout(finishWait, delayMs); + }); + void (async () => { + if (activationQuery.data?.state === "active") { + let storedProgress: MediaUsageProgress; + try { + storedProgress = await readProgress(); + } catch (error) { + if (!cancelled) await reconcileProgressFailure(error); + return; + } + if (cancelled || !mountedRef.current) return; + setProgress(storedProgress); + setProgressRequestError(false); + if (storedProgress.status !== "indexing") return; + } + let delayMs: 0 | 30_000 = 0; + for (;;) { + if (delayMs === 30_000) await wait(delayMs); + if (cancelled || !pageVisibleRef.current) return; + let result: MediaUsageProgressAdvanceResponse; + try { + result = await requestProgress(); + } catch (error) { + if (!cancelled) await reconcileProgressFailure(error); + return; + } + queryClient.setQueryData(MEDIA_USAGE_ACTIVATION_QUERY_KEY, result.activation); + if (result.progress) + queryClient.setQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY, result.progress); + if (cancelled || !mountedRef.current) return; + setProgress(result.progress ?? undefined); + setProgressRequestError(false); + const storedFailure = + result.activation.state === "activating" && result.activation.lastErrorCode !== null; + if ( + storedFailure || + result.progress?.status === "needs_attention" || + result.nextRequestInMs === null + ) + return; + delayMs = result.nextRequestInMs; + } + })(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + finishWait?.(); + }; + }, [ + activationQuery.isError, + activationQuery.isFetching, + canDriveProgress, + isAdmin, + pageVisible, + queryClient, + readProgress, + reconcileProgressFailure, + requestProgress, + resumeToken, + ]); + + React.useEffect(() => { + if (!isAdmin) return; + const visibilityChanged = () => { + const visible = document.visibilityState !== "hidden"; + pageVisibleRef.current = visible; + setPageVisible(visible); + if (!visible) { + wasHiddenRef.current = true; + if (!submittingRef.current) closeDialog(); + return; + } + if (!wasHiddenRef.current) return; + wasHiddenRef.current = false; + void (async () => { + const result = await activationQuery.refetch({ cancelRefetch: false }); + if (mountedRef.current && result.isSuccess) { + if (submittingRef.current && result.data.state !== "expanded") closeDialog(); + setResumeToken((current) => current + 1); + } + })(); + }; + document.addEventListener("visibilitychange", visibilityChanged); + return () => { + document.removeEventListener("visibilitychange", visibilityChanged); + }; + }, [activationQuery, closeDialog, isAdmin]); + + const advanceMutation = useMutation({ + mutationFn: async () => { + await queryClient.cancelQueries({ queryKey: MEDIA_USAGE_ACTIVATION_QUERY_KEY }); + return advanceMediaUsageActivation({ writersDrained: true }); + }, + retry: false, + onSuccess: (result) => { + queryClient.setQueryData(MEDIA_USAGE_ACTIVATION_QUERY_KEY, result.activation); + setResumeToken((current) => current + 1); + setNotice(null); + closeDialog(); + }, + onError: (caught) => { + submittingRef.current = false; + void handleAdvanceError(caught); + }, + onSettled: () => { + submittingRef.current = false; + }, + }); + + const handleAdvanceError = async (caught: unknown) => { + const error = + caught instanceof MediaUsageActivationRequestError + ? caught + : new MediaUsageActivationRequestError("unknown", null); + closeDialog(); + if (error.kind === "denied") return setNotice("denied"); + if (error.kind === "version_mismatch") return setNotice("version"); + if (error.kind === "validation") return setNotice("validation"); + await refreshStatus(true); + }; + + const activation = activationQuery.data; + React.useEffect(() => { + const heading = stateHeadingRef.current; + if ( + !focusAfterActionRef.current || + !pageVisible || + dialogOpen || + advanceMutation.isPending || + !activation || + !heading + ) + return; + focusAfterActionRef.current = false; + heading.focus(); + }, [activation, advanceMutation.isPending, dialogOpen, pageVisible]); + + const title = t`Media usage tracking`; + const description = t`Track where media is used across your content.`; + if (userLoading) return ; + if (!isAdmin || isActivationError(activationQuery.error, "denied") || notice === "denied") { + return ( + + ); + } + if (isActivationError(activationQuery.error, "version_mismatch") || notice === "version") { + return ( + + ); + } + if (notice === "validation") { + return ( + + ); + } + if (notice === "unconfirmed") { + return ( + void refreshStatus()}> + {t`Refresh status`} + + } + /> + ); + } + const activationReadError = activationQuery.isError || activationQuery.isRefetchError; + if (activationReadError && !activation) { + return ( + void refreshStatus()}> + {t`Try again`} + + } + /> + ); + } + if (activationQuery.isPending || !activation) { + return ; + } + if ( + !activationReadError && + activation.state === "active" && + progress === undefined && + !progressRequestError + ) { + return ; + } + + const storedFailure = activation.state === "activating" && activation.lastErrorCode !== null; + const canMutate = !activationReadError && (activation.state === "expanded" || storedFailure); + const submit = () => { + if (submittingRef.current) return; + submittingRef.current = true; + focusAfterActionRef.current = true; + advanceMutation.mutate(); + }; + + return ( + + void refreshStatus()}> + {t`Try again`} + + ) : progressRequestError ? ( + + ) : undefined + } + > + + {canMutate ? ( + + + + ) : null} + + + + + ); +} + +function StatusRow({ + activation, + progress, + progressError, + activationError, + stateHeadingRef, +}: { + activation: MediaUsageActivationStatus; + progress: MediaUsageProgress | undefined; + progressError: boolean; + activationError: boolean; + stateHeadingRef: React.RefObject; +}) { + const { t } = useLingui(); + const ready = progress?.readyCollections ?? 0; + const total = progress?.totalCollections ?? 0; + const active = activation.state === "active"; + const settingUp = activation.state === "activating"; + const storedFailure = activation.state === "activating" && activation.lastErrorCode !== null; + let heading = t`Media usage tracking is off`; + let detail: React.ReactNode = t`Enable tracking to index existing content and keep references up to date.`; + let badge = t`Off`; + let variant: "neutral" | "warning" | "success" | "error" = "neutral"; + if (settingUp) { + heading = storedFailure ? t`Needs attention` : t`Setting up`; + detail = storedFailure + ? t`Keep editing paused, fix the server issue, then retry setup.` + : t`Capture is being prepared. Keep editing paused until setup is complete.`; + badge = storedFailure ? t`Needs attention` : t`Setting up`; + variant = storedFailure ? "error" : "warning"; + } + if (active) { + heading = progressError + ? t`Needs attention` + : progress?.status === "ready" + ? t`Media usage tracking is ready` + : progress?.status === "needs_attention" + ? t`Needs attention` + : t`Indexing existing content`; + detail = progressError ? ( + t`Setup couldn’t continue. Try again.` + ) : progress?.status === "needs_attention" ? ( + t`Check the server logs, then use the media usage recovery API for the failed work.` + ) : progress?.status === "ready" ? ( + t`Existing content is indexed. New changes are tracked automatically.` + ) : progress ? ( + <> + {t`Ready`}:{" "} + + {ready} / {total} + + + ) : ( + t`EmDash is scanning existing content.` + ); + badge = progressError + ? t`Needs attention` + : progress?.status === "ready" + ? t`Ready` + : progress?.status === "needs_attention" + ? t`Needs attention` + : t`Indexing`; + variant = progressError + ? "error" + : progress?.status === "ready" + ? "success" + : progress?.status === "needs_attention" + ? "error" + : "warning"; + } + if (activationError) { + heading = t`Needs attention`; + detail = t`Refresh status before continuing.`; + badge = t`Needs attention`; + variant = "error"; + } + const progressing = + !activationError && + ((settingUp && !storedFailure) || + (active && !progressError && (!progress || progress.status === "indexing"))); + const isReady = !activationError && active && !progressError && progress?.status === "ready"; + const needsAttention = storedFailure || progressError || progress?.status === "needs_attention"; + return ( + +
+
+
+ {progressing ? : null} +

+ {heading} +

+
+

{detail}

+
+ + {isReady ? +
+
+ ); +} + +function ConfirmationDialog({ + open, + retry, + pending, + onOpenChange, + onConfirm, +}: { + open: boolean; + retry: boolean; + pending: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; +}) { + const { t } = useLingui(); + const [drainAcknowledged, setDrainAcknowledged] = React.useState(false); + React.useEffect(() => { + if (!open) setDrainAcknowledged(false); + }, [open]); + const confirm = () => { + if (!drainAcknowledged || pending) return; + onConfirm(); + }; + return ( + onOpenChange(false)} + title={retry ? t`Retry setup?` : t`Turn on media usage tracking?`} + description={ + retry + ? t`EmDash will continue setup and resume scanning existing content.` + : t`EmDash will scan existing content to show where media is used.` + } + confirmLabel={retry ? t`Retry setup` : t`Turn on`} + pendingLabel={retry ? t`Retrying…` : t`Turning on…`} + variant="primary" + compact + preventCloseWhilePending + confirmDisabled={!drainAcknowledged} + isPending={pending} + error={null} + onConfirm={confirm} + > +
+
+

{t`Before you continue:`}

+
    +
  • {t`Finish any current edits.`}
  • +
  • {t`Pause other tools that update content.`}
  • +
  • {t`Wait for updates already in progress to finish.`}
  • +
+
+

+ {t`Keep this page open until setup finishes. If you leave, return to continue where it stopped.`} +

+ +
+
+ ); +} + +function LoadingPage({ title, description }: { title: string; description: string }) { + const { t } = useLingui(); + return ( + + + +
+ + {t`Loading media usage tracking settings…`} +
+
+
+
+ ); +} + +function MessagePage({ + title, + description, + message, + action, +}: { + title: string; + description: string; + message: string; + action?: React.ReactNode; +}) { + const { t } = useLingui(); + return ( + + + + + + + + ); +} + +function isActivationError(error: unknown, kind: MediaUsageActivationRequestError["kind"]) { + return error instanceof MediaUsageActivationRequestError && error.kind === kind; +} diff --git a/packages/admin/src/lib/api/media-usage-activation.ts b/packages/admin/src/lib/api/media-usage-activation.ts new file mode 100644 index 0000000000..15df6cc365 --- /dev/null +++ b/packages/admin/src/lib/api/media-usage-activation.ts @@ -0,0 +1,220 @@ +import { API_BASE, apiFetch } from "./client.js"; + +const ACTIVATION_URL = `${API_BASE}/admin/media-usage/activation`; + +export const MEDIA_USAGE_ACTIVATION_QUERY_KEY = ["media-usage-activation"] as const; +export const MEDIA_USAGE_PROGRESS_QUERY_KEY = ["media-usage-progress"] as const; + +export interface MediaUsageActivationStatus { + state: "expanded" | "activating" | "active"; + collectionCursor: string | null; + attemptCount: number; + drainConfirmedAt: string | null; + lastAttemptedAt: string | null; + lastErrorCode: "MEDIA_USAGE_ACTIVATION_FAILED" | null; + leaseExpiresAt: string | null; + activatedAt: string | null; + updatedAt: string; +} + +export interface MediaUsageActivationAdvanceResponse { + outcome: "activating" | "active"; + processedCollections: number; + activation: MediaUsageActivationStatus; +} + +export interface MediaUsageProgress { + status: "indexing" | "ready" | "needs_attention"; + readyCollections: number; + totalCollections: number; +} + +export interface MediaUsageProgressAdvanceResponse { + activation: MediaUsageActivationStatus; + progress: MediaUsageProgress | null; + nextRequestInMs: 0 | 30_000 | null; +} + +export type MediaUsageActivationErrorKind = + | "busy" + | "ownership_conflict" + | "version_mismatch" + | "denied" + | "validation" + | "read_failure" + | "advance_failure" + | "unknown"; + +export class MediaUsageActivationRequestError extends Error { + constructor( + readonly kind: MediaUsageActivationErrorKind, + readonly status: number | null, + ) { + super("Media usage activation request failed"); + this.name = "MediaUsageActivationRequestError"; + } +} + +export async function fetchMediaUsageActivationStatus(): Promise { + const response = await activationFetch(ACTIVATION_URL); + if (!response.ok) throw await parseActivationError(response); + const data = await readSuccessData(response); + if (!isActivationStatus(data)) throw unknownResponse(response.status); + return data; +} + +export async function fetchMediaUsageProgress(): Promise { + const response = await activationFetch(`${API_BASE}/admin/media-usage/progress`); + if (!response.ok) throw await parseActivationError(response); + const data = await readSuccessData(response); + if (!isMediaUsageProgress(data)) throw unknownResponse(response.status); + return data; +} + +export async function advanceMediaUsageProgress(): Promise { + const response = await activationFetch(`${API_BASE}/admin/media-usage/progress`, { + method: "POST", + }); + if (!response.ok) throw await parseActivationError(response); + const data = await readSuccessData(response); + if (!isProgressAdvanceResponse(data)) throw unknownResponse(response.status); + return data; +} + +export async function advanceMediaUsageActivation(input: { + writersDrained: true; +}): Promise { + const response = await activationFetch(ACTIVATION_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + writersDrained: input.writersDrained, + }), + }); + if (!response.ok) throw await parseActivationError(response); + const data = await readSuccessData(response); + if (!isAdvanceResponse(data)) throw unknownResponse(response.status); + return data; +} + +async function activationFetch(input: string, init?: RequestInit): Promise { + try { + return await apiFetch(input, init); + } catch { + throw unknownResponse(null); + } +} + +async function parseActivationError(response: Response): Promise { + if (response.status === 401 || response.status === 403) { + return new MediaUsageActivationRequestError("denied", response.status); + } + + const body = await readJson(response); + const error = isRecord(body) && isRecord(body.error) ? body.error : null; + const code = error && typeof error.code === "string" ? error.code : null; + + switch (code) { + case "MEDIA_USAGE_ACTIVATION_BUSY": + return new MediaUsageActivationRequestError("busy", response.status); + case "MEDIA_USAGE_ACTIVATION_CONFLICT": + return new MediaUsageActivationRequestError("ownership_conflict", response.status); + case "MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH": + return new MediaUsageActivationRequestError("version_mismatch", response.status); + case "UNAUTHORIZED": + case "FORBIDDEN": + case "INSUFFICIENT_SCOPE": + return new MediaUsageActivationRequestError("denied", response.status); + case "VALIDATION_ERROR": + return new MediaUsageActivationRequestError("validation", response.status); + case "MEDIA_USAGE_ACTIVATION_READ_ERROR": + return new MediaUsageActivationRequestError("read_failure", response.status); + case "MEDIA_USAGE_ACTIVATION_ADVANCE_ERROR": + case "MEDIA_USAGE_PROGRESS_ADVANCE_ERROR": + return new MediaUsageActivationRequestError("advance_failure", response.status); + default: + return unknownResponse(response.status); + } +} + +function unknownResponse(status: number | null): MediaUsageActivationRequestError { + return new MediaUsageActivationRequestError("unknown", status); +} + +async function readSuccessData(response: Response): Promise { + const body = await readJson(response); + if (!isRecord(body) || body.success !== true || !("data" in body)) return undefined; + return body.data; +} + +async function readJson(response: Response): Promise { + return response.json().catch(() => undefined); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === "string"; +} + +function isActivationStatus(value: unknown): value is MediaUsageActivationStatus { + if (!isRecord(value)) return false; + return ( + (value.state === "expanded" || value.state === "activating" || value.state === "active") && + isNullableString(value.collectionCursor) && + Number.isInteger(value.attemptCount) && + typeof value.attemptCount === "number" && + value.attemptCount >= 0 && + isNullableString(value.drainConfirmedAt) && + isNullableString(value.lastAttemptedAt) && + (value.lastErrorCode === null || value.lastErrorCode === "MEDIA_USAGE_ACTIVATION_FAILED") && + isNullableString(value.leaseExpiresAt) && + isNullableString(value.activatedAt) && + typeof value.updatedAt === "string" + ); +} + +function isAdvanceResponse(value: unknown): value is MediaUsageActivationAdvanceResponse { + if (!isRecord(value) || !isActivationStatus(value.activation)) return false; + return ( + (value.outcome === "activating" || value.outcome === "active") && + value.activation.state === value.outcome && + typeof value.processedCollections === "number" && + Number.isInteger(value.processedCollections) && + value.processedCollections >= 0 && + value.processedCollections <= 1 + ); +} + +function isMediaUsageProgress(value: unknown): value is MediaUsageProgress { + if ( + !isRecord(value) || + (value.status !== "indexing" && value.status !== "ready" && value.status !== "needs_attention") + ) + return false; + const ready = value.readyCollections; + const total = value.totalCollections; + if ( + typeof ready !== "number" || + !Number.isSafeInteger(ready) || + ready < 0 || + typeof total !== "number" || + !Number.isSafeInteger(total) || + total < ready + ) + return false; + return value.status !== "ready" || ready === total; +} + +function isProgressAdvanceResponse(value: unknown): value is MediaUsageProgressAdvanceResponse { + if (!isRecord(value) || !isActivationStatus(value.activation)) return false; + if (value.progress !== null && !isMediaUsageProgress(value.progress)) return false; + if ((value.activation.state === "active") !== (value.progress !== null)) return false; + return ( + value.nextRequestInMs === 0 || + value.nextRequestInMs === 30_000 || + value.nextRequestInMs === null + ); +} diff --git a/packages/admin/src/router.tsx b/packages/admin/src/router.tsx index 95e5350d2b..f5e4328d48 100644 --- a/packages/admin/src/router.tsx +++ b/packages/admin/src/router.tsx @@ -62,6 +62,7 @@ import { ApiTokenSettings } from "./components/settings/ApiTokenSettings"; import { BackupSettings } from "./components/settings/BackupSettings"; import { EmailSettings } from "./components/settings/EmailSettings"; import { GeneralSettings } from "./components/settings/GeneralSettings"; +import { MediaUsageSettings } from "./components/settings/MediaUsageSettings"; import { SecuritySettings } from "./components/settings/SecuritySettings"; import { SeoSettings } from "./components/settings/SeoSettings"; import { SocialSettings } from "./components/settings/SocialSettings"; @@ -1850,6 +1851,12 @@ const settingsRoute = createRoute({ component: Settings, }); +const mediaUsageSettingsRoute = createRoute({ + getParentRoute: () => adminLayoutRoute, + path: "/settings/media-usage", + component: MediaUsageSettings, +}); + // Security settings route const securitySettingsRoute = createRoute({ getParentRoute: () => adminLayoutRoute, @@ -2432,6 +2439,7 @@ const adminRoutes = adminLayoutRoute.addChildren([ bylineSchemaRoute, widgetsRoute, settingsRoute, + mediaUsageSettingsRoute, generalSettingsRoute, socialSettingsRoute, seoSettingsRoute, diff --git a/packages/admin/tests/components/MediaLibrary.test.tsx b/packages/admin/tests/components/MediaLibrary.test.tsx index c7290a7421..add56ee9f2 100644 --- a/packages/admin/tests/components/MediaLibrary.test.tsx +++ b/packages/admin/tests/components/MediaLibrary.test.tsx @@ -6,6 +6,10 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { MediaLibrary } from "../../src/components/MediaLibrary"; import type { LocalMediaItem, MediaFolder, MediaItem, MediaProviderItem } from "../../src/lib/api"; import { ApiResponseError, deleteMedia } from "../../src/lib/api"; +import { + MEDIA_USAGE_ACTIVATION_QUERY_KEY, + MEDIA_USAGE_PROGRESS_QUERY_KEY, +} from "../../src/lib/api/media-usage-activation.js"; import { render } from "../utils/render.tsx"; const dndState = vi.hoisted(() => ({ @@ -73,6 +77,26 @@ vi.mock("../../src/components/RouterLinkButton.js", () => ({ const UPLOAD_CTA_PATTERN = /Upload images, videos, and documents to keep reusable assets/; const UPLOAD_TO_LIBRARY_PATTERN = /Upload to Library/; const UPLOAD_FILES_PATTERN = /Upload Files/; +const setupMocks = vi.hoisted(() => ({ + fetchStatus: vi.fn(), + fetchProgress: vi.fn(), + role: 40, +})); + +vi.mock("../../src/lib/api/media-usage-activation.js", async () => { + const actual = await vi.importActual< + typeof import("../../src/lib/api/media-usage-activation.js") + >("../../src/lib/api/media-usage-activation.js"); + return { + ...actual, + fetchMediaUsageActivationStatus: setupMocks.fetchStatus, + fetchMediaUsageProgress: setupMocks.fetchProgress, + }; +}); + +vi.mock("../../src/lib/api/current-user.js", () => ({ + useCurrentUser: () => ({ data: { id: "user-1", role: setupMocks.role } }), +})); function setInputFiles(input: HTMLInputElement, files: File[]) { const transfer = new DataTransfer(); @@ -115,17 +139,20 @@ vi.mock("../../src/lib/api", async () => { }); function QueryWrapper({ children }: { children: React.ReactNode }) { - const qc = new QueryClient({ + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, }); return ( - {children} + {children} ); } -function renderLibrary(props: Partial> = {}) { +function renderLibrary( + props: Partial> = {}, + queryClient?: QueryClient, +) { const defaultProps: React.ComponentProps = { items: [], isLoading: false, @@ -134,10 +161,13 @@ function renderLibrary(props: Partial> onItemUpdated: vi.fn(), ...props, }; + const library = ; return render( - - - , + queryClient ? ( + {library} + ) : ( + {library} + ), ); } @@ -187,6 +217,131 @@ describe("MediaLibrary", () => { beforeEach(() => { vi.clearAllMocks(); dndState.props = null; + setupMocks.role = 40; + setupMocks.fetchStatus.mockResolvedValue({ state: "active" }); + setupMocks.fetchProgress.mockResolvedValue({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + }); + + describe("Media Usage setup discovery", () => { + it("shows an administrator one setup action while activation is off", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockResolvedValue({ state: "expanded" }); + + const screen = await renderLibrary(); + + await expect.element(screen.getByText("Set up media usage tracking")).toBeInTheDocument(); + await expect.element(screen.getByRole("link", { name: "Open setup" })).toBeInTheDocument(); + }); + + it("links to the automatic setup status while activation is running", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockResolvedValue({ state: "activating" }); + + const screen = await renderLibrary(); + + await expect + .element(screen.getByText("Media usage tracking is setting up")) + .toBeInTheDocument(); + await expect.element(screen.getByRole("link", { name: "View setup" })).toBeInTheDocument(); + }); + + it("keeps the setup link while existing content is indexing", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockResolvedValue({ state: "active" }); + setupMocks.fetchProgress.mockResolvedValue({ + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + + const screen = await renderLibrary(); + + await expect + .element(screen.getByText("Media usage tracking is indexing existing content")) + .toBeVisible(); + await expect.element(screen.getByRole("link", { name: "View setup" })).toBeVisible(); + }); + + it("waits for progress before labelling an active site as incomplete", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockResolvedValue({ state: "active" }); + let finishProgress!: (value: { + status: "indexing"; + readyCollections: number; + totalCollections: number; + }) => void; + setupMocks.fetchProgress.mockImplementation( + () => new Promise((resolve) => (finishProgress = resolve)), + ); + + const screen = await renderLibrary(); + + await vi.waitFor(() => expect(setupMocks.fetchProgress).toHaveBeenCalledOnce()); + expect(screen.getByRole("link", { name: "View setup" }).query()).toBeNull(); + expect( + screen.getByText("Media usage tracking is indexing existing content").query(), + ).toBeNull(); + + finishProgress({ status: "indexing", readyCollections: 1, totalCollections: 2 }); + await expect.element(screen.getByRole("link", { name: "View setup" })).toBeVisible(); + }); + + it("does not label an unreadable progress state as indexing", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockResolvedValue({ state: "active" }); + setupMocks.fetchProgress.mockRejectedValue(new Error("status unavailable")); + + const screen = await renderLibrary(); + + await expect.element(screen.getByText("Media usage tracking needs attention")).toBeVisible(); + await expect.element(screen.getByRole("link", { name: "View setup" })).toBeVisible(); + }); + + it("refreshes fresh cached indexing state when the library mounts", async () => { + setupMocks.role = 50; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + queryClient.setQueryData(MEDIA_USAGE_ACTIVATION_QUERY_KEY, { state: "active" }); + queryClient.setQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY, { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + setupMocks.fetchStatus.mockResolvedValue({ state: "active" }); + setupMocks.fetchProgress.mockResolvedValue({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + + const screen = await renderLibrary({}, queryClient); + + await vi.waitFor(() => expect(setupMocks.fetchStatus).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(setupMocks.fetchProgress).toHaveBeenCalledOnce()); + await expect + .element(screen.getByRole("link", { name: "View setup" })) + .not.toBeInTheDocument(); + }); + + it("keeps the library usable when optional setup discovery fails", async () => { + setupMocks.role = 50; + setupMocks.fetchStatus.mockRejectedValue(new Error("status unavailable")); + + const screen = await renderLibrary({ + items: [makeMediaItem({ id: "1", filename: "still-usable.jpg" })], + }); + + await expect.element(screen.getByAltText("still-usable.jpg")).toBeInTheDocument(); + await expect + .element(screen.getByRole("button", { name: UPLOAD_FILES_PATTERN })) + .toBeInTheDocument(); + expect(screen.getByText("Set up media usage tracking").query()).toBeNull(); + }); }); describe("rendering items", () => { diff --git a/packages/admin/tests/components/Settings.test.tsx b/packages/admin/tests/components/Settings.test.tsx index 317f98dd28..71dc10e548 100644 --- a/packages/admin/tests/components/Settings.test.tsx +++ b/packages/admin/tests/components/Settings.test.tsx @@ -22,6 +22,7 @@ vi.mock("@tanstack/react-router", async () => { const mockFetchManifest = vi.fn<() => Promise>(); const mockSetLocale = vi.fn<(locale: string) => void>(); +const currentUser = vi.hoisted(() => ({ role: 50 })); vi.mock("../../src/lib/api", async () => { const actual = await vi.importActual("../../src/lib/api"); @@ -35,6 +36,13 @@ vi.mock("../../src/locales/useLocale.js", () => ({ useLocale: () => ({ locale: "en", setLocale: mockSetLocale }), })); +vi.mock("../../src/lib/api/current-user.js", () => ({ + useCurrentUser: () => ({ + data: { id: "user-1", email: "admin@example.com", role: currentUser.role }, + isLoading: false, + }), +})); + // Import after mocks const { Settings } = await import("../../src/components/Settings"); @@ -57,6 +65,7 @@ function Wrapper({ children }: { children: React.ReactNode }) { describe("Settings", () => { beforeEach(() => { vi.clearAllMocks(); + currentUser.role = 50; mockFetchManifest.mockResolvedValue(defaultManifest); }); @@ -92,6 +101,25 @@ describe("Settings", () => { await expect.element(screen.getByText("Email", { exact: true })).toBeInTheDocument(); }); + it("shows media usage tracking setup only to administrators", async () => { + const admin = await render( + + + , + ); + await expect + .element(admin.getByRole("link", { name: /Media usage tracking/ })) + .toBeInTheDocument(); + + currentUser.role = 40; + const editor = await render( + + + , + ); + expect(editor.getByRole("link", { name: /Media usage tracking/ }).query()).toBeNull(); + }); + it("groups settings into clear semantic sections", async () => { const screen = await render( diff --git a/packages/admin/tests/components/SetupWizard.test.tsx b/packages/admin/tests/components/SetupWizard.test.tsx index c3042951bd..d22a61a0bc 100644 --- a/packages/admin/tests/components/SetupWizard.test.tsx +++ b/packages/admin/tests/components/SetupWizard.test.tsx @@ -105,6 +105,16 @@ describe("SetupWizard", () => { await expect.element(screen.getByText("Create your account")).toBeInTheDocument(); }); + it("does not present media usage tracking as an onboarding choice", async () => { + const screen = await render( + + + , + ); + await expect.element(screen.getByText("Set up your site")).toBeInTheDocument(); + expect(screen.getByRole("checkbox", { name: /Track where media is used/ }).query()).toBeNull(); + }); + it("admin step shows email input", async () => { const screen = await render( diff --git a/packages/admin/tests/components/settings/MediaUsageSettings.test.tsx b/packages/admin/tests/components/settings/MediaUsageSettings.test.tsx new file mode 100644 index 0000000000..2cc5a5b119 --- /dev/null +++ b/packages/admin/tests/components/settings/MediaUsageSettings.test.tsx @@ -0,0 +1,932 @@ +import { i18n } from "@lingui/core"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import * as React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { userEvent } from "vitest/browser"; + +import { render } from "../../utils/render.tsx"; + +const activationMocks = vi.hoisted(() => ({ + fetchStatus: vi.fn(), + fetchProgress: vi.fn(), + advance: vi.fn(), + advanceProgress: vi.fn(), +})); +const currentUserMock = vi.hoisted(() => vi.fn()); + +vi.mock("../../../src/lib/api/media-usage-activation.js", async () => { + const actual = await vi.importActual< + typeof import("../../../src/lib/api/media-usage-activation.js") + >("../../../src/lib/api/media-usage-activation.js"); + return { + ...actual, + fetchMediaUsageActivationStatus: activationMocks.fetchStatus, + fetchMediaUsageProgress: activationMocks.fetchProgress, + advanceMediaUsageActivation: activationMocks.advance, + advanceMediaUsageProgress: activationMocks.advanceProgress, + }; +}); + +vi.mock("../../../src/lib/api/current-user.js", () => ({ + useCurrentUser: currentUserMock, +})); + +vi.mock("../../../src/components/settings/BackToSettingsLink.js", () => ({ + BackToSettingsLink: () => Back to Settings, +})); + +const { + MEDIA_USAGE_ACTIVATION_QUERY_KEY, + MEDIA_USAGE_PROGRESS_QUERY_KEY, + MediaUsageActivationRequestError, +} = await import("../../../src/lib/api/media-usage-activation.js"); +const { MediaUsageSettings } = + await import("../../../src/components/settings/MediaUsageSettings.js"); + +type ActivationState = "expanded" | "activating" | "active"; +const DRAIN_ACKNOWLEDGEMENT = + "I’ve completed these steps and understand tracking can’t be turned off."; + +function status(state: ActivationState, options: { failed?: boolean } = {}) { + return { + state, + collectionCursor: state === "activating" ? "posts" : null, + attemptCount: state === "expanded" ? 0 : 1, + drainConfirmedAt: state === "expanded" ? null : "2026-08-16T09:00:00.000Z", + lastAttemptedAt: state === "expanded" ? null : "2026-08-16T09:00:00.000Z", + lastErrorCode: options.failed ? ("MEDIA_USAGE_ACTIVATION_FAILED" as const) : null, + leaseExpiresAt: null, + activatedAt: state === "active" ? "2026-08-16T09:00:01.000Z" : null, + updatedAt: "2026-08-16T09:00:01.000Z", + }; +} + +function setCurrentUser(role: number | null, isLoading = false) { + currentUserMock.mockReturnValue({ + data: role === null ? null : { id: "user-1", email: "admin@example.com", role }, + isLoading, + }); +} + +function createQueryClient() { + return new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); +} + +async function renderPage(queryClient = createQueryClient()) { + const screen = await render( + + + , + ); + return { queryClient, screen }; +} + +async function openConfirmation(screen: Awaited>["screen"]) { + await userEvent.click(screen.getByRole("button", { name: "Enable tracking" })); + await expect + .element(screen.getByRole("dialog", { name: "Turn on media usage tracking?" })) + .toBeVisible(); + const dialog = screen.getByRole("dialog", { name: "Turn on media usage tracking?" }); + await expect + .element(dialog.getByText("EmDash will scan existing content to show where media is used.")) + .toBeVisible(); + await expect.element(dialog.getByText("Before you continue:")).toBeVisible(); + await expect.element(dialog.getByRole("list")).toBeVisible(); + for (const step of [ + "Finish any current edits.", + "Pause other tools that update content.", + "Wait for updates already in progress to finish.", + ]) { + await expect.element(dialog.getByText(step)).toBeVisible(); + } + await expect + .element( + dialog.getByText( + "Keep this page open until setup finishes. If you leave, return to continue where it stopped.", + ), + ) + .toBeVisible(); +} + +async function submitConfirmation(screen: Awaited>["screen"]) { + const dialog = screen.getByRole("dialog", { name: "Turn on media usage tracking?" }); + const acknowledgement = dialog.getByRole("checkbox", { name: DRAIN_ACKNOWLEDGEMENT }); + acknowledgement.element().focus(); + await userEvent.keyboard(" "); + const confirm = dialog.getByRole("button", { name: "Turn on" }); + confirm.element().focus(); + await userEvent.keyboard("{Enter}"); +} + +describe("MediaUsageSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + setCurrentUser(50); + activationMocks.fetchStatus.mockResolvedValue(status("expanded")); + activationMocks.fetchProgress.mockResolvedValue({ + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("denies a direct Editor visit before requesting activation status", async () => { + setCurrentUser(40); + + const { screen } = await renderPage(); + + await expect + .element(screen.getByRole("heading", { name: "Access denied" })) + .toBeInTheDocument(); + expect(activationMocks.fetchStatus).not.toHaveBeenCalled(); + expect(screen.getByRole("button", { name: "Enable tracking" }).query()).toBeNull(); + }); + + it("requires writer-drain acknowledgement before enabling tracking", async () => { + const activating = status("activating"); + activationMocks.advance.mockResolvedValue({ + outcome: "activating", + processedCollections: 1, + activation: activating, + }); + const { screen } = await renderPage(); + + await expect.element(screen.getByText("Media usage tracking is off")).toBeInTheDocument(); + expect(screen.getByRole("checkbox").query()).toBeNull(); + await openConfirmation(screen); + const dialog = screen.getByRole("dialog", { name: "Turn on media usage tracking?" }); + const acknowledgement = dialog.getByRole("checkbox", { name: DRAIN_ACKNOWLEDGEMENT }); + const confirm = dialog.getByRole("button", { name: "Turn on" }); + await expect.element(acknowledgement).not.toBeChecked(); + await expect.element(confirm).toBeDisabled(); + expect(activationMocks.advance).not.toHaveBeenCalled(); + acknowledgement.element().focus(); + await userEvent.keyboard(" "); + await expect.element(acknowledgement).toBeChecked(); + await expect.element(confirm).toBeEnabled(); + confirm.element().focus(); + await userEvent.keyboard("{Enter}"); + + await expect + .element(screen.getByRole("heading", { name: "Indexing existing content" })) + .toBeInTheDocument(); + expect(screen.getByRole("button", { name: /setup/i }).query()).toBeNull(); + expect(screen.getByRole("dialog").query()).toBeNull(); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + expect(activationMocks.advance).toHaveBeenCalledWith({ + writersDrained: true, + }); + expect(activationMocks.fetchStatus).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + }); + + it("reads durable Ready without advancing when an already-ready page opens", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.fetchProgress.mockResolvedValue({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + + const { queryClient, screen } = await renderPage(); + + await expect.element(screen.getByRole("heading", { name: "Status" })).toBeVisible(); + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(activationMocks.fetchProgress).toHaveBeenCalledOnce(); + expect(activationMocks.advanceProgress).not.toHaveBeenCalled(); + expect(queryClient.getQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY)).toEqual( + expect.objectContaining({ status: "ready" }), + ); + }); + + it("keeps cached Ready visible while revisit status reads are in flight", async () => { + let finishStatus!: (value: ReturnType) => void; + activationMocks.fetchStatus.mockImplementation( + () => new Promise((resolve) => (finishStatus = resolve)), + ); + activationMocks.fetchProgress.mockResolvedValue({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + const queryClient = createQueryClient(); + queryClient.setQueryData(MEDIA_USAGE_ACTIVATION_QUERY_KEY, status("active")); + queryClient.setQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY, { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + + const { screen } = await renderPage(queryClient); + + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(screen.getByRole("heading", { name: "Indexing existing content" }).query()).toBeNull(); + expect(activationMocks.advanceProgress).not.toHaveBeenCalled(); + + finishStatus(status("active")); + await vi.waitFor(() => expect(activationMocks.fetchProgress).toHaveBeenCalledOnce()); + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(activationMocks.advanceProgress).not.toHaveBeenCalled(); + }); + + it("does not claim indexing while the first durable progress read is pending", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + let finishProgress!: (value: { + status: "ready"; + readyCollections: number; + totalCollections: number; + }) => void; + activationMocks.fetchProgress.mockImplementation( + () => new Promise((resolve) => (finishProgress = resolve)), + ); + + const { screen } = await renderPage(); + + await vi.waitFor(() => expect(activationMocks.fetchProgress).toHaveBeenCalledOnce()); + await expect.element(screen.getByText("Loading media usage tracking settings…")).toBeVisible(); + expect(screen.getByRole("heading", { name: "Indexing existing content" }).query()).toBeNull(); + expect(activationMocks.advanceProgress).not.toHaveBeenCalled(); + + finishProgress({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + }); + + it("follows an existing continuation even when the page already shows Ready", async () => { + vi.useFakeTimers(); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: 0, + }) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + + await renderPage(); + await vi.advanceTimersByTimeAsync(0); + + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2)); + }); + + it("blocks duplicate activation and renders the confirmed active state", async () => { + let resolveAdvance!: (value: unknown) => void; + activationMocks.advance.mockImplementation( + () => new Promise((resolve) => (resolveAdvance = resolve)), + ); + const { screen } = await renderPage(); + await openConfirmation(screen); + + await submitConfirmation(screen); + const pending = screen.getByRole("button", { name: "Turning on…" }); + await expect.element(pending).toBeDisabled(); + pending.element().click(); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + window.dispatchEvent(new PageTransitionEvent("pagehide")); + window.dispatchEvent(new PageTransitionEvent("pageshow")); + expect(activationMocks.fetchStatus).toHaveBeenCalledOnce(); + + resolveAdvance({ outcome: "active", processedCollections: 1, activation: status("active") }); + await expect + .element(screen.getByRole("heading", { name: "Indexing existing content" })) + .toBeInTheDocument(); + expect(screen.getByRole("button", { name: /setup/i }).query()).toBeNull(); + }); + + it("keeps the irreversible confirmation open while activation is pending", async () => { + activationMocks.advance.mockImplementation(() => new Promise(() => {})); + const { screen } = await renderPage(); + await openConfirmation(screen); + + await submitConfirmation(screen); + const dialog = screen.getByRole("dialog", { name: "Turn on media usage tracking?" }); + await expect.element(dialog.getByRole("button", { name: "Cancel" })).toBeDisabled(); + await userEvent.keyboard("{Escape}"); + + await expect.element(dialog).toBeVisible(); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + }); + + it("returns focus to the enable action when the dialog is cancelled", async () => { + const { screen } = await renderPage(); + const trigger = screen.getByRole("button", { name: "Enable tracking" }); + await expect.element(trigger).toBeVisible(); + trigger.element().focus(); + await userEvent.keyboard("{Enter}"); + await expect + .element(screen.getByRole("dialog", { name: "Turn on media usage tracking?" })) + .toBeVisible(); + const cancel = screen.getByRole("button", { name: "Cancel" }); + cancel.element().focus(); + await userEvent.keyboard("{Enter}"); + + await expect.element(screen.getByRole("dialog")).not.toBeInTheDocument(); + expect(document.activeElement).toBe(trigger.element()); + await userEvent.keyboard("{Enter}"); + const reopened = screen.getByRole("dialog", { name: "Turn on media usage tracking?" }); + await expect.element(reopened.getByRole("checkbox")).not.toBeChecked(); + await expect.element(reopened.getByRole("button", { name: "Turn on" })).toBeDisabled(); + }); + + it("closes the confirmation when the page is hidden", async () => { + let visibility: DocumentVisibilityState = "visible"; + vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility); + const { screen } = await renderPage(); + await openConfirmation(screen); + + visibility = "hidden"; + document.dispatchEvent(new Event("visibilitychange")); + await expect.element(screen.getByRole("dialog")).not.toBeInTheDocument(); + }); + + it("keeps a pending confirmation until the return status proves activation started", async () => { + let visibility: DocumentVisibilityState = "visible"; + vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility); + activationMocks.advance.mockImplementation(() => new Promise(() => {})); + activationMocks.fetchStatus + .mockResolvedValueOnce(status("expanded")) + .mockResolvedValueOnce(status("activating")); + const { screen } = await renderPage(); + await openConfirmation(screen); + await submitConfirmation(screen); + + visibility = "hidden"; + document.dispatchEvent(new Event("visibilitychange")); + await new Promise((resolve) => window.setTimeout(resolve, 300)); + expect( + screen.getByRole("dialog", { name: "Turn on media usage tracking?" }).query(), + ).not.toBeNull(); + + visibility = "visible"; + document.dispatchEvent(new Event("visibilitychange")); + await vi.waitFor(() => expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2)); + await expect.element(screen.getByRole("dialog")).not.toBeInTheDocument(); + }); + + it("recovers an ambiguous activation response with one status read", async () => { + activationMocks.advance.mockRejectedValue( + new MediaUsageActivationRequestError("advance_failure", 500), + ); + const { screen } = await renderPage(); + await openConfirmation(screen); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + + await submitConfirmation(screen); + + await expect + .element(screen.getByRole("heading", { name: "Indexing existing content" })) + .toBeInTheDocument(); + expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + }); + + it("keeps writers paused when an ambiguous response cannot be confirmed", async () => { + activationMocks.advance.mockRejectedValue( + new MediaUsageActivationRequestError("advance_failure", 500), + ); + const { screen } = await renderPage(); + await openConfirmation(screen); + activationMocks.fetchStatus.mockRejectedValue( + new MediaUsageActivationRequestError("read_failure", 500), + ); + + await submitConfirmation(screen); + + await expect + .element( + screen.getByText( + "Activation cannot be confirmed. Keep editing paused and refresh the status.", + ), + ) + .toBeInTheDocument(); + expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2); + }); + + it("keeps validation blocking after leaving and returning", async () => { + activationMocks.advance.mockRejectedValue( + new MediaUsageActivationRequestError("validation", 400), + ); + const { screen } = await renderPage(); + await openConfirmation(screen); + await submitConfirmation(screen); + await expect + .element(screen.getByText("Reload after updating EmDash before trying again.")) + .toBeInTheDocument(); + + activationMocks.fetchStatus.mockResolvedValue(status("activating")); + window.dispatchEvent(new PageTransitionEvent("pagehide")); + window.dispatchEvent(new PageTransitionEvent("pageshow")); + + await expect + .element(screen.getByText("Reload after updating EmDash before trying again.")) + .toBeInTheDocument(); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + }); + + it("does not move focus when an already-active page first loads", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + + const { screen } = await renderPage(); + const heading = screen.getByRole("heading", { name: "Indexing existing content" }); + await expect.element(heading).toBeInTheDocument(); + expect(document.activeElement).not.toBe(heading.element()); + }); + + it.each([ + ["indexing", "Indexing existing content", "Ready: 1 / 2"], + [ + "ready", + "Media usage tracking is ready", + "Existing content is indexed. New changes are tracked automatically.", + ], + [ + "needs_attention", + "Needs attention", + "Check the server logs, then use the media usage recovery API for the failed work.", + ], + ] as const)("shows %s progress after activation", async (progressStatus, heading, summary) => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { + status: progressStatus, + readyCollections: progressStatus === "ready" ? 2 : 1, + totalCollections: 2, + }, + nextRequestInMs: progressStatus === "needs_attention" ? 0 : null, + }); + + const { screen } = await renderPage(); + + await expect.element(screen.getByRole("heading", { name: heading })).toBeInTheDocument(); + await expect.element(screen.getByText(summary)).toBeInTheDocument(); + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + if (progressStatus === "indexing") { + await expect.element(screen.getByText("Indexing", { exact: true })).toBeVisible(); + } + if (progressStatus === "ready") { + expect(screen.getByText("Ready: 2 / 2").query()).toBeNull(); + } + if (progressStatus === "needs_attention") { + expect(screen.getByRole("button", { name: "Retry setup" }).query()).toBeNull(); + expect(screen.getByRole("button", { name: "Try again" }).query()).toBeNull(); + } + }); + + it("renders progress counts with an incomplete RTL catalog", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + i18n.loadAndActivate({ locale: "ar", messages: {} }); + + try { + const { screen } = await renderPage(); + + await expect.element(screen.getByText("Ready: 1 / 2")).toBeVisible(); + await expect.element(screen.getByText("1 / 2")).toHaveAttribute("dir", "ltr"); + } finally { + i18n.loadAndActivate({ locale: "en", messages: {} }); + } + }); + + it("shows Retry setup only for a stored activation failure", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("activating", { failed: true })); + + const { screen } = await renderPage(); + + await expect.element(screen.getByRole("heading", { name: "Needs attention" })).toBeVisible(); + await userEvent.click(screen.getByRole("button", { name: "Retry setup" })); + const dialog = screen.getByRole("dialog", { name: "Retry setup?" }); + await expect.element(dialog).toBeVisible(); + await expect.element(dialog.getByRole("button", { name: "Retry setup" })).toBeDisabled(); + await expect + .element(dialog.getByRole("checkbox", { name: DRAIN_ACKNOWLEDGEMENT })) + .not.toBeChecked(); + expect(activationMocks.advanceProgress).not.toHaveBeenCalled(); + }); + + it("restarts progress after retrying a failure discovered by the page loop", async () => { + activationMocks.fetchStatus + .mockResolvedValueOnce(status("active")) + .mockResolvedValue(status("activating", { failed: true })); + activationMocks.advanceProgress + .mockRejectedValueOnce(new MediaUsageActivationRequestError("advance_failure", 500)) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + activationMocks.advance.mockResolvedValue({ + outcome: "activating", + processedCollections: 1, + activation: status("activating"), + }); + const { screen } = await renderPage(); + await expect.element(screen.getByRole("button", { name: "Retry setup" })).toBeVisible(); + + await userEvent.click(screen.getByRole("button", { name: "Retry setup" })); + const dialog = screen.getByRole("dialog", { name: "Retry setup?" }); + const acknowledgement = dialog.getByRole("checkbox", { name: DRAIN_ACKNOWLEDGEMENT }); + acknowledgement.element().focus(); + await userEvent.keyboard(" "); + const confirm = dialog.getByRole("button", { name: "Retry setup" }); + confirm.element().focus(); + await userEvent.keyboard("{Enter}"); + + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2); + }); + + it("shows no mutation action while ordinary activation is progressing", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("activating")); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("activating"), + progress: null, + nextRequestInMs: 30_000, + }); + + const { screen } = await renderPage(); + + await expect.element(screen.getByRole("heading", { name: "Setting up" })).toBeVisible(); + expect(screen.getByRole("button", { name: /setup/i }).query()).toBeNull(); + }); + + it("keeps active setup visible when progress cannot be loaded", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress.mockRejectedValue( + new MediaUsageActivationRequestError("advance_failure", 500), + ); + activationMocks.fetchProgress.mockRejectedValue( + new MediaUsageActivationRequestError("read_failure", 500), + ); + + const { screen } = await renderPage(); + + await expect.element(screen.getByRole("heading", { name: "Needs attention" })).toBeVisible(); + await expect.element(screen.getByRole("button", { name: "Try again" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /setup/i }).query()).toBeNull(); + }); + + it("reconciles durable progress before offering a transient retry", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.fetchProgress.mockResolvedValue({ + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + activationMocks.advanceProgress + .mockRejectedValueOnce(new MediaUsageActivationRequestError("advance_failure", 500)) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + + const { screen } = await renderPage(); + + await expect.element(screen.getByRole("button", { name: "Try again" })).toBeVisible(); + expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2); + expect(activationMocks.fetchProgress).toHaveBeenCalledTimes(2); + await userEvent.click(screen.getByRole("button", { name: "Try again" })); + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2); + expect(activationMocks.advance).not.toHaveBeenCalled(); + }); + + it("accepts durable Ready after the final progress response is lost", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.fetchProgress.mockResolvedValue({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + activationMocks.advanceProgress.mockRejectedValue( + new MediaUsageActivationRequestError("advance_failure", 500), + ); + + const { screen } = await renderPage(); + + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + expect(screen.getByRole("button", { name: "Try again" }).query()).toBeNull(); + expect(activationMocks.fetchProgress).toHaveBeenCalledOnce(); + }); + + it("follows another owner after a busy response without another mutation", async () => { + activationMocks.advance.mockRejectedValue(new MediaUsageActivationRequestError("busy", 409)); + activationMocks.fetchStatus + .mockResolvedValueOnce(status("expanded")) + .mockResolvedValue(status("activating")); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("activating"), + progress: null, + nextRequestInMs: 30_000, + }); + const { screen } = await renderPage(); + await openConfirmation(screen); + await submitConfirmation(screen); + + await expect.element(screen.getByRole("heading", { name: "Setting up" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Retry setup" }).query()).toBeNull(); + expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2); + expect(activationMocks.advance).toHaveBeenCalledOnce(); + }); + + it("moves focus to the recovered state after a slow busy status read", async () => { + activationMocks.advance.mockRejectedValue(new MediaUsageActivationRequestError("busy", 409)); + let finishStatus!: (value: ReturnType) => void; + activationMocks.fetchStatus + .mockResolvedValueOnce(status("expanded")) + .mockImplementation(() => new Promise((resolve) => (finishStatus = resolve))); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("activating"), + progress: null, + nextRequestInMs: 30_000, + }); + const { screen } = await renderPage(); + await openConfirmation(screen); + await submitConfirmation(screen); + await expect.element(screen.getByRole("button", { name: "Refresh status" })).toBeVisible(); + + finishStatus(status("activating")); + const heading = screen.getByRole("heading", { name: "Setting up" }); + await expect.element(heading).toBeVisible(); + expect(document.activeElement).toBe(heading.element()); + }); + + it("follows an immediate continuation without the old polling delay", async () => { + vi.useFakeTimers(); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }, + nextRequestInMs: 0, + }) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + const { screen } = await renderPage(); + + await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2)); + await expect + .element(screen.getByRole("heading", { name: "Media usage tracking is ready" })) + .toBeVisible(); + }); + + it("waits for the server-provided delayed continuation", async () => { + vi.useFakeTimers(); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }, + nextRequestInMs: 30_000, + }) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + await renderPage(); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + await vi.advanceTimersByTimeAsync(20_000); + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(10_000); + expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2); + }); + + it("cancels a delayed successor while hidden and resumes after a status read", async () => { + vi.useFakeTimers(); + let visibility: DocumentVisibilityState = "visible"; + vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }, + nextRequestInMs: 30_000, + }) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }); + await renderPage(); + await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + visibility = "hidden"; + document.dispatchEvent(new Event("visibilitychange")); + await vi.advanceTimersByTimeAsync(30_000); + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + + visibility = "visible"; + document.dispatchEvent(new Event("visibilitychange")); + await vi.advanceTimersByTimeAsync(0); + expect(activationMocks.fetchStatus).toHaveBeenCalledTimes(2); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2)); + }); + + it("does not resume from stale activation data when the return status read fails", async () => { + vi.useFakeTimers(); + let visibility: DocumentVisibilityState = "visible"; + vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility); + activationMocks.fetchStatus + .mockResolvedValueOnce(status("active")) + .mockRejectedValue(new MediaUsageActivationRequestError("read_failure", 500)); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { status: "indexing", readyCollections: 1, totalCollections: 2 }, + nextRequestInMs: null, + }); + const { screen } = await renderPage(); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + visibility = "hidden"; + document.dispatchEvent(new Event("visibilitychange")); + visibility = "visible"; + document.dispatchEvent(new Event("visibilitychange")); + await vi.advanceTimersByTimeAsync(0); + + await expect.element(screen.getByRole("heading", { name: "Needs attention" })).toBeVisible(); + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + }); + + it("does not overlap progress requests when visibility returns", async () => { + vi.useFakeTimers(); + let visibility: DocumentVisibilityState = "visible"; + vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + let finishProgress!: (value: { + activation: ReturnType; + progress: { status: "indexing"; readyCollections: number; totalCollections: number }; + nextRequestInMs: 0; + }) => void; + activationMocks.advanceProgress + .mockImplementationOnce(() => new Promise((resolve) => (finishProgress = resolve))) + .mockResolvedValueOnce({ + activation: status("active"), + progress: { status: "ready", readyCollections: 2, totalCollections: 2 }, + nextRequestInMs: null, + }); + await renderPage(); + await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + visibility = "hidden"; + document.dispatchEvent(new Event("visibilitychange")); + visibility = "visible"; + document.dispatchEvent(new Event("visibilitychange")); + await vi.advanceTimersByTimeAsync(0); + + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + finishProgress({ + activation: status("active"), + progress: { status: "indexing", readyCollections: 1, totalCollections: 2 }, + nextRequestInMs: 0, + }); + await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledTimes(2)); + }); + + it("cancels a delayed successor when the page unmounts", async () => { + vi.useFakeTimers(); + activationMocks.fetchStatus.mockResolvedValue(status("active")); + activationMocks.advanceProgress.mockResolvedValue({ + activation: status("active"), + progress: { status: "indexing", readyCollections: 1, totalCollections: 2 }, + nextRequestInMs: 30_000, + }); + const { screen } = await renderPage(); + await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + await screen.unmount(); + await vi.advanceTimersByTimeAsync(30_000); + expect(activationMocks.advanceProgress).toHaveBeenCalledOnce(); + }); + + it("keeps shared progress truthful when an in-flight step finishes after unmount", async () => { + activationMocks.fetchStatus.mockResolvedValue(status("active")); + let finishProgress!: (value: { + activation: ReturnType; + progress: { status: "ready"; readyCollections: number; totalCollections: number }; + nextRequestInMs: null; + }) => void; + activationMocks.advanceProgress.mockImplementation( + () => new Promise((resolve) => (finishProgress = resolve)), + ); + const { queryClient, screen } = await renderPage(); + await vi.waitFor(() => expect(activationMocks.advanceProgress).toHaveBeenCalledOnce()); + + await screen.unmount(); + finishProgress({ + activation: status("active"), + progress: { status: "ready", readyCollections: 2, totalCollections: 2 }, + nextRequestInMs: null, + }); + + await vi.waitFor(() => + expect(queryClient.getQueryData(MEDIA_USAGE_PROGRESS_QUERY_KEY)).toEqual( + expect.objectContaining({ status: "ready" }), + ), + ); + }); +}); diff --git a/packages/admin/tests/lib/media-usage-activation-api.test.ts b/packages/admin/tests/lib/media-usage-activation-api.test.ts new file mode 100644 index 0000000000..ba5abdf101 --- /dev/null +++ b/packages/admin/tests/lib/media-usage-activation-api.test.ts @@ -0,0 +1,262 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + MediaUsageActivationRequestError, + advanceMediaUsageActivation, + advanceMediaUsageProgress, + fetchMediaUsageActivationStatus, + fetchMediaUsageProgress, +} from "../../src/lib/api/media-usage-activation.js"; + +const activationUrl = "/_emdash/api/admin/media-usage/activation"; +const progressUrl = "/_emdash/api/admin/media-usage/progress"; + +function activationStatus(state: "expanded" | "activating" | "active" = "expanded") { + return { + state, + collectionCursor: state === "activating" ? "posts" : null, + attemptCount: state === "expanded" ? 0 : 1, + drainConfirmedAt: state === "expanded" ? null : "2026-08-16T09:00:00.000Z", + lastAttemptedAt: state === "expanded" ? null : "2026-08-16T09:00:00.000Z", + lastErrorCode: null, + leaseExpiresAt: null, + activatedAt: state === "active" ? "2026-08-16T09:00:01.000Z" : null, + updatedAt: "2026-08-16T09:00:01.000Z", + } as const; +} + +function success(data: unknown): Response { + return Response.json({ success: true, data }); +} + +function failure(status: number, code: string, details?: unknown): Response { + return Response.json( + { + success: false, + error: { code, message: "private server detail", ...(details ? { details } : {}) }, + }, + { status }, + ); +} + +async function caught(run: () => Promise): Promise { + const error = await run().catch((value: unknown) => value); + expect(error).toBeInstanceOf(MediaUsageActivationRequestError); + return error as MediaUsageActivationRequestError; +} + +describe("media usage activation admin API", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("reads and validates every public activation status field", async () => { + const data = activationStatus("activating"); + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + + await expect(fetchMediaUsageActivationStatus()).resolves.toEqual(data); + expect(fetch).toHaveBeenCalledOnce(); + expect(fetch.mock.calls[0]?.[0]).toBe(activationUrl); + const headers = new Headers(fetch.mock.calls[0]?.[1]?.headers); + expect(headers.get("X-EmDash-Request")).toBe("1"); + }); + + it("reads validated aggregate indexing progress", async () => { + const data = { + status: "indexing", + readyCollections: 0, + totalCollections: 2, + } as const; + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + + await expect(fetchMediaUsageProgress()).resolves.toEqual(data); + expect(fetch.mock.calls[0]?.[0]).toBe(progressUrl); + }); + + it("accepts indexing while collection cleanup remains after content types are ready", async () => { + const data = { status: "indexing", readyCollections: 2, totalCollections: 2 } as const; + vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + + await expect(fetchMediaUsageProgress()).resolves.toEqual(data); + }); + + it("rejects contradictory aggregate progress", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + success({ status: "ready", readyCollections: 2, totalCollections: 1 }), + ); + + await expect(caught(() => fetchMediaUsageProgress())).resolves.toMatchObject({ + kind: "unknown", + }); + }); + + it("accepts attention when every remaining content type is ready", async () => { + const data = { status: "needs_attention", readyCollections: 1, totalCollections: 1 } as const; + vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + + await expect(fetchMediaUsageProgress()).resolves.toEqual(data); + }); + + it("advances once with only writer confirmation", async () => { + const activation = activationStatus("activating"); + const data = { outcome: "activating", processedCollections: 1, activation }; + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + const input = { + writersDrained: true, + extra: "must not cross the API boundary", + } as const; + + await expect(advanceMediaUsageActivation(input)).resolves.toEqual(data); + expect(fetch).toHaveBeenCalledOnce(); + const [url, init] = fetch.mock.calls[0]!; + expect(url).toBe(activationUrl); + expect(init?.method).toBe("POST"); + const headers = new Headers(init?.headers); + expect(headers.get("Content-Type")).toBe("application/json"); + expect(headers.get("X-EmDash-Request")).toBe("1"); + expect(typeof init?.body).toBe("string"); + const requestBody = typeof init?.body === "string" ? init.body : ""; + expect(JSON.parse(requestBody)).toEqual({ + writersDrained: true, + }); + }); + + it("advances one progress step without a request body", async () => { + const data = { + activation: activationStatus("active"), + progress: { status: "ready", readyCollections: 2, totalCollections: 2 }, + nextRequestInMs: null, + } as const; + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(success(data)); + + await expect(advanceMediaUsageProgress()).resolves.toEqual(data); + const [url, init] = fetch.mock.calls[0]!; + expect(url).toBe(progressUrl); + expect(init?.method).toBe("POST"); + expect(new Headers(init?.headers).get("X-EmDash-Request")).toBe("1"); + expect(init?.body).toBeUndefined(); + }); + + it.each([ + [ + "active activation without progress", + { activation: activationStatus("active"), progress: null }, + ], + [ + "incomplete activation with progress", + { + activation: activationStatus("activating"), + progress: { status: "indexing", readyCollections: 0, totalCollections: 2 }, + }, + ], + ] as const)("rejects %s in a progress advance response", async (_label, value) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(success({ ...value, nextRequestInMs: null })); + + await expect(caught(() => advanceMediaUsageProgress())).resolves.toMatchObject({ + kind: "unknown", + status: 200, + }); + }); + + it.each([ + ["MEDIA_USAGE_ACTIVATION_CONFLICT", "ownership_conflict"], + ["MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH", "version_mismatch"], + ["VALIDATION_ERROR", "validation"], + ["MEDIA_USAGE_ACTIVATION_READ_ERROR", "read_failure"], + ["MEDIA_USAGE_ACTIVATION_ADVANCE_ERROR", "advance_failure"], + ] as const)("maps %s to %s without retaining the server message", async (code, kind) => { + const status = code === "VALIDATION_ERROR" ? 400 : code.includes("ERROR") ? 500 : 409; + vi.spyOn(globalThis, "fetch").mockResolvedValue(failure(status, code)); + + const error = await caught(() => fetchMediaUsageActivationStatus()); + + expect(error).toMatchObject({ kind, status }); + expect(error.message).not.toContain("private server detail"); + }); + + it.each([401, 403])("maps a malformed %s response to denied before parsing", async (status) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("not json", { status, statusText: "private detail" }), + ); + + await expect(caught(() => fetchMediaUsageActivationStatus())).resolves.toMatchObject({ + kind: "denied", + status, + }); + }); + + it.each(["UNAUTHORIZED", "FORBIDDEN", "INSUFFICIENT_SCOPE"])( + "maps %s to denied", + async (code) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(failure(500, code)); + + await expect(caught(() => fetchMediaUsageActivationStatus())).resolves.toMatchObject({ + kind: "denied", + }); + }, + ); + + it("maps a busy response without retaining server details", async () => { + vi.spyOn(globalThis, "fetch") + .mockResolvedValueOnce( + failure(409, "MEDIA_USAGE_ACTIVATION_BUSY", { + leaseExpiresAt: "2026-08-16T09:05:00.000Z", + }), + ) + .mockResolvedValueOnce(failure(409, "MEDIA_USAGE_ACTIVATION_BUSY", { leaseExpiresAt: 123 })); + + await expect(caught(() => fetchMediaUsageActivationStatus())).resolves.toMatchObject({ + kind: "busy", + status: 409, + }); + const malformed = await caught(() => fetchMediaUsageActivationStatus()); + expect(malformed.kind).toBe("busy"); + expect(malformed).not.toHaveProperty("leaseExpiresAt"); + }); + + it.each([ + ["invalid state", { ...activationStatus(), state: "invalid" }], + ["negative attempts", { ...activationStatus(), attemptCount: -1 }], + ["unknown error", failure(418, "UNKNOWN_CODE")], + ["malformed JSON", new Response("not json")], + ] as const)("rejects %s as an unknown read error", async (_label, value) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + value instanceof Response ? value : success(value), + ); + + await expect(caught(() => fetchMediaUsageActivationStatus())).resolves.toMatchObject({ + kind: "unknown", + }); + }); + + it("wraps network failures without retaining their message", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("secret upstream hostname")); + + const error = await caught(() => fetchMediaUsageActivationStatus()); + + expect(error).toMatchObject({ kind: "unknown", status: null }); + expect(error.message).not.toContain("secret upstream hostname"); + }); + + it.each([ + ["negative count", { outcome: "activating", processedCollections: -1 }], + ["count above limit", { outcome: "activating", processedCollections: 2 }], + ["fractional count", { outcome: "activating", processedCollections: 0.5 }], + ["outcome mismatch", { outcome: "active", processedCollections: 1 }], + ["nested expanded", { outcome: "activating", processedCollections: 0, state: "expanded" }], + ] as const)("treats malformed POST success (%s) as unknown", async (_label, shape) => { + const state = + "state" in shape ? shape.state : shape.outcome === "active" ? "activating" : shape.outcome; + vi.spyOn(globalThis, "fetch").mockResolvedValue( + success({ + outcome: shape.outcome, + processedCollections: shape.processedCollections, + activation: activationStatus(state as "expanded" | "activating" | "active"), + }), + ); + + await expect( + caught(() => advanceMediaUsageActivation({ writersDrained: true })), + ).resolves.toMatchObject({ kind: "unknown", status: 200 }); + }); +}); diff --git a/packages/cloudflare/src/worker.ts b/packages/cloudflare/src/worker.ts index d2272d38a3..fed7c3fbff 100644 --- a/packages/cloudflare/src/worker.ts +++ b/packages/cloudflare/src/worker.ts @@ -1,10 +1,9 @@ /** * Cloudflare Worker entry for EmDash sites. * - * Wraps the Astro Cloudflare server handler with a `scheduled()` handler so a - * Cron Triggers drive general maintenance and the separately bounded Media - * Usage lane without request side effects. Re-exports the `PluginBridge` - * Durable Object so the sandbox binding resolves against the entry module. + * Wraps the Astro Cloudflare server handler with a `scheduled()` handler for + * general maintenance. Re-exports the `PluginBridge` Durable Object so the + * sandbox binding resolves against the entry module. * * The `@astrojs/cloudflare/entrypoints/server` import is resolved by the * consuming app's Astro build (it pulls the build-time `virtual:astro:app` @@ -14,7 +13,7 @@ // @ts-ignore - resolved against the consuming app's Astro build import astroHandler from "@astrojs/cloudflare/entrypoints/server"; import { createApp } from "astro/app/entrypoint"; -import { runScheduledMediaUsageTasks, runScheduledTasks } from "emdash/middleware"; +import { runScheduledTasks } from "emdash/middleware"; export { PluginBridge } from "./sandbox/index.js"; @@ -40,59 +39,44 @@ async function invalidatePublishedTags( } /** - * Build a Worker `scheduled()` handler. By default the every-two-minutes - * expression runs Media Usage maintenance and every other expression runs - * general maintenance. Configuring a general expression changes that lane - * from catch-all to exact. + * Build a Worker `scheduled()` handler for general maintenance. */ export interface ScheduledHandlerOptions { generalCron?: string; - mediaUsageCron?: string; } -const DEFAULT_MEDIA_USAGE_CRON = "*/2 * * * *"; - export function createScheduledHandler( options?: ScheduledHandlerOptions, ): ExportedHandlerScheduledHandler { const generalCron = options?.generalCron?.trim(); - const mediaUsageCron = options?.mediaUsageCron?.trim() ?? DEFAULT_MEDIA_USAGE_CRON; - if ((options?.generalCron !== undefined && !generalCron) || !mediaUsageCron) { + if (options?.generalCron !== undefined && !generalCron) { throw new Error("Configured scheduled-handler expressions must be non-empty"); } - if (generalCron === mediaUsageCron) { - throw new Error("General and Media Usage Cron expressions must differ"); - } return (controller, _env, ctx) => { - if (controller.cron === mediaUsageCron) { - ctx.waitUntil( - runScheduledMediaUsageTasks().catch((error: unknown) => { - console.error("[scheduled] Media Usage maintenance failed:", error); - }), - ); - return; - } if (generalCron !== undefined && controller.cron !== generalCron) { console.warn(`[scheduled] Ignoring unexpected Cron expression: ${controller.cron}`); return; } + ctx.waitUntil( - // Invalidate incrementally as each collection batch publishes, so a - // scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a - // large backlog) still purged the cache tags for everything it managed - // to publish — not just whatever completed before a single end-of-sweep - // purge that may never run. - runScheduledTasks({ onPublished: invalidatePublishedTags }) - .then(({ published }) => { + (async () => { + try { + // Invalidate incrementally as each collection batch publishes, so a + // scheduled() invocation killed mid-sweep (CPU/wall-clock limits on a + // large backlog) still purged the cache tags for everything it managed + // to publish — not just whatever completed before a single end-of-sweep + // purge that may never run. + const { published } = await runScheduledTasks({ + onPublished: invalidatePublishedTags, + }); if (published.length > 0) { console.log(`[scheduled] Published ${published.length} scheduled item(s)`); } - return undefined; - }) - .catch((error: unknown) => { + } catch (error) { console.error("[scheduled] runScheduledTasks failed:", error); - }), + } + })(), ); }; } diff --git a/packages/cloudflare/tests/worker-scheduled.test.ts b/packages/cloudflare/tests/worker-scheduled.test.ts index 034844f207..1dc1e4c5c6 100644 --- a/packages/cloudflare/tests/worker-scheduled.test.ts +++ b/packages/cloudflare/tests/worker-scheduled.test.ts @@ -1,98 +1,54 @@ import { beforeEach, expect, it, vi } from "vitest"; const scheduled = vi.hoisted(() => ({ - general: vi.fn(async () => ({ published: [] })), - mediaUsage: vi.fn(async () => ({ outcome: "inactive", taskClass: null, turn: null })), + general: vi.fn(async (_options?: unknown) => ({ published: [] })), })); -vi.mock("@astrojs/cloudflare/entrypoints/server", () => ({ default: { fetch: vi.fn() } })); +vi.mock("@astrojs/cloudflare/entrypoints/server", () => ({ default: {} })); vi.mock("astro/app/entrypoint", () => ({ createApp: () => ({ pipeline: { getCacheProvider: async () => null } }), })); -vi.mock("emdash/middleware", () => ({ - runScheduledTasks: scheduled.general, - runScheduledMediaUsageTasks: scheduled.mediaUsage, -})); +vi.mock("emdash/middleware", () => ({ runScheduledTasks: scheduled.general })); vi.mock("../src/sandbox/index.js", () => ({ PluginBridge: vi.fn() })); import { createScheduledHandler } from "../src/worker.js"; beforeEach(() => { + vi.restoreAllMocks(); scheduled.general.mockClear(); - scheduled.mediaUsage.mockClear(); }); -it("uses the default Media Usage expression and treats every other expression as general", async () => { - const handler = createScheduledHandler(); +it("runs general maintenance for the configured Cron", async () => { + const handler = createScheduledHandler({ generalCron: "* * * * *" }); - await invoke(handler, "custom expression"); - expect(scheduled.general).toHaveBeenCalledOnce(); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + await invoke(handler, "* * * * *"); - scheduled.general.mockClear(); - await invoke(handler, "*/2 * * * *"); - expect(scheduled.general).not.toHaveBeenCalled(); - expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); + expect(scheduled.general).toHaveBeenCalledOnce(); }); -it("dispatches distinct configured cron expressions to exactly one lane", async () => { - const handler = createScheduledHandler({ - generalCron: "* * * * *", - mediaUsageCron: "*/2 * * * *", - }); - - await invoke(handler, "* * * * *"); - expect(scheduled.general).toHaveBeenCalledOnce(); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); +it("ignores unexpected Cron expressions", async () => { + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + const handler = createScheduledHandler({ generalCron: "* * * * *" }); - scheduled.general.mockClear(); - await invoke(handler, "*/2 * * * *"); - expect(scheduled.general).not.toHaveBeenCalled(); - expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); + await invoke(handler, "0 * * * *"); - scheduled.mediaUsage.mockClear(); - await invoke(handler, "0 0 * * *"); expect(scheduled.general).not.toHaveBeenCalled(); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); + expect(warning).toHaveBeenCalledExactlyOnceWith( + "[scheduled] Ignoring unexpected Cron expression: 0 * * * *", + ); }); -it("allows either default expression to be overridden independently", async () => { - const customMedia = createScheduledHandler({ mediaUsageCron: "*/5 * * * *" }); - await invoke(customMedia, "*/5 * * * *"); - expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); - expect(scheduled.general).not.toHaveBeenCalled(); - - scheduled.mediaUsage.mockClear(); - await invoke(customMedia, "15 * * * *"); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); - expect(scheduled.general).toHaveBeenCalledOnce(); +it("runs any configured trigger when no expression is specified", async () => { + const handler = createScheduledHandler(); - scheduled.general.mockClear(); - const customGeneral = createScheduledHandler({ generalCron: "0 * * * *" }); - await invoke(customGeneral, "*/2 * * * *"); - expect(scheduled.mediaUsage).toHaveBeenCalledOnce(); - expect(scheduled.general).not.toHaveBeenCalled(); + await invoke(handler, "custom expression"); - scheduled.mediaUsage.mockClear(); - await invoke(customGeneral, "0 * * * *"); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); expect(scheduled.general).toHaveBeenCalledOnce(); - - scheduled.general.mockClear(); - await invoke(customGeneral, "15 * * * *"); - expect(scheduled.mediaUsage).not.toHaveBeenCalled(); - expect(scheduled.general).not.toHaveBeenCalled(); }); -it("rejects empty or aliased configured expressions", () => { - expect(() => - createScheduledHandler({ generalCron: "* * * * *", mediaUsageCron: "* * * * *" }), - ).toThrow(/must differ/i); - expect(() => createScheduledHandler({ generalCron: "", mediaUsageCron: "*/2 * * * *" })).toThrow( - /non-empty/i, - ); - expect(() => createScheduledHandler({ mediaUsageCron: " " })).toThrow(/non-empty/i); - expect(() => createScheduledHandler({ generalCron: " */2 * * * * " })).toThrow(/must differ/i); +it("rejects an empty configured expression", () => { + expect(() => createScheduledHandler({ generalCron: "" })).toThrow(/non-empty/i); + expect(createScheduledHandler({ generalCron: " * * * * * " })).toBeTypeOf("function"); }); async function invoke(handler: ExportedHandlerScheduledHandler, cron: string): Promise { diff --git a/packages/core/src/api/errors.ts b/packages/core/src/api/errors.ts index c01e4376c9..0b0d8784c6 100644 --- a/packages/core/src/api/errors.ts +++ b/packages/core/src/api/errors.ts @@ -93,6 +93,9 @@ export const ErrorCode = { MEDIA_FOLDER_DELETE_ERROR: "MEDIA_FOLDER_DELETE_ERROR", MEDIA_USAGE_READ_ERROR: "MEDIA_USAGE_READ_ERROR", MEDIA_USAGE_REPAIR_ERROR: "MEDIA_USAGE_REPAIR_ERROR", + MEDIA_USAGE_PROGRESS_READ_ERROR: "MEDIA_USAGE_PROGRESS_READ_ERROR", + MEDIA_USAGE_PROGRESS_ADVANCE_ERROR: "MEDIA_USAGE_PROGRESS_ADVANCE_ERROR", + MEDIA_USAGE_PROGRESS_NOT_ACTIVE: "MEDIA_USAGE_PROGRESS_NOT_ACTIVE", MEDIA_USAGE_WORK_LIST_ERROR: "MEDIA_USAGE_WORK_LIST_ERROR", MEDIA_USAGE_WORK_RETRY_ERROR: "MEDIA_USAGE_WORK_RETRY_ERROR", MEDIA_USAGE_COLLECTION_DELETION_LIST_ERROR: "MEDIA_USAGE_COLLECTION_DELETION_LIST_ERROR", @@ -474,6 +477,7 @@ export function mapErrorStatus(code: string | undefined): number { case ErrorCode.MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH: case ErrorCode.MEDIA_USAGE_ACTIVATION_BUSY: case ErrorCode.MEDIA_USAGE_ACTIVATION_CONFLICT: + case ErrorCode.MEDIA_USAGE_PROGRESS_NOT_ACTIVE: return 409; // 410 Gone diff --git a/packages/core/src/api/handlers/index.ts b/packages/core/src/api/handlers/index.ts index be8049b287..b9026aeec8 100644 --- a/packages/core/src/api/handlers/index.ts +++ b/packages/core/src/api/handlers/index.ts @@ -71,6 +71,8 @@ export { export { aggregateMediaUsageCoverageStatus, handleMediaUsageDetails, + handleMediaUsageProgress, + handleMediaUsageProgressAdvance, handleMediaUsageSummaries, handleMediaUsageRepair, toMediaUsageRepairResponse, @@ -79,6 +81,8 @@ export { type MediaUsageDetailsResponse, type MediaUsageEntryDetail, type MediaUsageOccurrenceDetail, + type MediaUsageProgress, + type MediaUsageProgressAdvanceResponse, type MediaUsageRepairResponse, type MediaUsageSourceDetail, type MediaUsageSummary, diff --git a/packages/core/src/api/handlers/media-usage.ts b/packages/core/src/api/handlers/media-usage.ts index 7e2bd98547..41ad7df0f1 100644 --- a/packages/core/src/api/handlers/media-usage.ts +++ b/packages/core/src/api/handlers/media-usage.ts @@ -8,6 +8,10 @@ import { import { MediaRepository } from "../../database/repositories/media.js"; import { InvalidCursorError } from "../../database/repositories/types.js"; import type { Database } from "../../database/types.js"; +import { + getMediaUsageActivationStatus, + MediaUsageActivationVersionMismatchError, +} from "../../media/usage/activation.js"; import { CONTENT_MEDIA_USAGE_ADAPTER_ID, CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, @@ -19,6 +23,10 @@ import { type ContentMediaUsageRepairCollectionResult, } from "../../media/usage/content-repair.js"; import { CONTENT_SOURCE_SCHEMA_VERSION } from "../../media/usage/content-snapshots.js"; +import { + runMediaUsageMaintenanceStep, + type MediaUsageMaintenanceContinuation, +} from "../../media/usage/maintenance-engine.js"; import { ErrorCode } from "../errors.js"; import type { MediaUsageCoverage, @@ -26,6 +34,8 @@ import type { MediaUsageDetailsResponse, MediaUsageEntryDetail, MediaUsageOccurrenceDetail, + MediaUsageProgress, + MediaUsageProgressAdvanceResponse, MediaUsageRepairRequest, MediaUsageRepairResponse, MediaUsageSummary, @@ -38,6 +48,8 @@ export type { MediaUsageDetailsResponse, MediaUsageEntryDetail, MediaUsageOccurrenceDetail, + MediaUsageProgress, + MediaUsageProgressAdvanceResponse, MediaUsageSourceDetail, MediaUsageRepairRequest, MediaUsageRepairResponse, @@ -48,6 +60,102 @@ type ContentMediaUsageRepairResult = | ContentMediaUsageRepairCollectionResult | ContentMediaUsageRepairAllResult; +export async function handleMediaUsageProgress( + db: Kysely, +): Promise> { + try { + const progress = await new MediaUsageRepository(db).findCollectionProgress(); + if (!progress) { + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_PROGRESS_NOT_ACTIVE, + message: "Media Usage is not active", + }, + }; + } + return { success: true, data: progress }; + } catch (error) { + if (error instanceof MediaUsageActivationVersionMismatchError) { + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH, + message: "Media Usage activation version does not match this runtime", + }, + }; + } + console.error("[media-usage] progress read failed:", error); + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_PROGRESS_READ_ERROR, + message: "Failed to read media usage progress", + }, + }; + } +} + +export async function handleMediaUsageProgressAdvance( + db: Kysely, +): Promise> { + try { + const step = await runMediaUsageMaintenanceStep(db); + const activation = await getMediaUsageActivationStatus(db); + if (activation.state === "expanded") { + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_PROGRESS_NOT_ACTIVE, + message: "Media Usage activation has not started", + }, + }; + } + + const progress = + activation.state === "active" + ? await new MediaUsageRepository(db).findCollectionProgress() + : null; + return { + success: true, + data: { + activation, + progress, + nextRequestInMs: continuationDelayMs(step.continuation, progress), + }, + }; + } catch (error) { + if (error instanceof MediaUsageActivationVersionMismatchError) { + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH, + message: "Media Usage activation version does not match this runtime", + }, + }; + } + console.error("[media-usage] progress advance failed:", error); + return { + success: false, + error: { + code: ErrorCode.MEDIA_USAGE_PROGRESS_ADVANCE_ERROR, + message: "Failed to advance media usage progress", + }, + }; + } +} + +function continuationDelayMs( + continuation: MediaUsageMaintenanceContinuation, + progress: MediaUsageProgress | null, +): 0 | 30_000 | null { + if (progress?.status === "needs_attention") return null; + if (continuation.kind === "immediate") return 0; + if (continuation.kind === "delayed") return 30_000; + if (progress?.status === "indexing") return 0; + return null; +} + export function aggregateMediaUsageCoverageStatus( scopes: readonly MediaUsageCollectionIndexStatusScope[], ): MediaUsageCoverageStatus { diff --git a/packages/core/src/api/openapi/document.ts b/packages/core/src/api/openapi/document.ts index c465850351..29e6bb7114 100644 --- a/packages/core/src/api/openapi/document.ts +++ b/packages/core/src/api/openapi/document.ts @@ -40,6 +40,8 @@ import { import { mediaUsageDetailsQuery, mediaUsageDetailsResponseSchema, + mediaUsageProgressSchema, + mediaUsageProgressAdvanceResponseSchema, mediaUsageCollectionDeletionListQuery, mediaUsageCollectionDeletionListResponseSchema, mediaUsageCollectionDeletionRetryBody, @@ -903,6 +905,46 @@ function buildMediaPaths(maxUploadSize: number) { }, }, }, + "/_emdash/api/admin/media-usage/progress": { + get: { + operationId: "getMediaUsageProgress", + summary: "Get media usage indexing progress", + description: + "Returns aggregate indexing readiness for current content collections after controlled activation is active. Requires `schema:manage`; bearer tokens also require the `admin` scope.", + tags: ["Media"], + responses: { + "200": { + description: "Aggregate media usage indexing progress", + content: { + [JSON_CONTENT]: { schema: successEnvelope(mediaUsageProgressSchema) }, + }, + }, + ...authErrors, + ...standardErrors(409), + ...standardErrors(500), + }, + }, + post: { + operationId: "advanceMediaUsageProgress", + summary: "Advance media usage indexing", + description: + "Runs one bounded Media Usage maintenance step and returns the stored activation and indexing state. Requires `schema:manage`; bearer tokens also require the `admin` scope.", + tags: ["Media"], + responses: { + "200": { + description: "Media Usage progress after one maintenance step", + content: { + [JSON_CONTENT]: { + schema: successEnvelope(mediaUsageProgressAdvanceResponseSchema), + }, + }, + }, + ...authErrors, + ...standardErrors(409), + ...standardErrors(500), + }, + }, + }, "/_emdash/api/admin/media-usage/work": { get: { operationId: "listMediaUsageWork", @@ -945,7 +987,7 @@ function buildMediaPaths(maxUploadSize: number) { operationId: "advanceMediaUsageActivation", summary: "Advance media usage activation", description: - "Starts, resumes, or retries exactly one bounded activation batch after the operator confirms that all writers are drained and automatic maintenance is ready. Requires `schema:manage`; bearer tokens also require the `admin` scope.", + "Starts, resumes, or retries exactly one bounded activation batch after the operator confirms that all writers are drained. Continue setup through the Media Usage progress endpoint. Requires `schema:manage`; bearer tokens also require the `admin` scope.", tags: ["Media"], requestBody: { required: true, diff --git a/packages/core/src/api/schemas/media-usage.ts b/packages/core/src/api/schemas/media-usage.ts index d09345fb34..670b48b3e0 100644 --- a/packages/core/src/api/schemas/media-usage.ts +++ b/packages/core/src/api/schemas/media-usage.ts @@ -13,6 +13,14 @@ export const mediaUsageCoverageSchema = z }) .meta({ id: "MediaUsageCoverage" }); +export const mediaUsageProgressSchema = z + .object({ + status: z.enum(["indexing", "ready", "needs_attention"]), + readyCollections: z.number().int().min(0), + totalCollections: z.number().int().min(0), + }) + .meta({ id: "MediaUsageProgress" }); + export const mediaUsageSummarySchema = z .object({ count: z.number().int().min(0).nullable(), @@ -203,7 +211,6 @@ export const mediaUsageActivationStatusSchema = z export const mediaUsageActivationAdvanceBody = z .object({ writersDrained: z.literal(true), - maintenanceReady: z.literal(true), }) .strict() .meta({ id: "MediaUsageActivationAdvanceBody" }); @@ -216,6 +223,14 @@ export const mediaUsageActivationAdvanceResponseSchema = z }) .meta({ id: "MediaUsageActivationAdvanceResponse" }); +export const mediaUsageProgressAdvanceResponseSchema = z + .object({ + activation: mediaUsageActivationStatusSchema, + progress: mediaUsageProgressSchema.nullable(), + nextRequestInMs: z.union([z.literal(0), z.literal(30_000), z.null()]), + }) + .meta({ id: "MediaUsageProgressAdvanceResponse" }); + export const mediaUsageActivationConflictSchema = z.object({ success: z.literal(false), error: z.discriminatedUnion("code", [ @@ -280,6 +295,10 @@ export const mediaUsageCollectionDeletionRetryResponseSchema = z export type MediaUsageRepairRequest = z.infer; export type MediaUsageRepairResponse = z.infer; +export type MediaUsageProgress = z.infer; +export type MediaUsageProgressAdvanceResponse = z.infer< + typeof mediaUsageProgressAdvanceResponseSchema +>; export type MediaUsageWorkListQuery = z.infer; export type MediaUsageWorkItem = z.infer; export type MediaUsageWorkListResponse = z.infer; diff --git a/packages/core/src/astro/integration/routes.ts b/packages/core/src/astro/integration/routes.ts index 069726fdd6..d0963608f9 100644 --- a/packages/core/src/astro/integration/routes.ts +++ b/packages/core/src/astro/integration/routes.ts @@ -272,6 +272,10 @@ export function injectCoreRoutes( pattern: "/_emdash/api/admin/media-usage/activation", entrypoint: resolveRoute("api/admin/media-usage/activation.ts"), }); + injectRoute({ + pattern: "/_emdash/api/admin/media-usage/progress", + entrypoint: resolveRoute("api/admin/media-usage/progress.ts"), + }); injectRoute({ pattern: "/_emdash/api/admin/media-usage/collection-deletions", entrypoint: resolveRoute("api/admin/media-usage/collection-deletions/index.ts"), diff --git a/packages/core/src/astro/middleware.ts b/packages/core/src/astro/middleware.ts index a9b9cc9085..246c9a78ab 100644 --- a/packages/core/src/astro/middleware.ts +++ b/packages/core/src/astro/middleware.ts @@ -48,7 +48,6 @@ import { createDeferredTaskTracker } from "../deferred-tasks.js"; import { DB_INIT_DEADLINE_MS, EmDashRuntime, - type MediaUsageMaintenanceResult, type RuntimeDependencies, type SandboxedPluginEntry, type MediaProviderEntry, @@ -300,12 +299,6 @@ export async function runScheduledTasks( return runOutsideRequest(config, (runtime) => runtime.runScheduledTasks(options)); } -export async function runScheduledMediaUsageTasks(): Promise { - const config = getConfig(); - if (!config) return { outcome: "inactive", taskClass: null, turn: null }; - return runOutsideRequest(config, (runtime) => runtime.runScheduledMediaUsageTasks()); -} - /** * Run a callback against the EmDash runtime outside any HTTP request — from a * Cloudflare Queue consumer, a `scheduled()` handler, or any other diff --git a/packages/core/src/astro/routes/api/admin/media-usage/progress.ts b/packages/core/src/astro/routes/api/admin/media-usage/progress.ts new file mode 100644 index 0000000000..38b4c132bf --- /dev/null +++ b/packages/core/src/astro/routes/api/admin/media-usage/progress.ts @@ -0,0 +1,37 @@ +import type { APIRoute } from "astro"; + +import { requirePerm } from "#api/authorize.js"; +import { requireDb, unwrapResult } from "#api/error.js"; +import { + handleMediaUsageProgress, + handleMediaUsageProgressAdvance, +} from "#api/handlers/media-usage.js"; +import { requireScope } from "#auth/scopes.js"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const { emdash, user } = locals; + const dbErr = requireDb(emdash?.db); + if (dbErr) return dbErr; + + const denied = requirePerm(user, "schema:manage"); + if (denied) return denied; + const scopeDenied = requireScope(locals, "admin"); + if (scopeDenied) return scopeDenied; + + return unwrapResult(await handleMediaUsageProgress(emdash.db)); +}; + +export const POST: APIRoute = async ({ locals }) => { + const { emdash, user } = locals; + const dbErr = requireDb(emdash?.db); + if (dbErr) return dbErr; + + const denied = requirePerm(user, "schema:manage"); + if (denied) return denied; + const scopeDenied = requireScope(locals, "admin"); + if (scopeDenied) return scopeDenied; + + return unwrapResult(await handleMediaUsageProgressAdvance(emdash.db)); +}; diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts index a1d4eb4c42..1f18c54711 100644 --- a/packages/core/src/client/index.ts +++ b/packages/core/src/client/index.ts @@ -244,6 +244,18 @@ export interface MediaUsageRepairResponse { collections: MediaUsageRepairCollectionSummary[]; } +export interface MediaUsageProgress { + status: "indexing" | "ready" | "needs_attention"; + readyCollections: number; + totalCollections: number; +} + +export interface MediaUsageProgressAdvanceResponse { + activation: MediaUsageActivationStatus; + progress: MediaUsageProgress | null; + nextRequestInMs: 0 | 30_000 | null; +} + /** Durable media usage entry-work state */ export type MediaUsageWorkState = "pending" | "retry" | "leased" | "failed"; @@ -301,7 +313,6 @@ export interface MediaUsageActivationStatus { export interface MediaUsageActivationAdvanceInput { writersDrained: true; - maintenanceReady: true; } export interface MediaUsageActivationAdvanceResponse { @@ -995,6 +1006,16 @@ export class EmDashClient { return this.request("POST", "/admin/media-usage/repair", input); } + /** Read aggregate Media Usage indexing progress */ + async mediaGetUsageProgress(): Promise { + return this.request("GET", "/admin/media-usage/progress"); + } + + /** Advance exactly one Media Usage maintenance step */ + async mediaAdvanceUsageProgress(): Promise { + return this.request("POST", "/admin/media-usage/progress"); + } + /** Read the redacted controlled-activation status */ async mediaGetUsageActivation(): Promise { return this.request("GET", "/admin/media-usage/activation"); diff --git a/packages/core/src/database/json-recordset.ts b/packages/core/src/database/json-recordset.ts new file mode 100644 index 0000000000..6680bc6640 --- /dev/null +++ b/packages/core/src/database/json-recordset.ts @@ -0,0 +1,16 @@ +import { sql, type Kysely, type RawBuilder } from "kysely"; + +import { isPostgres } from "./dialect-helpers.js"; +import type { Database } from "./types.js"; + +export function jsonTextValues( + db: Kysely, + values: readonly string[], +): RawBuilder<{ value: string }> { + const payload = JSON.stringify([...new Set(values)]); + return isPostgres(db) + ? sql<{ + value: string; + }>`SELECT value::text AS value FROM jsonb_array_elements_text(${payload}::jsonb) AS value` + : sql<{ value: string }>`SELECT value AS value FROM json_each(${payload})`; +} diff --git a/packages/core/src/database/repositories/media-usage-work.ts b/packages/core/src/database/repositories/media-usage-work.ts index 115967a4ab..1b15a6821e 100644 --- a/packages/core/src/database/repositories/media-usage-work.ts +++ b/packages/core/src/database/repositories/media-usage-work.ts @@ -8,14 +8,14 @@ import { decodeCursor, encodeCursor, type FindManyResult } from "./types.js"; export type MediaUsageWorkState = "pending" | "retry" | "leased" | "failed"; export type MediaUsageWorkVersion = number | string; const MAX_PORTABLE_DURATION_SECONDS = 365 * 24 * 60 * 60; -const MAX_WORK_SELECTION_LIMIT = 100; +const MAX_WORK_SELECTION_LIMIT = 1_000; const NON_NEGATIVE_DECIMAL_PATTERN = /^(?:0|[1-9][0-9]*)$/; const POSITIVE_DECIMAL_PATTERN = /^[1-9][0-9]*$/; const STABLE_ERROR_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; export const MEDIA_USAGE_WORK_OPERATOR_DEFAULT_LIMIT = 50; export const MEDIA_USAGE_WORK_OPERATOR_MAX_LIMIT = 100; -export const MEDIA_USAGE_RECONCILIATION_PAGE_LIMIT = 50; +export const MEDIA_USAGE_RECONCILIATION_PAGE_LIMIT = 1_000; const MEDIA_USAGE_WORK_STATES = ["pending", "retry", "leased", "failed"] as const; export interface MediaUsageWorkIdentity { @@ -79,18 +79,18 @@ export class MediaUsageWorkRepository { assertNonNegativeDecimal(input.changeEpoch, "change epoch"); const contentIds = [...new Set(input.contentIds)]; if (contentIds.length < 1 || contentIds.length > MEDIA_USAGE_RECONCILIATION_PAGE_LIMIT) { - throw new Error("Reconciliation enqueue requires from 1 to 50 content IDs"); + throw new Error("Reconciliation enqueue requires from 1 to 1000 content IDs"); } if (contentIds.some((contentId) => !contentId)) { throw new Error("Reconciliation enqueue content IDs must not be empty"); } const now = this.timestampOffset(0); - const pageValues = sql.join( - contentIds.map((contentId) => sql`(${contentId})`), - sql`, `, - ); + const contentIdsJson = JSON.stringify(contentIds); + const page = isPostgres(this.db) + ? sql`SELECT value::text AS content_id FROM jsonb_array_elements_text(${contentIdsJson}::jsonb) AS value` + : sql`SELECT value AS content_id FROM json_each(${contentIdsJson})`; await sql` - WITH page(content_id) AS (VALUES ${pageValues}) + WITH page(content_id) AS (${page}) INSERT INTO _emdash_media_usage_work ( collection_id, collection_slug, content_id, change_epoch, work_version, state, attempt_count, next_attempt_at, lease_token, lease_expires_at, @@ -308,10 +308,11 @@ export class MediaUsageWorkRepository { .updateTable("_emdash_media_usage_index_status as status") .set({ change_epoch: sql`change_epoch + 1`, - status: sql`CASE WHEN status = 'complete' THEN 'stale' ELSE status END`, + status: sql`CASE WHEN status IN ('complete', 'partial', 'failed') THEN 'stale' ELSE status END`, completed_at: sql< string | null - >`CASE WHEN status = 'complete' THEN NULL ELSE completed_at END`, + >`CASE WHEN status IN ('complete', 'partial', 'failed') THEN NULL ELSE completed_at END`, + last_error_code: null, updated_at: this.timestampOffset(0), }) .where("status.adapter_id", "=", "content-media") @@ -508,42 +509,101 @@ export class MediaUsageWorkRepository { return row?.lease_expires_at ?? null; } - async findDueWork(limit: number): Promise { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_WORK_SELECTION_LIMIT) { + async claimDueWorkBatch(input: { + limit: number; + leaseDurationSeconds: number; + }): Promise { + if ( + !Number.isSafeInteger(input.limit) || + input.limit < 1 || + input.limit > MAX_WORK_SELECTION_LIMIT + ) { throw new Error( `Media usage due-work limit must be a whole number from 1 to ${MAX_WORK_SELECTION_LIMIT}`, ); } + const leaseDurationSeconds = durationSeconds( + input.leaseDurationSeconds, + "lease duration", + false, + ); + const leaseToken = ulid(); + const now = this.timestampOffset(0); + let candidates = this.db + .selectFrom("_emdash_media_usage_work as due") + .select(["due.collection_id", "due.content_id", "due.work_version"]) + .where((eb) => + eb.or([ + eb.and([ + eb("due.state", "in", ["pending", "retry"]), + this.timestampIsDue("due.next_attempt_at"), + ]), + eb.and([ + eb("due.state", "=", "leased"), + eb("due.lease_expires_at", "is not", null), + this.timestampIsDue("due.lease_expires_at"), + ]), + ]), + ) + .orderBy( + sql`CASE + WHEN due.state = 'leased' THEN due.lease_expires_at + ELSE due.next_attempt_at + END`, + "asc", + ) + .orderBy("due.updated_at", "asc") + .orderBy("due.collection_id", "asc") + .orderBy("due.content_id", "asc") + .limit(input.limit); + if (isPostgres(this.db)) candidates = candidates.forUpdate().skipLocked(); + const rows = await this.db + .with("due_media_usage_work", () => candidates) + .updateTable("_emdash_media_usage_work") + .set({ + state: "leased", + lease_token: leaseToken, + lease_expires_at: this.timestampOffset(leaseDurationSeconds), + last_attempted_at: now, + updated_at: now, + }) + .where((eb) => + eb.and([ + eb.exists( + eb + .selectFrom("due_media_usage_work as due") + .select("due.content_id") + .whereRef("due.collection_id", "=", "_emdash_media_usage_work.collection_id") + .whereRef("due.content_id", "=", "_emdash_media_usage_work.content_id") + .whereRef("due.work_version", "=", "_emdash_media_usage_work.work_version"), + ), + eb.or([ + eb.and([ + eb("state", "in", ["pending", "retry"]), + this.timestampIsDue("next_attempt_at"), + ]), + eb.and([ + eb("state", "=", "leased"), + eb("lease_expires_at", "is not", null), + this.timestampIsDue("lease_expires_at"), + ]), + ]), + ]), + ) + .returningAll() + .execute(); - const pendingRows = await this.findDueRows("pending", "next_attempt_at", limit); - const retryRows = await this.findDueRows("retry", "next_attempt_at", limit); - const leasedRows = await this.findDueRows("leased", "lease_expires_at", limit); - - return [...pendingRows, ...retryRows, ...leasedRows] - .map(rowToWork) - .toSorted(compareDueWork) - .slice(0, limit); + return rows.map(rowToWork); } - private async findDueRows( - state: "pending" | "retry" | "leased", - timestampColumn: "next_attempt_at" | "lease_expires_at", - limit: number, - ): Promise[]> { - let query = this.db + async hasNonterminalWork(): Promise { + const row = await this.db .selectFrom("_emdash_media_usage_work") - .selectAll() - .where("state", "=", state) - .where(this.timestampIsDue(timestampColumn)) - .orderBy(timestampColumn, "asc"); - if (timestampColumn === "next_attempt_at") { - query = query.orderBy("updated_at", "asc"); - } - return query - .orderBy("collection_id", "asc") - .orderBy("content_id", "asc") - .limit(limit) - .execute(); + .select("content_id") + .where("state", "in", ["pending", "retry", "leased"]) + .limit(1) + .executeTakeFirst(); + return row !== undefined; } async findWorkForContent( @@ -608,18 +668,230 @@ export class MediaUsageWorkRepository { return row ? rowToWork(row) : null; } - async completeWork(input: MediaUsageWorkLease): Promise { - assertLease(input); - const result = await this.db - .deleteFrom("_emdash_media_usage_work") - .where("collection_id", "=", input.collectionId) - .where("content_id", "=", input.contentId) - .where("work_version", "=", input.workVersion) - .where("state", "=", "leased") - .where("lease_token", "=", input.leaseToken) - .where(this.leaseIsLive()) - .executeTakeFirst(); - return Number(result.numDeletedRows ?? 0) > 0; + async completeWorkBatch(inputs: readonly MediaUsageWorkLease[]): Promise> { + const unique = [ + ...new Map( + inputs.map((input) => { + assertLease(input); + return [workIdentityKey(input), input] as const; + }), + ).values(), + ]; + const completed = new Set(); + if (unique.length === 0) return completed; + const payload = JSON.stringify( + unique.map((input) => ({ + collection_id: input.collectionId, + content_id: input.contentId, + work_version: String(input.workVersion), + lease_token: input.leaseToken, + })), + ); + const claimed = this.claimedWorkBatchInput(payload); + const result = await sql<{ + collection_id: string; + content_id: string; + work_version: MediaUsageWorkVersion; + }>` + WITH claimed AS (${claimed}) + DELETE FROM _emdash_media_usage_work AS work + WHERE work.state = 'leased' + AND ${this.qualifiedLeaseIsLive("work.lease_expires_at")} + AND EXISTS ( + SELECT 1 + FROM claimed + WHERE claimed.collection_id = work.collection_id + AND claimed.content_id = work.content_id + AND claimed.work_version = work.work_version + AND claimed.lease_token = work.lease_token + ) + RETURNING collection_id, content_id, work_version + `.execute(this.db); + for (const row of result.rows) { + completed.add( + workIdentityKey({ + collectionId: row.collection_id, + contentId: row.content_id, + workVersion: row.work_version, + }), + ); + } + return completed; + } + + async lockClaimedWorkBatch(inputs: readonly MediaUsageWorkLease[]): Promise> { + const unique = [ + ...new Map( + inputs.map((input) => { + assertLease(input); + return [workIdentityKey(input), input] as const; + }), + ).values(), + ].toSorted((left, right) => workIdentityKey(left).localeCompare(workIdentityKey(right))); + if (!isPostgres(this.db)) return new Set(unique.map(workIdentityKey)); + if (unique.length === 0) return new Set(); + const payload = JSON.stringify( + unique.map((input) => ({ + collection_id: input.collectionId, + content_id: input.contentId, + work_version: String(input.workVersion), + lease_token: input.leaseToken, + })), + ); + const claimed = this.claimedWorkBatchInput(payload); + const result = await sql<{ + collection_id: string; + content_id: string; + work_version: MediaUsageWorkVersion; + }>` + WITH claimed AS (${claimed}) + SELECT work.collection_id, work.content_id, work.work_version + FROM _emdash_media_usage_work AS work + INNER JOIN claimed + ON claimed.collection_id = work.collection_id + AND claimed.content_id = work.content_id + AND claimed.work_version = work.work_version + AND claimed.lease_token = work.lease_token + WHERE work.state = 'leased' + AND ${this.qualifiedLeaseIsLive("work.lease_expires_at")} + ORDER BY work.collection_id, work.content_id + FOR UPDATE + `.execute(this.db); + return new Set( + result.rows.map((row) => + workIdentityKey({ + collectionId: row.collection_id, + contentId: row.content_id, + workVersion: row.work_version, + }), + ), + ); + } + + async retryClaimedWorkBatch(input: { + work: readonly MediaUsageWorkLease[]; + errorCode: string; + retryDelaySeconds: number; + maxAttempts: number; + }): Promise> { + assertErrorCode(input.errorCode); + const retryDelaySeconds = durationSeconds(input.retryDelaySeconds, "retry delay", true); + if (!Number.isSafeInteger(input.maxAttempts) || input.maxAttempts < 1) { + throw new Error("Media usage work max attempts must be a positive whole number"); + } + const unique = [ + ...new Map( + input.work.map((work) => { + assertLease(work); + return [workIdentityKey(work), work] as const; + }), + ).values(), + ]; + const transitioned = new Map(); + if (unique.length === 0) return transitioned; + const payload = JSON.stringify( + unique.map((work) => ({ + collection_id: work.collectionId, + content_id: work.contentId, + work_version: String(work.workVersion), + lease_token: work.leaseToken, + })), + ); + const claimed = this.claimedWorkBatchInput(payload); + const result = await sql<{ + collection_id: string; + content_id: string; + work_version: MediaUsageWorkVersion; + state: "retry" | "failed"; + }>` + WITH claimed AS (${claimed}) + UPDATE _emdash_media_usage_work AS work + SET state = CASE WHEN work.attempt_count + 1 >= ${input.maxAttempts} THEN 'failed' ELSE 'retry' END, + attempt_count = work.attempt_count + 1, + next_attempt_at = ${this.timestampOffset(retryDelaySeconds)}, + lease_token = NULL, + lease_expires_at = NULL, + last_error_code = ${input.errorCode}, + updated_at = ${this.timestampOffset(0)} + WHERE work.state = 'leased' + AND ${this.qualifiedLeaseIsLive("work.lease_expires_at")} + AND EXISTS ( + SELECT 1 + FROM claimed + WHERE claimed.collection_id = work.collection_id + AND claimed.content_id = work.content_id + AND claimed.work_version = work.work_version + AND claimed.lease_token = work.lease_token + ) + RETURNING collection_id, content_id, work_version, state + `.execute(this.db); + for (const row of result.rows) { + transitioned.set( + workIdentityKey({ + collectionId: row.collection_id, + contentId: row.content_id, + workVersion: row.work_version, + }), + row.state, + ); + } + return transitioned; + } + + async releaseClaimedWorkBatch(inputs: readonly MediaUsageWorkLease[]): Promise> { + const unique = [ + ...new Map( + inputs.map((input) => { + assertLease(input); + return [workIdentityKey(input), input] as const; + }), + ).values(), + ]; + const released = new Set(); + if (unique.length === 0) return released; + const payload = JSON.stringify( + unique.map((input) => ({ + collection_id: input.collectionId, + content_id: input.contentId, + work_version: String(input.workVersion), + lease_token: input.leaseToken, + })), + ); + const claimed = this.claimedWorkBatchInput(payload); + const result = await sql<{ + collection_id: string; + content_id: string; + work_version: MediaUsageWorkVersion; + }>` + WITH claimed AS (${claimed}) + UPDATE _emdash_media_usage_work AS work + SET state = 'pending', + next_attempt_at = ${this.timestampOffset(0)}, + lease_token = NULL, + lease_expires_at = NULL, + updated_at = ${this.timestampOffset(0)} + WHERE work.state = 'leased' + AND ${this.qualifiedLeaseIsLive("work.lease_expires_at")} + AND EXISTS ( + SELECT 1 + FROM claimed + WHERE claimed.collection_id = work.collection_id + AND claimed.content_id = work.content_id + AND claimed.work_version = work.work_version + AND claimed.lease_token = work.lease_token + ) + RETURNING collection_id, content_id, work_version + `.execute(this.db); + for (const row of result.rows) { + released.add( + workIdentityKey({ + collectionId: row.collection_id, + contentId: row.content_id, + workVersion: row.work_version, + }), + ); + } + return released; } async deleteWorkThroughEpoch( @@ -636,53 +908,49 @@ export class MediaUsageWorkRepository { return Number(result.numDeletedRows ?? 0); } - async retryWork( - input: MediaUsageWorkLease & { - retryDelaySeconds: number; - errorCode: string; - }, - ): Promise { - const retryDelaySeconds = durationSeconds(input.retryDelaySeconds, "retry delay", true); - assertErrorCode(input.errorCode); - return this.transitionFailure(input, "retry", { - next_attempt_at: this.timestampOffset(retryDelaySeconds), - }); - } - - async failWork( - input: MediaUsageWorkLease & { - errorCode: string; - }, - ): Promise { - assertErrorCode(input.errorCode); - return this.transitionFailure(input, "failed"); - } - - private async transitionFailure( - input: MediaUsageWorkLease & { errorCode: string }, - state: "retry" | "failed", - extra: { next_attempt_at?: RawBuilder } = {}, - ): Promise { - assertLease(input); + async deleteObsoleteReconciliationWork(input: { + collectionId: string; + collectionSlug: string; + runToken: string; + leaseToken: string; + targetEpoch: number | string; + }): Promise { + assertNonNegativeDecimal(input.targetEpoch, "target epoch"); const result = await this.db - .updateTable("_emdash_media_usage_work") - .set({ - state, - attempt_count: sql`attempt_count + 1`, - lease_token: null, - lease_expires_at: null, - last_error_code: input.errorCode, - updated_at: this.timestampOffset(0), - ...extra, - }) - .where("collection_id", "=", input.collectionId) - .where("content_id", "=", input.contentId) - .where("work_version", "=", input.workVersion) - .where("state", "=", "leased") - .where("lease_token", "=", input.leaseToken) - .where(this.leaseIsLive()) + .deleteFrom("_emdash_media_usage_work as work") + .where("work.collection_id", "=", input.collectionId) + .where("work.change_epoch", "<", input.targetEpoch) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_reconciliations as reconciliation") + .innerJoin("_emdash_media_usage_index_status as status", (join) => + join + .onRef("status.collection_id", "=", "reconciliation.collection_id") + .onRef("status.scope_key", "=", "reconciliation.collection_slug"), + ) + .innerJoin("_emdash_collections as collection", (join) => + join + .onRef("collection.id", "=", "reconciliation.collection_id") + .onRef("collection.slug", "=", "reconciliation.collection_slug"), + ) + .select("reconciliation.collection_id") + .where("reconciliation.collection_id", "=", input.collectionId) + .where("reconciliation.collection_slug", "=", input.collectionSlug) + .where("reconciliation.run_token", "=", input.runToken) + .where("reconciliation.target_epoch", "=", input.targetEpoch) + .where("reconciliation.state", "=", "leased") + .where("reconciliation.lease_token", "=", input.leaseToken) + .where(this.qualifiedLeaseIsLive("reconciliation.lease_expires_at")) + .where("status.capture_state", "=", "active") + .where("status.reconciliation_required", "=", 1) + .where("status.status", "=", "running") + .where("status.cursor", "=", input.runToken) + .where("status.change_epoch", "=", input.targetEpoch), + ), + ) .executeTakeFirst(); - return Number(result.numUpdatedRows ?? 0) > 0; + return Number(result.numDeletedRows ?? 0); } private leaseIsLive(): RawBuilder { @@ -699,7 +967,13 @@ export class MediaUsageWorkRepository { } private timestampIsDue( - column: "next_attempt_at" | "lease_expires_at" | "work.lease_expires_at", + column: + | "next_attempt_at" + | "lease_expires_at" + | "work.lease_expires_at" + | "work.next_attempt_at" + | "due.next_attempt_at" + | "due.lease_expires_at", ): RawBuilder { return isPostgres(this.db) ? sql`${sql.ref(column)} <= to_char( @@ -722,6 +996,31 @@ export class MediaUsageWorkRepository { ${`${offsetSeconds >= 0 ? "+" : ""}${offsetSeconds} seconds`} )`; } + + private claimedWorkBatchInput(payload: string): RawBuilder { + if (isPostgres(this.db)) { + return sql` + SELECT + entry.value ->> 'collection_id' AS collection_id, + entry.value ->> 'content_id' AS content_id, + CAST(entry.value ->> 'work_version' AS bigint) AS work_version, + entry.value ->> 'lease_token' AS lease_token + FROM jsonb_array_elements(${payload}::jsonb) AS entry(value) + `; + } + return sql` + SELECT + json_extract(entry.value, '$.collection_id') AS collection_id, + json_extract(entry.value, '$.content_id') AS content_id, + CAST(json_extract(entry.value, '$.work_version') AS integer) AS work_version, + json_extract(entry.value, '$.lease_token') AS lease_token + FROM json_each(${payload}) AS entry + `; + } +} + +function workIdentityKey(input: MediaUsageWorkIdentity): string { + return `${input.collectionId}\u0000${input.contentId}\u0000${String(input.workVersion)}`; } function operatorLimit(value: number | undefined): number { @@ -827,22 +1126,7 @@ function isMediaUsageWorkState(value: string): value is MediaUsageWorkState { return value === "pending" || value === "retry" || value === "leased" || value === "failed"; } -function compareDueWork(a: MediaUsageWorkRecord, b: MediaUsageWorkRecord): number { - const eligibility = dueTimestamp(a).localeCompare(dueTimestamp(b)); - if (eligibility !== 0) return eligibility; - const updated = a.updatedAt.localeCompare(b.updatedAt); - if (updated !== 0) return updated; - const collection = a.collectionId.localeCompare(b.collectionId); - return collection !== 0 ? collection : a.contentId.localeCompare(b.contentId); -} - function compareOperatorWork(a: MediaUsageOperatorWorkItem, b: MediaUsageOperatorWorkItem): number { const updated = b.updatedAt.localeCompare(a.updatedAt); return updated !== 0 ? updated : b.contentId.localeCompare(a.contentId); } - -function dueTimestamp(work: MediaUsageWorkRecord): string { - if (work.state !== "leased") return work.nextAttemptAt; - if (!work.leaseExpiresAt) throw new Error("Due leased media usage work must have a lease expiry"); - return work.leaseExpiresAt; -} diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 49ccb3f6f4..44c399cc91 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -9,11 +9,21 @@ import { } from "kysely"; import { ulid } from "ulidx"; +import { + MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION, + MediaUsageActivationVersionMismatchError, +} from "../../media/usage/activation.js"; import { isMediaUsageProjectionFingerprint } from "../../media/usage/projection-fingerprint.js"; import type { MediaUsageContentSourceVariant } from "../../media/usage/source-key.js"; -import type { MediaKind, MediaUsageReferenceType } from "../../media/usage/types.js"; +import { + CONTENT_SOURCE_SCHEMA_VERSION, + type MediaKind, + type MediaUsageReferenceType, +} from "../../media/usage/types.js"; +import { getRequestContext } from "../../request-context.js"; import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { isPostgres } from "../dialect-helpers.js"; +import { jsonTextValues } from "../json-recordset.js"; import { withTransaction } from "../transaction.js"; import type { Database, @@ -33,11 +43,30 @@ type MediaUsageSourceNullableStringColumn = | "updated_at" | "last_attempted_at" | "last_error_code"; +type BatchSourceExpectedColumn = + | "source.collection_id" + | "source.source_fingerprint" + | "source.source_updated_at" + | "source.source_version" + | "source.identity_version" + | "source.revision_id" + | "source.last_attempted_at" + | "source.last_error_code"; +type BatchInputExpectedColumn = + | "input.expected_collection_id" + | "input.expected_source_fingerprint" + | "input.expected_source_updated_at" + | "input.expected_source_version" + | "input.expected_identity_version" + | "input.expected_revision_id" + | "input.expected_last_attempted_at" + | "input.expected_last_error_code"; const OCCURRENCE_BIND_COLUMNS = 13; +const D1_MAX_BOUND_PARAMETERS = 100; export const MEDIA_USAGE_GENERATION_WRITE_LEASE_MS = 60 * 60 * 1000; const OCCURRENCE_INSERT_BATCH_SIZE = Math.max( 1, - Math.floor(SQL_BATCH_SIZE / OCCURRENCE_BIND_COLUMNS), + Math.floor(D1_MAX_BOUND_PARAMETERS / OCCURRENCE_BIND_COLUMNS), ); function cleanupDeleteBatchSize(cleanupLease: MediaUsageCleanupLease | undefined): number { @@ -48,6 +77,84 @@ function canIssueCleanupStatement(canIssueStatement: (() => boolean) | undefined return canIssueStatement?.() ?? true; } +function nullableRefMatch( + left: "content.live_revision_id" | "content.draft_revision_id", + right: "input.revision_id", +): RawBuilder { + const leftRef = sql.ref(left); + const rightRef = sql.ref(right); + return sql`( + (${leftRef} IS NULL AND ${rightRef} IS NULL) + OR ${leftRef} = ${rightRef} + )`; +} + +const MAX_JSON_BIND_BYTES = 1_900_000; +const MEDIA_USAGE_EVENT_QUERY_CEILING = 900; +const MEDIA_USAGE_QUERY_RESERVE = 150; + +function chunkJsonRows(rows: readonly T[]): T[][] { + const batches: T[][] = []; + let batch: T[] = []; + let batchBytes = 2; + for (const row of rows) { + const rowBytes = + new TextEncoder().encode(JSON.stringify(row)).byteLength + (batch.length > 0 ? 1 : 0); + if (batch.length > 0 && batchBytes + rowBytes > MAX_JSON_BIND_BYTES) { + batches.push(batch); + batch = []; + batchBytes = 2; + } + batch.push(row); + batchBytes += rowBytes; + } + if (batch.length > 0) batches.push(batch); + return batches; +} + +function mediaUsageQueryBudgetAllows(queries: number): boolean { + const metrics = getRequestContext()?.metrics; + return ( + !metrics || + metrics.dbCount + queries + MEDIA_USAGE_QUERY_RESERVE <= MEDIA_USAGE_EVENT_QUERY_CEILING + ); +} + +function mergeInto(target: Set, source: ReadonlySet): void { + for (const value of source) target.add(value); +} + +function batchOccurrenceRows( + prepared: readonly { + projection: { occurrences: readonly MediaUsageOccurrenceInput[] }; + generation: string; + leaseToken: string; + row: { source_key: string }; + }[], + now: string, +) { + return prepared.flatMap((item) => + item.projection.occurrences.map((occurrence) => ({ + id: ulid(), + source_key: item.row.source_key, + generation: item.generation, + field_slug: occurrence.fieldSlug, + field_path: occurrence.fieldPath, + occurrence_index: occurrence.occurrenceIndex ?? 0, + reference_type: occurrence.referenceType, + media_id: occurrence.mediaId, + provider: occurrence.provider, + provider_asset_id: occurrence.providerAssetId, + media_kind: occurrence.mediaKind ?? null, + mime_type: occurrence.mimeType ?? null, + created_at: now, + lease_token: item.leaseToken, + })), + ); +} + +type BatchOccurrenceRow = ReturnType[number]; + function cleanupDurationSeconds(value: number): number { if (!Number.isSafeInteger(value) || value < 0) { throw new Error("Media usage cleanup duration must be a non-negative whole number of seconds"); @@ -168,6 +275,15 @@ export interface MediaUsageGuardedReplaceResult { source: MediaUsageSource | null; } +export interface MediaUsageNewSourceProjection { + source: MediaUsageSourceInput; + occurrences: readonly MediaUsageOccurrenceInput[]; +} + +export interface MediaUsageExistingSourceProjection extends MediaUsageNewSourceProjection { + expectedSource: MediaUsageSource; +} + export interface MediaUsageGuardedDeleteResult { deleted: boolean; source: MediaUsageSource | null; @@ -340,6 +456,12 @@ export interface MediaUsageCollectionIndexStatusScope { reconciliationRequired: boolean; } +export interface MediaUsageCollectionProgress { + status: "indexing" | "ready" | "needs_attention"; + readyCollections: number; + totalCollections: number; +} + export interface MediaUsageEntrySource { source: MediaUsageSource; occurrences: MediaUsageOccurrence[]; @@ -537,14 +659,15 @@ export class MediaUsageRepository { const uniqueSourceKeys = [...new Set(sourceKeys)]; const sources = new Map(); if (uniqueSourceKeys.length === 0) return sources; - - for (const sourceKeyBatch of chunks(uniqueSourceKeys, SQL_BATCH_SIZE)) { - const rows = await this.db - .selectFrom("_emdash_media_usage_sources") - .selectAll() - .where("source_key", "in", sourceKeyBatch) - .execute(); - for (const row of rows) { + for (const sourceKeyBatch of chunkJsonRows(uniqueSourceKeys)) { + const input = jsonTextValues(this.db, sourceKeyBatch); + const result = await sql>` + WITH requested AS (${input}) + SELECT source.* + FROM _emdash_media_usage_sources AS source + INNER JOIN requested ON requested.value = source.source_key + `.execute(this.db); + for (const row of result.rows) { const source = rowToSource(row); sources.set(source.sourceKey, source); } @@ -619,6 +742,441 @@ export class MediaUsageRepository { }; } + async replaceNewSourcesBatch( + projections: readonly MediaUsageNewSourceProjection[], + ): Promise> { + const unique = [ + ...new Map( + projections.map((projection) => [projection.source.sourceKey, projection]), + ).values(), + ]; + if (unique.length === 0) return new Set(); + const collectionSlug = unique[0]?.source.collectionSlug; + const collectionId = unique[0]?.source.collectionId; + if (!collectionSlug || !collectionId) { + throw new Error("Canonical media usage batch requires collection identity"); + } + validateIdentifier(collectionSlug, "collection slug"); + if ( + unique.some( + (projection) => + projection.source.collectionSlug !== collectionSlug || + projection.source.collectionId !== collectionId, + ) + ) { + throw new Error("Canonical media usage batch must contain one collection"); + } + + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + MEDIA_USAGE_GENERATION_WRITE_LEASE_MS).toISOString(); + const prepared = unique.map((projection) => { + const generation = ulid(); + return { + projection, + generation, + leaseToken: ulid(), + row: this.buildSourceRow(projection.source, generation, now), + }; + }); + const sourceRows = prepared.map((item) => ({ + ...item.row, + lease_token: item.leaseToken, + })); + const occurrenceRows = batchOccurrenceRows(prepared, now); + const estimatedQueries = + 2 + chunkJsonRows(occurrenceRows).length + chunkJsonRows(sourceRows).length; + if (!mediaUsageQueryBudgetAllows(estimatedQueries)) { + if (unique.length === 1) return new Set(); + const midpoint = Math.ceil(unique.length / 2); + const inserted = new Set(); + mergeInto(inserted, await this.replaceNewSourcesBatch(unique.slice(0, midpoint))); + mergeInto(inserted, await this.replaceNewSourcesBatch(unique.slice(midpoint))); + return inserted; + } + + const leasesPayload = JSON.stringify( + prepared.map((item) => ({ + source_key: item.row.source_key, + collection_id: item.row.collection_id, + collection_slug: item.row.collection_slug, + generation: item.generation, + lease_token: item.leaseToken, + expires_at: expiresAt, + created_at: now, + })), + ); + const leases = this.generationWriteBatchInput(leasesPayload); + await sql` + WITH input AS (${leases}) + INSERT INTO _emdash_media_usage_generation_writes ( + source_key, generation, lease_token, expires_at, created_at + ) + SELECT source_key, generation, lease_token, expires_at, created_at + FROM input + WHERE EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = collection.id + AND status.scope_key = collection.slug + WHERE collection.id = input.collection_id + AND collection.slug = input.collection_slug + AND status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = input.collection_id + ) + ) + `.execute(this.db); + + try { + await this.insertBatchOccurrences(occurrenceRows); + const tableName = `ec_${collectionSlug}`; + validateIdentifier(tableName, "content table"); + const inserted = new Set(); + for (const sourceRowBatch of chunkJsonRows(sourceRows)) { + const input = this.sourceBatchInput(JSON.stringify(sourceRowBatch)); + const result = await sql<{ source_key: string }>` + WITH input AS (${input}) + INSERT INTO _emdash_media_usage_sources ( + source_key, source_type, collection_id, collection_slug, content_id, + source_variant, locale, translation_group, content_slug, content_title, + content_status, content_scheduled_at, content_deleted_at, revision_id, + current_generation, schema_version, source_updated_at, source_version, + source_fingerprint, identity_version, source_completeness, + last_attempted_at, last_error_code, indexed_at, updated_at + ) + SELECT + input.source_key, input.source_type, input.collection_id, input.collection_slug, + input.content_id, input.source_variant, input.locale, input.translation_group, + input.content_slug, input.content_title, input.content_status, + input.content_scheduled_at, input.content_deleted_at, input.revision_id, + input.current_generation, input.schema_version, input.source_updated_at, + input.source_version, input.source_fingerprint, input.identity_version, + input.source_completeness, input.last_attempted_at, input.last_error_code, + input.indexed_at, input.updated_at + FROM input + WHERE EXISTS ( + SELECT 1 + FROM _emdash_media_usage_generation_writes AS writer + WHERE writer.source_key = input.source_key + AND writer.generation = input.current_generation + AND writer.lease_token = input.lease_token + AND ${this.generationWriteLeaseExpiryIsInFuture("writer.expires_at")} + ) + AND EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = collection.id + AND status.scope_key = collection.slug + WHERE collection.id = input.collection_id + AND collection.slug = input.collection_slug + AND status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND NOT EXISTS ( + SELECT 1 + FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = input.collection_id + ) + ) + AND EXISTS ( + SELECT 1 + FROM ${sql.ref(tableName)} AS content + WHERE content.id = input.content_id + AND content.version = input.source_version + AND content.updated_at = input.source_updated_at + AND ( + (input.source_variant = 'columns' AND ${nullableRefMatch("content.live_revision_id", "input.revision_id")}) + OR + (input.source_variant = 'draft_overlay' AND ${nullableRefMatch("content.draft_revision_id", "input.revision_id")}) + ) + ) + ON CONFLICT (source_key) DO NOTHING + RETURNING source_key + `.execute(this.db); + for (const row of result.rows) inserted.add(row.source_key); + } + return inserted; + } finally { + await sql` + WITH input AS (${leases}) + DELETE FROM _emdash_media_usage_generation_writes AS writer + WHERE EXISTS ( + SELECT 1 + FROM input + WHERE input.source_key = writer.source_key + AND input.generation = writer.generation + AND input.lease_token = writer.lease_token + ) + `.execute(this.db); + } + } + + async replaceExistingSourcesBatch( + projections: readonly MediaUsageExistingSourceProjection[], + ): Promise> { + const unique = [ + ...new Map( + projections.map((projection) => [projection.source.sourceKey, projection]), + ).values(), + ]; + if (unique.length === 0) return new Set(); + const collectionSlug = unique[0]?.source.collectionSlug; + const collectionId = unique[0]?.source.collectionId; + if (!collectionSlug || !collectionId) { + throw new Error("Canonical media usage batch requires collection identity"); + } + validateIdentifier(collectionSlug, "collection slug"); + if ( + unique.some( + (projection) => + projection.source.collectionSlug !== collectionSlug || + projection.source.collectionId !== collectionId, + ) + ) { + throw new Error("Canonical media usage batch must contain one collection"); + } + + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + MEDIA_USAGE_GENERATION_WRITE_LEASE_MS).toISOString(); + const prepared = unique.map((projection) => { + const generation = ulid(); + return { + projection, + generation, + leaseToken: ulid(), + row: this.buildSourceRow(projection.source, generation, now), + }; + }); + const sourceRows = prepared.map((item) => ({ + ...item.row, + lease_token: item.leaseToken, + expected_generation: item.projection.expectedSource.currentGeneration, + expected_collection_id: item.projection.expectedSource.collectionId, + expected_updated_at: item.projection.expectedSource.updatedAt, + expected_source_fingerprint: item.projection.expectedSource.sourceFingerprint, + expected_source_updated_at: item.projection.expectedSource.sourceUpdatedAt, + expected_source_version: item.projection.expectedSource.sourceVersion, + expected_identity_version: item.projection.expectedSource.identityVersion, + expected_revision_id: item.projection.expectedSource.revisionId, + expected_source_completeness: item.projection.expectedSource.sourceCompleteness, + expected_last_attempted_at: item.projection.expectedSource.lastAttemptedAt, + expected_last_error_code: item.projection.expectedSource.lastErrorCode, + })); + const occurrenceRows = batchOccurrenceRows(prepared, now); + const estimatedQueries = + 2 + chunkJsonRows(occurrenceRows).length + chunkJsonRows(sourceRows).length; + if (!mediaUsageQueryBudgetAllows(estimatedQueries)) { + if (unique.length === 1) return new Set(); + const midpoint = Math.ceil(unique.length / 2); + const replaced = new Set(); + mergeInto(replaced, await this.replaceExistingSourcesBatch(unique.slice(0, midpoint))); + mergeInto(replaced, await this.replaceExistingSourcesBatch(unique.slice(midpoint))); + return replaced; + } + const leasesPayload = JSON.stringify( + prepared.map((item) => ({ + source_key: item.row.source_key, + collection_id: item.row.collection_id, + collection_slug: item.row.collection_slug, + generation: item.generation, + lease_token: item.leaseToken, + expires_at: expiresAt, + created_at: now, + })), + ); + const leases = this.generationWriteBatchInput(leasesPayload); + await sql` + WITH input AS (${leases}) + INSERT INTO _emdash_media_usage_generation_writes ( + source_key, generation, lease_token, expires_at, created_at + ) + SELECT source_key, generation, lease_token, expires_at, created_at + FROM input + WHERE EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = collection.id + AND status.scope_key = collection.slug + WHERE collection.id = input.collection_id + AND collection.slug = input.collection_slug + AND status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = input.collection_id + ) + ) + `.execute(this.db); + + try { + await this.insertBatchOccurrences(occurrenceRows); + const tableName = `ec_${collectionSlug}`; + validateIdentifier(tableName, "content table"); + const replaced = new Set(); + for (const sourceRowBatch of chunkJsonRows(sourceRows)) { + const input = this.sourceBatchInput(JSON.stringify(sourceRowBatch)); + const result = await sql<{ source_key: string }>` + WITH input AS (${input}) + UPDATE _emdash_media_usage_sources AS source + SET source_type = input.source_type, + collection_id = input.collection_id, + collection_slug = input.collection_slug, + content_id = input.content_id, + source_variant = input.source_variant, + locale = input.locale, + translation_group = input.translation_group, + content_slug = input.content_slug, + content_title = input.content_title, + content_status = input.content_status, + content_scheduled_at = input.content_scheduled_at, + content_deleted_at = input.content_deleted_at, + revision_id = input.revision_id, + current_generation = input.current_generation, + schema_version = input.schema_version, + source_updated_at = input.source_updated_at, + source_version = input.source_version, + source_fingerprint = input.source_fingerprint, + identity_version = input.identity_version, + source_completeness = input.source_completeness, + last_attempted_at = input.last_attempted_at, + last_error_code = input.last_error_code, + indexed_at = input.indexed_at, + updated_at = input.updated_at + FROM input + WHERE source.source_key = input.source_key + AND source.current_generation = input.expected_generation + AND ${this.batchRefsMatch("source.collection_id", "input.expected_collection_id")} + AND source.updated_at = input.expected_updated_at + AND ${this.batchRefsMatch("source.source_fingerprint", "input.expected_source_fingerprint")} + AND ${this.batchRefsMatch("source.source_updated_at", "input.expected_source_updated_at")} + AND ${this.batchRefsMatch("source.source_version", "input.expected_source_version")} + AND ${this.batchRefsMatch("source.identity_version", "input.expected_identity_version")} + AND ${this.batchRefsMatch("source.revision_id", "input.expected_revision_id")} + AND source.source_completeness = input.expected_source_completeness + AND ${this.batchRefsMatch("source.last_attempted_at", "input.expected_last_attempted_at")} + AND ${this.batchRefsMatch("source.last_error_code", "input.expected_last_error_code")} + AND EXISTS ( + SELECT 1 + FROM _emdash_media_usage_generation_writes AS writer + WHERE writer.source_key = input.source_key + AND writer.generation = input.current_generation + AND writer.lease_token = input.lease_token + AND ${this.generationWriteLeaseExpiryIsInFuture("writer.expires_at")} + ) + AND EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = collection.id + AND status.scope_key = collection.slug + WHERE collection.id = input.collection_id + AND collection.slug = input.collection_slug + AND status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND NOT EXISTS ( + SELECT 1 + FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = input.collection_id + ) + ) + AND EXISTS ( + SELECT 1 + FROM ${sql.ref(tableName)} AS content + WHERE content.id = input.content_id + AND content.version = input.source_version + AND content.updated_at = input.source_updated_at + AND ( + (input.source_variant = 'columns' AND ${nullableRefMatch("content.live_revision_id", "input.revision_id")}) + OR + (input.source_variant = 'draft_overlay' AND ${nullableRefMatch("content.draft_revision_id", "input.revision_id")}) + ) + ) + RETURNING ${isPostgres(this.db) ? sql`source.source_key` : sql`source_key`} AS source_key + `.execute(this.db); + for (const row of result.rows) replaced.add(row.source_key); + } + return replaced; + } finally { + await sql` + WITH input AS (${leases}) + DELETE FROM _emdash_media_usage_generation_writes AS writer + WHERE EXISTS ( + SELECT 1 + FROM input + WHERE input.source_key = writer.source_key + AND input.generation = writer.generation + AND input.lease_token = writer.lease_token + ) + `.execute(this.db); + } + } + + async matchingExistingSourcesBatch( + projections: readonly MediaUsageExistingSourceProjection[], + ): Promise> { + if (projections.length === 0) return new Set(); + const now = new Date().toISOString(); + const rows = projections.map((projection) => ({ + ...this.buildSourceRow(projection.source, projection.expectedSource.currentGeneration, now), + expected_generation: projection.expectedSource.currentGeneration, + expected_collection_id: projection.expectedSource.collectionId, + expected_updated_at: projection.expectedSource.updatedAt, + expected_source_fingerprint: projection.expectedSource.sourceFingerprint, + expected_source_updated_at: projection.expectedSource.sourceUpdatedAt, + expected_source_version: projection.expectedSource.sourceVersion, + expected_identity_version: projection.expectedSource.identityVersion, + expected_revision_id: projection.expectedSource.revisionId, + expected_source_completeness: projection.expectedSource.sourceCompleteness, + expected_last_attempted_at: projection.expectedSource.lastAttemptedAt, + expected_last_error_code: projection.expectedSource.lastErrorCode, + })); + if (!mediaUsageQueryBudgetAllows(chunkJsonRows(rows).length)) { + if (projections.length === 1) return new Set(); + const midpoint = Math.ceil(projections.length / 2); + const matched = new Set(); + mergeInto(matched, await this.matchingExistingSourcesBatch(projections.slice(0, midpoint))); + mergeInto(matched, await this.matchingExistingSourcesBatch(projections.slice(midpoint))); + return matched; + } + const matched = new Set(); + for (const batch of chunkJsonRows(rows)) { + const input = this.sourceBatchInput(JSON.stringify(batch)); + const result = await sql<{ source_key: string }>` + WITH input AS (${input}) + SELECT source.source_key + FROM _emdash_media_usage_sources AS source + INNER JOIN input ON input.source_key = source.source_key + WHERE source.current_generation = input.expected_generation + AND ${this.batchRefsMatch("source.collection_id", "input.expected_collection_id")} + AND source.updated_at = input.expected_updated_at + AND ${this.batchRefsMatch("source.source_fingerprint", "input.expected_source_fingerprint")} + AND ${this.batchRefsMatch("source.source_updated_at", "input.expected_source_updated_at")} + AND ${this.batchRefsMatch("source.source_version", "input.expected_source_version")} + AND ${this.batchRefsMatch("source.identity_version", "input.expected_identity_version")} + AND ${this.batchRefsMatch("source.revision_id", "input.expected_revision_id")} + AND source.source_completeness = input.expected_source_completeness + AND ${this.batchRefsMatch("source.last_attempted_at", "input.expected_last_attempted_at")} + AND ${this.batchRefsMatch("source.last_error_code", "input.expected_last_error_code")} + AND EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + WHERE collection.id = source.collection_id + AND collection.slug = source.collection_slug + ) + `.execute(this.db); + for (const row of result.rows) matched.add(row.source_key); + } + return matched; + } + async markSourceAttempted(source: MediaUsageSourceInput): Promise { if (source.collectionId !== undefined && source.collectionId !== null) { const expectedSource = await this.findSource(source.sourceKey); @@ -762,6 +1320,129 @@ export class MediaUsageRepository { })); } + async findCollectionProgress(): Promise { + const result = await sql<{ + activation_active: boolean | number; + activation_generation: number | string | null; + needs_attention: boolean | number; + ready_collections: number | string; + total_collections: number | string; + cleanup_pending: boolean | number; + }>` + WITH collection_progress AS ( + SELECT + CASE WHEN status.status = 'complete' + AND status.schema_version = ${CONTENT_SOURCE_SCHEMA_VERSION} + AND status.reconciliation_required = 0 + AND status.capture_state = 'active' + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_work AS work + WHERE work.collection_id = collection.id + AND work.collection_slug = collection.slug + ) + THEN 1 ELSE 0 END AS is_ready, + CASE WHEN status.collection_id IS NULL + OR COALESCE(status.capture_state, '') <> 'active' + OR status.status NOT IN ('complete', 'never', 'running', 'partial', 'failed', 'stale') + OR status.status = 'failed' + OR ( + status.reconciliation_required = 0 + AND ( + status.status <> 'complete' + OR COALESCE(status.schema_version, -1) <> ${CONTENT_SOURCE_SCHEMA_VERSION} + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_work AS work + WHERE work.collection_id = collection.id + AND work.collection_slug = collection.slug + ) + ) + OR EXISTS ( + SELECT 1 FROM _emdash_media_usage_work AS work + WHERE work.collection_id = collection.id + AND work.collection_slug = collection.slug + AND work.state = 'failed' + ) + OR EXISTS ( + SELECT 1 FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = collection.id + AND reconciliation.collection_slug = collection.slug + AND reconciliation.state = 'failed' + AND status.reconciliation_required = 1 + AND ( + reconciliation.target_epoch IS NULL + OR reconciliation.target_epoch >= status.change_epoch + ) + ) + THEN 1 ELSE 0 END AS needs_attention + FROM _emdash_collections AS collection + LEFT JOIN _emdash_media_usage_index_status AS status + ON status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.collection_id = collection.id + AND status.scope_key = collection.slug + WHERE NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = collection.id + AND deletion.collection_slug = collection.slug + ) + ) + SELECT + ( + SELECT activation.runtime_generation + FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = 'incremental_capture' + ) AS activation_generation, + EXISTS ( + SELECT 1 FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = 'incremental_capture' + AND activation.state = 'active' + ) AS activation_active, + COUNT(*) AS total_collections, + COALESCE(SUM(is_ready), 0) AS ready_collections, + EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.state IN ('pending', 'retry', 'leased') + ) AS cleanup_pending, + CASE WHEN COALESCE(MAX(needs_attention), 0) = 1 OR EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.state = 'failed' + ) THEN 1 ELSE 0 END AS needs_attention + FROM collection_progress + `.execute(this.db); + const row = result.rows[0]; + if (!row) throw new Error("Media usage progress query returned no result"); + if (!row.activation_active) return null; + if (Number(row.activation_generation) !== MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION) { + throw new MediaUsageActivationVersionMismatchError( + "Media usage activation runtime generation is incompatible", + ); + } + const readyCollections = Number(row.ready_collections); + const totalCollections = Number(row.total_collections); + const cleanupPending = Number(row.cleanup_pending); + const needsAttention = Number(row.needs_attention); + if ( + !Number.isSafeInteger(readyCollections) || + !Number.isSafeInteger(totalCollections) || + readyCollections < 0 || + totalCollections < readyCollections || + (cleanupPending !== 0 && cleanupPending !== 1) || + (needsAttention !== 0 && needsAttention !== 1) + ) { + throw new Error("Media usage progress query returned invalid counts"); + } + return { + status: needsAttention + ? "needs_attention" + : readyCollections === totalCollections && cleanupPending === 0 + ? "ready" + : "indexing", + readyCollections, + totalCollections, + }; + } + async findCurrentEntryUsagePageByMediaId( mediaId: string, options: FindMediaUsageOptions = {}, @@ -1694,6 +2375,7 @@ export class MediaUsageRepository { last_error_code: sql< string | null >`CASE WHEN ${canComplete} THEN NULL ELSE last_error_code END`, + cursor: sql`CASE WHEN ${canComplete} THEN NULL ELSE cursor END`, last_incremental_success_at: now, updated_at: now, }) @@ -1715,6 +2397,94 @@ export class MediaUsageRepository { return Number(result.numUpdatedRows ?? 0) > 0; } + async prepareIncrementalFinalization( + input: MediaUsageIncrementalStatusIdentity, + ): Promise< + { outcome: "marked"; marker: string } | { outcome: "not_required" } | { outcome: "lost" } + > { + const observed = await this.db + .selectFrom("_emdash_media_usage_index_status") + .select(["change_epoch", "reconciliation_required"]) + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", input.collectionSlug) + .where("collection_id", "=", input.collectionId) + .where("capture_state", "=", "active") + .executeTakeFirst(); + if (!observed) return { outcome: "lost" }; + if (observed.reconciliation_required !== 0) return { outcome: "not_required" }; + const marker = `incremental-finalize:${String(observed.change_epoch)}:${ulid()}`; + const result = await this.db + .updateTable("_emdash_media_usage_index_status") + .set({ cursor: marker, updated_at: this.sortableUtcTimestamp() }) + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", input.collectionSlug) + .where("collection_id", "=", input.collectionId) + .where("capture_state", "=", "active") + .where("reconciliation_required", "=", 0) + .where("change_epoch", "=", observed.change_epoch) + .where( + sql`EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + WHERE collection.id = ${input.collectionId} + AND collection.slug = ${input.collectionSlug} + )`, + ) + .where( + sql`NOT EXISTS ( + SELECT 1 + FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = ${input.collectionId} + )`, + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) > 0 + ? { outcome: "marked", marker } + : { outcome: "lost" }; + } + + async recoverIncrementalFinalizations(): Promise { + const markerMatchesEpoch = isPostgres(this.db) + ? sql`status.cursor LIKE ('incremental-finalize:' || status.change_epoch::text || ':%')` + : sql`status.cursor LIKE ('incremental-finalize:' || CAST(status.change_epoch AS text) || ':%')`; + const now = new Date().toISOString(); + const result = await sql<{ collection_id: string }>` + UPDATE _emdash_media_usage_index_status AS status + SET status = 'complete', + completed_at = ${now}, + cursor = NULL, + last_error_code = NULL, + last_incremental_success_at = ${now}, + updated_at = ${now} + WHERE status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.capture_state = 'active' + AND status.reconciliation_required = 0 + AND status.cursor IS NOT NULL + AND ${markerMatchesEpoch} + AND EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + WHERE collection.id = status.collection_id + AND collection.slug = status.scope_key + ) + AND NOT EXISTS ( + SELECT 1 + FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = status.collection_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM _emdash_media_usage_work AS work + WHERE work.collection_id = status.collection_id + ) + RETURNING collection_id + `.execute(this.db); + return result.rows.length; + } + async recordIncrementalFailure( input: MediaUsageIncrementalStatusIdentity & { contentId: string; @@ -1778,6 +2548,71 @@ export class MediaUsageRepository { return Number(result.numUpdatedRows ?? 0) > 0; } + async recordIncrementalFailuresByCollection(input: { + collectionIds: readonly string[]; + errorCode: string; + }): Promise> { + const collectionIds = [...new Set(input.collectionIds)]; + if (collectionIds.length === 0) return new Set(); + const collections = jsonTextValues(this.db, collectionIds); + const now = new Date().toISOString(); + const result = await sql<{ collection_id: string }>` + WITH failed_collections AS (${collections}) + UPDATE _emdash_media_usage_index_status AS status + SET status = CASE + WHEN EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = status.collection_id + AND reconciliation.run_token = status.cursor + ) THEN status.status + WHEN status.reconciliation_required = 0 THEN 'partial' + WHEN status.status = 'running' THEN 'stale' + ELSE status.status + END, + completed_at = CASE + WHEN EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = status.collection_id + AND reconciliation.run_token = status.cursor + ) THEN status.completed_at + WHEN status.reconciliation_required = 0 OR status.status = 'running' THEN NULL + ELSE status.completed_at + END, + cursor = CASE + WHEN EXISTS ( + SELECT 1 + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = status.collection_id + AND reconciliation.run_token = status.cursor + ) THEN status.cursor + WHEN status.status = 'running' THEN NULL + ELSE status.cursor + END, + last_error_code = ${input.errorCode}, + updated_at = ${now} + WHERE status.adapter_id = 'content-media' + AND status.scope_type = 'collection' + AND status.collection_id IN (SELECT value FROM failed_collections) + AND EXISTS ( + SELECT 1 + FROM _emdash_media_usage_work AS work + WHERE work.collection_id = status.collection_id + AND work.state = 'failed' + AND work.last_error_code = ${input.errorCode} + ) + AND EXISTS ( + SELECT 1 + FROM _emdash_collections AS collection + WHERE collection.id = status.collection_id + AND collection.slug = status.scope_key + ) + RETURNING collection_id + `.execute(this.db); + return new Set(result.rows.map((row) => row.collection_id)); + } + async findIndexStatus( identity: MediaUsageIndexStatusIdentity, ): Promise { @@ -2204,6 +3039,199 @@ export class MediaUsageRepository { } } + private async insertBatchOccurrences(rows: readonly BatchOccurrenceRow[]): Promise { + for (const rowBatch of chunkJsonRows(rows)) { + const input = this.occurrenceBatchInput(JSON.stringify(rowBatch)); + await sql` + WITH input AS (${input}) + INSERT INTO _emdash_media_usage ( + id, source_key, generation, field_slug, field_path, occurrence_index, + reference_type, media_id, provider, provider_asset_id, media_kind, + mime_type, created_at + ) + SELECT + id, source_key, generation, field_slug, field_path, occurrence_index, + reference_type, media_id, provider, provider_asset_id, media_kind, + mime_type, created_at + FROM input + WHERE EXISTS ( + SELECT 1 + FROM _emdash_media_usage_generation_writes AS writer + WHERE writer.source_key = input.source_key + AND writer.generation = input.generation + AND writer.lease_token = input.lease_token + AND ${this.generationWriteLeaseExpiryIsInFuture("writer.expires_at")} + ) + `.execute(this.db); + } + } + + private occurrenceBatchInput(payload: string): RawBuilder { + if (isPostgres(this.db)) { + return sql` + SELECT + entry.value ->> 'id' AS id, + entry.value ->> 'source_key' AS source_key, + entry.value ->> 'generation' AS generation, + entry.value ->> 'field_slug' AS field_slug, + entry.value ->> 'field_path' AS field_path, + CAST(entry.value ->> 'occurrence_index' AS integer) AS occurrence_index, + entry.value ->> 'reference_type' AS reference_type, + entry.value ->> 'media_id' AS media_id, + entry.value ->> 'provider' AS provider, + entry.value ->> 'provider_asset_id' AS provider_asset_id, + entry.value ->> 'media_kind' AS media_kind, + entry.value ->> 'mime_type' AS mime_type, + entry.value ->> 'created_at' AS created_at, + entry.value ->> 'lease_token' AS lease_token + FROM jsonb_array_elements(${payload}::jsonb) AS entry(value) + `; + } + return sql` + SELECT + json_extract(entry.value, '$.id') AS id, + json_extract(entry.value, '$.source_key') AS source_key, + json_extract(entry.value, '$.generation') AS generation, + json_extract(entry.value, '$.field_slug') AS field_slug, + json_extract(entry.value, '$.field_path') AS field_path, + CAST(json_extract(entry.value, '$.occurrence_index') AS integer) AS occurrence_index, + json_extract(entry.value, '$.reference_type') AS reference_type, + json_extract(entry.value, '$.media_id') AS media_id, + json_extract(entry.value, '$.provider') AS provider, + json_extract(entry.value, '$.provider_asset_id') AS provider_asset_id, + json_extract(entry.value, '$.media_kind') AS media_kind, + json_extract(entry.value, '$.mime_type') AS mime_type, + json_extract(entry.value, '$.created_at') AS created_at, + json_extract(entry.value, '$.lease_token') AS lease_token + FROM json_each(${payload}) AS entry + `; + } + + private sourceBatchInput(payload: string): RawBuilder { + if (isPostgres(this.db)) { + return sql` + SELECT + entry.value ->> 'source_key' AS source_key, + entry.value ->> 'source_type' AS source_type, + entry.value ->> 'collection_id' AS collection_id, + entry.value ->> 'collection_slug' AS collection_slug, + entry.value ->> 'content_id' AS content_id, + entry.value ->> 'source_variant' AS source_variant, + entry.value ->> 'locale' AS locale, + entry.value ->> 'translation_group' AS translation_group, + entry.value ->> 'content_slug' AS content_slug, + entry.value ->> 'content_title' AS content_title, + entry.value ->> 'content_status' AS content_status, + entry.value ->> 'content_scheduled_at' AS content_scheduled_at, + entry.value ->> 'content_deleted_at' AS content_deleted_at, + entry.value ->> 'revision_id' AS revision_id, + entry.value ->> 'current_generation' AS current_generation, + CAST(entry.value ->> 'schema_version' AS integer) AS schema_version, + entry.value ->> 'source_updated_at' AS source_updated_at, + CAST(entry.value ->> 'source_version' AS bigint) AS source_version, + entry.value ->> 'source_fingerprint' AS source_fingerprint, + CAST(entry.value ->> 'identity_version' AS integer) AS identity_version, + entry.value ->> 'source_completeness' AS source_completeness, + entry.value ->> 'last_attempted_at' AS last_attempted_at, + entry.value ->> 'last_error_code' AS last_error_code, + entry.value ->> 'indexed_at' AS indexed_at, + entry.value ->> 'updated_at' AS updated_at, + entry.value ->> 'lease_token' AS lease_token, + entry.value ->> 'expected_generation' AS expected_generation, + entry.value ->> 'expected_collection_id' AS expected_collection_id, + entry.value ->> 'expected_updated_at' AS expected_updated_at, + entry.value ->> 'expected_source_fingerprint' AS expected_source_fingerprint, + entry.value ->> 'expected_source_updated_at' AS expected_source_updated_at, + CAST(entry.value ->> 'expected_source_version' AS bigint) AS expected_source_version, + CAST(entry.value ->> 'expected_identity_version' AS integer) AS expected_identity_version, + entry.value ->> 'expected_revision_id' AS expected_revision_id, + entry.value ->> 'expected_source_completeness' AS expected_source_completeness, + entry.value ->> 'expected_last_attempted_at' AS expected_last_attempted_at, + entry.value ->> 'expected_last_error_code' AS expected_last_error_code + FROM jsonb_array_elements(${payload}::jsonb) AS entry(value) + `; + } + return sql` + SELECT + json_extract(entry.value, '$.source_key') AS source_key, + json_extract(entry.value, '$.source_type') AS source_type, + json_extract(entry.value, '$.collection_id') AS collection_id, + json_extract(entry.value, '$.collection_slug') AS collection_slug, + json_extract(entry.value, '$.content_id') AS content_id, + json_extract(entry.value, '$.source_variant') AS source_variant, + json_extract(entry.value, '$.locale') AS locale, + json_extract(entry.value, '$.translation_group') AS translation_group, + json_extract(entry.value, '$.content_slug') AS content_slug, + json_extract(entry.value, '$.content_title') AS content_title, + json_extract(entry.value, '$.content_status') AS content_status, + json_extract(entry.value, '$.content_scheduled_at') AS content_scheduled_at, + json_extract(entry.value, '$.content_deleted_at') AS content_deleted_at, + json_extract(entry.value, '$.revision_id') AS revision_id, + json_extract(entry.value, '$.current_generation') AS current_generation, + CAST(json_extract(entry.value, '$.schema_version') AS integer) AS schema_version, + json_extract(entry.value, '$.source_updated_at') AS source_updated_at, + CAST(json_extract(entry.value, '$.source_version') AS integer) AS source_version, + json_extract(entry.value, '$.source_fingerprint') AS source_fingerprint, + CAST(json_extract(entry.value, '$.identity_version') AS integer) AS identity_version, + json_extract(entry.value, '$.source_completeness') AS source_completeness, + json_extract(entry.value, '$.last_attempted_at') AS last_attempted_at, + json_extract(entry.value, '$.last_error_code') AS last_error_code, + json_extract(entry.value, '$.indexed_at') AS indexed_at, + json_extract(entry.value, '$.updated_at') AS updated_at, + json_extract(entry.value, '$.lease_token') AS lease_token, + json_extract(entry.value, '$.expected_generation') AS expected_generation, + json_extract(entry.value, '$.expected_collection_id') AS expected_collection_id, + json_extract(entry.value, '$.expected_updated_at') AS expected_updated_at, + json_extract(entry.value, '$.expected_source_fingerprint') AS expected_source_fingerprint, + json_extract(entry.value, '$.expected_source_updated_at') AS expected_source_updated_at, + CAST(json_extract(entry.value, '$.expected_source_version') AS integer) AS expected_source_version, + CAST(json_extract(entry.value, '$.expected_identity_version') AS integer) AS expected_identity_version, + json_extract(entry.value, '$.expected_revision_id') AS expected_revision_id, + json_extract(entry.value, '$.expected_source_completeness') AS expected_source_completeness, + json_extract(entry.value, '$.expected_last_attempted_at') AS expected_last_attempted_at, + json_extract(entry.value, '$.expected_last_error_code') AS expected_last_error_code + FROM json_each(${payload}) AS entry + `; + } + + private generationWriteBatchInput(payload: string): RawBuilder { + if (isPostgres(this.db)) { + return sql` + SELECT + entry.value ->> 'source_key' AS source_key, + entry.value ->> 'collection_id' AS collection_id, + entry.value ->> 'collection_slug' AS collection_slug, + entry.value ->> 'generation' AS generation, + entry.value ->> 'lease_token' AS lease_token, + entry.value ->> 'expires_at' AS expires_at, + entry.value ->> 'created_at' AS created_at + FROM jsonb_array_elements(${payload}::jsonb) AS entry(value) + `; + } + return sql` + SELECT + json_extract(entry.value, '$.source_key') AS source_key, + json_extract(entry.value, '$.collection_id') AS collection_id, + json_extract(entry.value, '$.collection_slug') AS collection_slug, + json_extract(entry.value, '$.generation') AS generation, + json_extract(entry.value, '$.lease_token') AS lease_token, + json_extract(entry.value, '$.expires_at') AS expires_at, + json_extract(entry.value, '$.created_at') AS created_at + FROM json_each(${payload}) AS entry + `; + } + + private batchRefsMatch( + left: BatchSourceExpectedColumn, + right: BatchInputExpectedColumn, + ): RawBuilder { + const leftRef = sql.ref(left); + const rightRef = sql.ref(right); + return isPostgres(this.db) + ? sql`${leftRef} IS NOT DISTINCT FROM ${rightRef}` + : sql`${leftRef} IS ${rightRef}`; + } + private async lockCanonicalSourceCollection( db: DatabaseExecutor, source: MediaUsageSourceInput, diff --git a/packages/core/src/database/transaction.ts b/packages/core/src/database/transaction.ts index 238d586dcc..ebd60a051b 100644 --- a/packages/core/src/database/transaction.ts +++ b/packages/core/src/database/transaction.ts @@ -29,6 +29,7 @@ export async function withTransaction( db: Kysely, fn: (trx: Kysely | Transaction) => Promise, ): Promise { + if (db.isTransaction) return fn(db); // Fast path: we already know transactions work if (transactionsSupported === true) { return db.transaction().execute(fn); diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index 33ac7bb8c0..4565ac6b98 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -9,7 +9,7 @@ import { Permissions } from "@emdash-cms/auth"; import type { Element } from "@emdash-cms/blocks"; -import { Kysely, sql, type Dialect } from "kysely"; +import { Kysely, type Dialect } from "kysely"; import virtualConfig from "virtual:emdash/config"; import { z } from "zod"; @@ -48,25 +48,14 @@ import { repairLocaleCasing } from "./i18n/repair-locale-casing.js"; import { warnAboutUnconfiguredTaxonomyLocales } from "./i18n/taxonomy-locale-diagnostic.js"; import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; -import { - MEDIA_USAGE_COLLECTION_DELETION_LIMITS, - processDueMediaUsageCollectionDeletions, -} from "./media/usage/collection-deletion-processor.js"; +import { activateMediaUsageCapture } from "./media/usage/activation.js"; import { deleteContentMediaUsage, findNonTranslatableSiblingContentIds, markContentMediaUsageCollectionStale, refreshContentMediaUsageAfterWrite, } from "./media/usage/content-refresh.js"; -import { - MEDIA_USAGE_RECONCILIATION_LIMITS, - processDueMediaUsageReconciliation, -} from "./media/usage/reconciliation-processor.js"; -import { - MEDIA_USAGE_WORK_PROCESSING_LIMITS, - processDueMediaUsageWork, - processMediaUsageWorkAfterWrite, -} from "./media/usage/work-processor.js"; +import { processMediaUsageWorkAfterWrite } from "./media/usage/work-processor.js"; import { createSandboxRunnerOptions } from "./plugins/sandbox/runner-options.js"; import { getSandboxRouteErrorDetails } from "./plugins/sandbox/types.js"; import type { @@ -553,62 +542,6 @@ const marketplaceManifestCache = new Map< const sandboxedRouteMetaCache = new Map>(); let sandboxRunner: SandboxRunner | null = null; -export const MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS = Object.freeze({ - entryWork: MEDIA_USAGE_WORK_PROCESSING_LIMITS.ordinaryStatementsPerJob, - collectionDeletion: MEDIA_USAGE_COLLECTION_DELETION_LIMITS.maxQueriesPerTick, - reconciliation: MEDIA_USAGE_RECONCILIATION_LIMITS.maxQueriesPerTick, - maxClassQueries: Math.max( - MEDIA_USAGE_WORK_PROCESSING_LIMITS.ordinaryStatementsPerJob, - MEDIA_USAGE_COLLECTION_DELETION_LIMITS.maxQueriesPerTick, - MEDIA_USAGE_RECONCILIATION_LIMITS.maxQueriesPerTick, - ), - eventCeiling: 40, -}); - -export type MediaUsageMaintenanceTaskClass = - | "entry_work" - | "collection_deletion" - | "reconciliation"; - -export type MediaUsageMaintenanceResult = - | { outcome: "inactive" | "admission_closed"; taskClass: null; turn: null } - | { outcome: "processed"; taskClass: MediaUsageMaintenanceTaskClass; turn: number }; - -async function runScheduledMediaUsageLane( - db: Kysely, -): Promise { - const queriesAlreadySpent = getRequestContext()?.metrics?.dbCount ?? 0; - if ( - queriesAlreadySpent + 1 + MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.maxClassQueries > - MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling - ) { - return { outcome: "admission_closed", taskClass: null, turn: null }; - } - - const activation = await db - .updateTable("_emdash_media_usage_activation") - .set({ - media_usage_maintenance_turn: sql`(media_usage_maintenance_turn + 1) % 3`, - }) - .where("task_key", "=", "incremental_capture") - .where("state", "=", "active") - .returning("media_usage_maintenance_turn") - .executeTakeFirst(); - if (!activation) return { outcome: "inactive", taskClass: null, turn: null }; - - const turn = activation.media_usage_maintenance_turn; - if (turn === 0) { - await processDueMediaUsageWork(db); - return { outcome: "processed", taskClass: "entry_work", turn }; - } - if (turn === 1) { - await processDueMediaUsageCollectionDeletions(db); - return { outcome: "processed", taskClass: "collection_deletion", turn }; - } - await processDueMediaUsageReconciliation(db); - return { outcome: "processed", taskClass: "reconciliation", turn }; -} - /** * EmDashRuntime - singleton per worker */ @@ -814,10 +747,6 @@ export class EmDashRuntime { return { published }; } - async runScheduledMediaUsageTasks(): Promise { - return runScheduledMediaUsageLane(this.db); - } - /** * Materialize plugin-declared storage indexes, once per process. * @@ -1448,6 +1377,15 @@ export class EmDashRuntime { // per-isolate lock keyed by the configured db so a reclaimed-and-rerun // create() can't apply the seed a second time concurrently. if (seedGate.collectionCount === 0 && !seedGate.setupDone) { + try { + const activation = await activateMediaUsageCapture(db, { writersDrained: true }); + if (activation.outcome !== "active") { + throw new Error("Fresh-site media usage activation did not complete"); + } + } catch (error) { + await disposeReadDb(); + throw error; + } const seedKey = deps.config.database?.entrypoint ?? "default"; const seedHolder = getSeedHolder(); try { @@ -1756,14 +1694,6 @@ export class EmDashRuntime { if (deps.createScheduler) { const scheduler = deps.createScheduler(cronExecutor); cronScheduler = scheduler; - const runMediaUsageMaintenance = async () => { - const runtime = runtimeRef.current; - if (runtime) { - await runtime.runScheduledMediaUsageTasks(); - } else { - await runScheduledMediaUsageLane(db); - } - }; // Run scheduled publishing and system cleanup alongside each tick. // Pass storage so cleanupPendingUploads can delete orphaned files. @@ -1797,16 +1727,7 @@ export class EmDashRuntime { // Never throws; no-op unless scheduled backups are enabled and due. await maybeRunScheduledBackup(db, storage ?? undefined); await recordSchedulerHeartbeatSafely(db); - if (!scheduler.setMediaUsageMaintenance) { - try { - await runMediaUsageMaintenance(); - } catch (error) { - console.error("[media-usage] Scheduled maintenance failed:", error); - } - } }); - scheduler.setMediaUsageMaintenance?.(runMediaUsageMaintenance); - // start() is void on the timer scheduler but the interface // allows a promise (alarm-backed schedulers); we don't block on it. void scheduler.start(); @@ -3700,14 +3621,14 @@ export class EmDashRuntime { for (const contentId of new Set(contentIds)) { try { const work = await processMediaUsageWorkAfterWrite(this.db, collection, contentId); - if (work.outcome !== "inactive") return; + if (work.outcome !== "inactive") continue; await refreshContentMediaUsageAfterWrite(this.db, collection, contentId); } catch (error) { console.error( `[media-usage] Failed after content write ${collection}/${contentId}:`, error, ); - return; + continue; } } } diff --git a/packages/core/src/media/usage/activation.ts b/packages/core/src/media/usage/activation.ts index 993cea8285..a4f1eb4086 100644 --- a/packages/core/src/media/usage/activation.ts +++ b/packages/core/src/media/usage/activation.ts @@ -41,6 +41,12 @@ export type MediaUsageActivationResult = | { outcome: "lease_active"; leaseExpiresAt: string } | { outcome: "conflict"; processedCollections: number }; +export type MediaUsageActivationContinuationResult = + | MediaUsageActivationResult + | { outcome: "inactive" | "failed" }; + +type MediaUsageActivationClaimMode = "confirmed" | "background"; + export interface MediaUsageCollectionCapturePreparation { captureRequired: boolean; collectionId: string; @@ -274,16 +280,45 @@ export async function activateMediaUsageCapture( if (input.writersDrained !== true) { throw new Error("Media usage activation requires confirmation that writers are drained"); } + return advanceMediaUsageActivation(db, "confirmed"); +} +export async function continueMediaUsageActivation( + db: Kysely, +): Promise { + return advanceMediaUsageActivation(db, "background"); +} + +async function advanceMediaUsageActivation( + db: Kysely, + mode: "confirmed", +): Promise; +async function advanceMediaUsageActivation( + db: Kysely, + mode: "background", +): Promise; +async function advanceMediaUsageActivation( + db: Kysely, + mode: MediaUsageActivationClaimMode, +): Promise { const before = await findActivation(db); assertRuntimeGeneration(before); if (before.state === "active") { return { outcome: "active", processedCollections: 0 }; } + if (mode === "background") { + if (before.state === "expanded" || before.drain_confirmed_at === null) { + return { outcome: "inactive" }; + } + if (before.state !== "activating") { + throw new Error("Invalid media usage activation state"); + } + if (before.last_error_code !== null) return { outcome: "failed" }; + } const leaseToken = ulid(); - const lease = await claimActivation(db, leaseToken); - if (!lease) return activationClaimLoss(db); + const lease = await claimActivation(db, leaseToken, mode); + if (!lease) return activationClaimLoss(db, mode); let processedCollections = 0; try { @@ -377,47 +412,76 @@ function assertRuntimeGeneration(activation: { runtime_generation: number }): vo async function claimActivation( db: Kysely, leaseToken: string, + mode: MediaUsageActivationClaimMode, ): Promise | null> { const now = timestampOffset(db, 0); - return ( - (await db - .updateTable("_emdash_media_usage_activation") - .set({ - state: "activating", - drain_confirmed_at: now, - lease_token: leaseToken, - lease_expires_at: timestampOffset(db, MEDIA_USAGE_ACTIVATION_LIMITS.leaseDurationSeconds), - attempt_count: sql`attempt_count + 1`, - last_attempted_at: now, - last_error_code: null, - updated_at: now, - }) - .where("task_key", "=", ACTIVATION_KEY) - .where("runtime_generation", "=", MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION) - .where((eb) => - eb.or([ - eb("state", "=", "expanded"), - eb.and([ - eb("state", "=", "activating"), - eb.or([ - eb("lease_token", "is", null), - eb.and([ - eb("lease_expires_at", "is not", null), - timestampIsDue(db, "lease_expires_at"), - ]), + const claimValues = { + lease_token: leaseToken, + lease_expires_at: timestampOffset(db, MEDIA_USAGE_ACTIVATION_LIMITS.leaseDurationSeconds), + attempt_count: sql`attempt_count + 1`, + last_attempted_at: now, + updated_at: now, + }; + let claim = db + .updateTable("_emdash_media_usage_activation") + .set( + mode === "confirmed" + ? { + ...claimValues, + state: "activating", + drain_confirmed_at: now, + last_error_code: null, + } + : claimValues, + ) + .where("task_key", "=", ACTIVATION_KEY) + .where("runtime_generation", "=", MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION); + if (mode === "confirmed") { + claim = claim.where((eb) => + eb.or([ + eb("state", "=", "expanded"), + eb.and([ + eb("state", "=", "activating"), + eb.or([ + eb("lease_token", "is", null), + eb.and([ + eb("lease_expires_at", "is not", null), + timestampIsDue(db, "lease_expires_at"), ]), ]), ]), - ) - .returningAll() - .executeTakeFirst()) ?? null - ); + ]), + ); + } else { + claim = claim + .where("state", "=", "activating") + .where("drain_confirmed_at", "is not", null) + .where("last_error_code", "is", null) + .where((eb) => + eb.or([ + eb("lease_token", "is", null), + eb.and([eb("lease_expires_at", "is not", null), timestampIsDue(db, "lease_expires_at")]), + ]), + ); + } + return (await claim.returningAll().executeTakeFirst()) ?? null; } -async function activationClaimLoss(db: Kysely): Promise { +async function activationClaimLoss( + db: Kysely, + mode: MediaUsageActivationClaimMode, +): Promise { const current = await findActivation(db); assertRuntimeGeneration(current); if (current.state === "active") return { outcome: "active", processedCollections: 0 }; + if (mode === "background") { + if (current.state === "expanded" || current.drain_confirmed_at === null) { + return { outcome: "inactive" }; + } + if (current.state === "activating" && current.last_error_code !== null) { + return { outcome: "failed" }; + } + } if ( current.state === "activating" && current.lease_token && @@ -426,6 +490,9 @@ async function activationClaimLoss(db: Kysely): Promise { const identifiers = await captureIdentifiers(identity); if (isPostgres(db)) await installPostgresFunction(db); - const installedNames = await listOwnedCaptureTriggers(db, identifiers.tableName); + const installedNames = await listOwnedCaptureTriggers( + db, + identifiers.tableName, + MAX_OWNED_CAPTURE_TRIGGERS + 1, + ); + if (installedNames.length > MAX_OWNED_CAPTURE_TRIGGERS) { + throw new Error("Media usage capture trigger set exceeds the activation limit"); + } if (await hasExactCaptureTriggers(db, identifiers, identity, installedNames)) { return; } @@ -77,7 +85,7 @@ export async function verifyMediaUsageCaptureTriggers( db, identifiers, identity, - await listOwnedCaptureTriggers(db, identifiers.tableName), + await listOwnedCaptureTriggers(db, identifiers.tableName, MAX_OWNED_CAPTURE_TRIGGERS + 1), ); } @@ -150,7 +158,9 @@ async function removeCaptureTriggers( async function listOwnedCaptureTriggers( db: Kysely, tableName: string, + limit?: number, ): Promise { + const limitClause = limit === undefined ? sql`` : sql`LIMIT ${limit}`; if (isPostgres(db)) { const result = await sql<{ name: string }>` SELECT trigger.tgname AS name @@ -161,6 +171,7 @@ async function listOwnedCaptureTriggers( AND relation.relname = ${tableName} AND NOT trigger.tgisinternal AND left(trigger.tgname, 10) = ${OWNED_TRIGGER_PREFIX} + ${limitClause} `.execute(db); return result.rows.map((row) => row.name); } @@ -171,6 +182,7 @@ async function listOwnedCaptureTriggers( WHERE type = 'trigger' AND tbl_name = ${tableName} AND substr(name, 1, 10) = ${OWNED_TRIGGER_PREFIX} + ${limitClause} `.execute(db); return result.rows.map((row) => row.name); } @@ -307,7 +319,11 @@ async function assertExpectedTriggerSet( }, identity: MediaUsageCaptureIdentity, ): Promise { - const actual = await listOwnedCaptureTriggers(db, identifiers.tableName); + const actual = await listOwnedCaptureTriggers( + db, + identifiers.tableName, + MAX_OWNED_CAPTURE_TRIGGERS + 1, + ); if (!(await hasExactCaptureTriggers(db, identifiers, identity, actual))) { throw new Error("Media usage capture trigger installation is incomplete"); } diff --git a/packages/core/src/media/usage/collection-deletion-processor.ts b/packages/core/src/media/usage/collection-deletion-processor.ts index f0adb7ce98..db76d8c436 100644 --- a/packages/core/src/media/usage/collection-deletion-processor.ts +++ b/packages/core/src/media/usage/collection-deletion-processor.ts @@ -12,13 +12,11 @@ import { export const MEDIA_USAGE_COLLECTION_DELETION_LIMITS = Object.freeze({ candidatesPerTick: 4, - deletionsPerTick: 1, rowsPerBatch: 50, leaseDurationSeconds: 5 * 60, maxAttempts: 5, retryBaseSeconds: 30, retryMaxSeconds: 15 * 60, - maxQueriesPerTick: 30, }); export interface MediaUsageCollectionDeletionTickResult { @@ -36,7 +34,13 @@ export async function processDueMediaUsageCollectionDeletions( const candidates = await repository.findDue( MEDIA_USAGE_COLLECTION_DELETION_LIMITS.candidatesPerTick, ); - if (candidates.length === 0) return { candidateCount: 0, claimedCount: 0, outcome: "idle" }; + if (candidates.length === 0) { + return { + candidateCount: (await repository.hasNonterminal()) ? 1 : 0, + claimedCount: 0, + outcome: "idle", + }; + } let claim: (MediaUsageCollectionDeletionRecord & { leaseToken: string }) | null = null; for (const candidate of candidates) { diff --git a/packages/core/src/media/usage/collection-deletion.ts b/packages/core/src/media/usage/collection-deletion.ts index bb762cb0ac..94dde94833 100644 --- a/packages/core/src/media/usage/collection-deletion.ts +++ b/packages/core/src/media/usage/collection-deletion.ts @@ -193,6 +193,16 @@ export class MediaUsageCollectionDeletionRepository { return result.rows.map(rowToRecord); } + async hasNonterminal(): Promise { + const row = await this.db + .selectFrom("_emdash_media_usage_collection_deletions") + .select("collection_id") + .where("state", "in", ["pending", "retry", "leased"]) + .limit(1) + .executeTakeFirst(); + return row !== undefined; + } + async findOperatorPage(options: { state?: MediaUsageCollectionDeletionState; limit?: number; diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index 6f648c425d..317e059874 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -2,17 +2,24 @@ import { sql, type Kysely } from "kysely"; import { tableExists } from "../../database/dialect-helpers.js"; import { + type MediaUsageExistingSourceProjection, MediaUsageRepository, + type MediaUsageNewSourceProjection, type MediaUsageSource, } from "../../database/repositories/media-usage.js"; import type { Database } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; import { isI18nEnabled } from "../../i18n/config.js"; -import { loadContentMediaUsageFields } from "./content-fields.js"; +import { + loadContentMediaUsageFields, + type ContentMediaUsageFieldDiscovery, +} from "./content-fields.js"; import { CONTENT_SOURCE_SCHEMA_VERSION, loadContentMediaUsageSnapshots, + loadContentMediaUsageSnapshotsBatch, type ContentMediaUsageSnapshot, + type LoadContentMediaUsageSnapshotsResult, } from "./content-snapshots.js"; import { buildContentMediaUsageSourceKey, @@ -27,8 +34,11 @@ const CONTENT_USAGE_COLLECTION_LOCKS_KEY = Symbol.for("emdash.mediaUsage.collect const CONTENT_USAGE_REFRESH_MAX_ATTEMPTS = 2; export const MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS = Object.freeze({ - maxOccurrenceMutationUnitsPerClaim: 12, - maxProjectionMutationBytesPerClaim: 512 * 1024, + maxOccurrenceMutationUnitsPerClaim: 500, + maxProjectionMutationBytesPerVariant: 2_000_000, + maxProjectionMutationBytesPerClaim: 4_000_000, + maxOccurrenceMutationUnitsPerBatch: 50_000, + maxProjectionMutationBytesPerBatch: 16_000_000, }); export interface ContentMediaUsageAdmissionBudget { @@ -66,6 +76,9 @@ interface ContentMediaUsageRefreshOptions { collectionId?: string; durableWork?: boolean; admissionBudget?: ContentMediaUsageAdmissionBudget; + fieldDiscovery?: ContentMediaUsageFieldDiscovery; + observedSources?: ReadonlyMap; + snapshotsResult?: LoadContentMediaUsageSnapshotsResult; } export interface ContentMediaUsageRefreshResult { @@ -83,11 +96,18 @@ const ZERO_RESULT: ContentMediaUsageRefreshResult = { failedSourceCount: 0, }; -export function createContentMediaUsageAdmissionBudget(): ContentMediaUsageAdmissionBudget { +export function createContentMediaUsageAdmissionBudget( + limits: { + maxOccurrenceMutationUnits?: number; + maxProjectionMutationBytes?: number; + } = {}, +): ContentMediaUsageAdmissionBudget { return { remainingOccurrenceMutationUnits: + limits.maxOccurrenceMutationUnits ?? MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxOccurrenceMutationUnitsPerClaim, remainingProjectionMutationBytes: + limits.maxProjectionMutationBytes ?? MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxProjectionMutationBytesPerClaim, hasReservedMutation: false, }; @@ -107,6 +127,7 @@ export async function planContentMediaUsageProjectionAdmission( .filter((source): source is MediaUsageSource => source !== undefined); let deletionOccurrenceUnits = 0; let deletionBytes = 0; + let largestDeletionBytes = 0; for (const source of absentSources) { const measurement = await repo.measureSourceGenerationDeletion( source.sourceKey, @@ -119,7 +140,10 @@ export async function planContentMediaUsageProjectionAdmission( : { outcome: "intrinsic_resource_limit" }; } deletionOccurrenceUnits += measurement.occurrenceCount; - deletionBytes += storedMediaUsageSourceByteLength(source) + measurement.occurrenceBytes * 2; + const sourceDeletionBytes = + storedMediaUsageSourceByteLength(source) + measurement.occurrenceBytes * 2; + deletionBytes += sourceDeletionBytes; + largestDeletionBytes = Math.max(largestDeletionBytes, sourceDeletionBytes); } const noOpSourceKeys = new Set(); @@ -128,6 +152,7 @@ export async function planContentMediaUsageProjectionAdmission( noOpSourceKeys, deletionOccurrenceUnits, deletionBytes, + largestDeletionBytes, ); if (exceedsProjectionAdmissionLimits(cost)) { for (const snapshot of snapshots) { @@ -144,6 +169,7 @@ export async function planContentMediaUsageProjectionAdmission( noOpSourceKeys, deletionOccurrenceUnits, deletionBytes, + largestDeletionBytes, ); } @@ -175,6 +201,7 @@ export async function planContentMediaUsageProjectionAdmission( interface ProjectionAdmissionCost { occurrenceMutationUnits: number; projectionMutationBytes: number; + largestProjectionMutationBytes: number; } function projectionAdmissionCost( @@ -182,17 +209,23 @@ function projectionAdmissionCost( noOpSourceKeys: ReadonlySet, deletionOccurrenceUnits: number, deletionBytes: number, + largestDeletionBytes: number, ): ProjectionAdmissionCost { return snapshots.reduce( (cost, snapshot) => { if (noOpSourceKeys.has(snapshot.source.sourceKey)) return cost; cost.occurrenceMutationUnits += snapshot.occurrences.length; cost.projectionMutationBytes += snapshot.projectionByteLength; + cost.largestProjectionMutationBytes = Math.max( + cost.largestProjectionMutationBytes, + snapshot.projectionByteLength, + ); return cost; }, { occurrenceMutationUnits: deletionOccurrenceUnits, projectionMutationBytes: deletionBytes, + largestProjectionMutationBytes: largestDeletionBytes, }, ); } @@ -201,6 +234,8 @@ function exceedsProjectionAdmissionLimits(cost: ProjectionAdmissionCost): boolea return ( cost.occurrenceMutationUnits > MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxOccurrenceMutationUnitsPerClaim || + cost.largestProjectionMutationBytes > + MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxProjectionMutationBytesPerVariant || cost.projectionMutationBytes > MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxProjectionMutationBytesPerClaim ); @@ -223,22 +258,209 @@ export async function refreshContentMediaUsage( ); } -export async function refreshContentMediaUsageForWork( +export interface ContentMediaUsageWorkRefreshInput { + collectionId: string; + collectionSlug: string; + contentId: string; +} + +export interface ContentMediaUsageWorkBatchOptions { + shouldContinue?: () => boolean; +} + +export async function refreshContentMediaUsageForWorkBatch( db: Kysely, - collectionId: string, - collectionSlug: string, - contentId: string, -): Promise { - validateIdentifier(collectionSlug, "collection slug"); - if (!collectionId) throw new Error("Durable media usage work requires a collection identity"); - return withContentUsageCollectionLock(collectionSlug, () => - withContentUsageLock(collectionSlug, contentId, () => - refreshContentMediaUsageUnlocked(db, collectionSlug, contentId, { - collectionId, - durableWork: true, - }), - ), - ); + items: readonly ContentMediaUsageWorkRefreshInput[], + options: ContentMediaUsageWorkBatchOptions = {}, +): Promise> { + const results = new Map(); + const batchBudget = createContentMediaUsageAdmissionBudget({ + maxOccurrenceMutationUnits: + MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxOccurrenceMutationUnitsPerBatch, + maxProjectionMutationBytes: + MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxProjectionMutationBytesPerBatch, + }); + const collections = new Map(); + for (const item of items) { + const key = `${item.collectionId}\u0000${item.collectionSlug}`; + const collectionItems = collections.get(key) ?? []; + collectionItems.push(item); + collections.set(key, collectionItems); + } + for (const collectionItems of collections.values()) { + if (options.shouldContinue && !options.shouldContinue()) break; + const first = collectionItems[0]; + if (!first) continue; + validateIdentifier(first.collectionSlug, "collection slug"); + if (!first.collectionId) + throw new Error("Durable media usage work requires a collection identity"); + await withContentUsageCollectionLock(first.collectionSlug, async () => { + const fieldDiscovery = await loadContentMediaUsageFields( + db, + first.collectionSlug, + first.collectionId, + ); + const sourceKeys = collectionItems.flatMap((item) => + contentSourceKeys(item.collectionSlug, item.contentId, item.collectionId), + ); + const repo = new MediaUsageRepository(db); + const observedSources = await repo.findSources(sourceKeys); + const snapshots = await loadContentMediaUsageSnapshotsBatch( + db, + first.collectionSlug, + collectionItems.map((item) => item.contentId), + fieldDiscovery, + { collectionId: first.collectionId, identityVersion: 1 }, + { + shouldContinue: options.shouldContinue, + maxOccurrenceCount: + MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxOccurrenceMutationUnitsPerBatch, + maxProjectionBytes: + MEDIA_USAGE_PROJECTION_ADMISSION_LIMITS.maxProjectionMutationBytesPerBatch, + }, + ); + const newSourceProjections: MediaUsageNewSourceProjection[] = []; + const newSourceKeys = new Map(); + const existingSourceProjections: MediaUsageExistingSourceProjection[] = []; + const unchangedSourceProjections: MediaUsageExistingSourceProjection[] = []; + const existingSourceKeys = new Map< + string, + { allCount: number; changed: string[]; unchanged: string[] } + >(); + for (const item of collectionItems) { + if (options.shouldContinue && !options.shouldContinue()) break; + const snapshotsResult = snapshots.get(item.contentId); + const itemSourceKeys = contentSourceKeys( + item.collectionSlug, + item.contentId, + item.collectionId, + ); + if (!snapshotsResult?.success) { + continue; + } + const snapshotsByKey = new Map( + snapshotsResult.snapshots.map( + (snapshot) => [snapshot.source.sourceKey, snapshot] as const, + ), + ); + const existing = itemSourceKeys + .map((sourceKey) => observedSources.get(sourceKey)) + .filter((source): source is MediaUsageSource => source !== undefined); + const allNew = existing.length === 0; + const allExisting = + existing.length === snapshotsResult.snapshots.length && + existing.every((source) => snapshotsByKey.has(source.sourceKey)); + if (!allNew && !allExisting) continue; + const admission = await planContentMediaUsageProjectionAdmission( + repo, + snapshotsResult.snapshots, + observedSources, + itemSourceKeys, + batchBudget, + ); + if (admission.outcome === "claim_budget_deferred") break; + if (admission.outcome !== "admitted") { + results.set( + contentRefreshKey(item.collectionId, item.contentId), + admissionFailureResult(admission.outcome), + ); + continue; + } + const key = contentRefreshKey(item.collectionId, item.contentId); + if (allNew) { + newSourceProjections.push( + ...snapshotsResult.snapshots.map((snapshot) => ({ + source: snapshot.source, + occurrences: snapshot.occurrences, + })), + ); + newSourceKeys.set( + key, + snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey), + ); + continue; + } + const changed: string[] = []; + const unchanged: string[] = []; + for (const snapshot of snapshotsResult.snapshots) { + const expectedSource = observedSources.get(snapshot.source.sourceKey); + if (!expectedSource) continue; + if ( + expectedSource.sourceFingerprint === snapshot.source.sourceFingerprint && + expectedSource.sourceCompleteness === + (snapshot.source.sourceCompleteness ?? "complete") && + expectedSource.lastErrorCode === null + ) { + unchanged.push(snapshot.source.sourceKey); + unchangedSourceProjections.push({ + source: snapshot.source, + occurrences: snapshot.occurrences, + expectedSource, + }); + continue; + } + changed.push(snapshot.source.sourceKey); + existingSourceProjections.push({ + source: snapshot.source, + occurrences: snapshot.occurrences, + expectedSource, + }); + } + existingSourceKeys.set(key, { + allCount: snapshotsResult.snapshots.length, + changed, + unchanged, + }); + } + const insertedSourceKeys = await repo.replaceNewSourcesBatch(newSourceProjections); + const replacedSourceKeys = await repo.replaceExistingSourcesBatch(existingSourceProjections); + const matchedSourceKeys = await repo.matchingExistingSourcesBatch(unchangedSourceProjections); + for (const [key, expectedSourceKeys] of newSourceKeys) { + results.set( + key, + expectedSourceKeys.every((sourceKey) => insertedSourceKeys.has(sourceKey)) + ? { + success: true, + refreshedSourceCount: expectedSourceKeys.length, + deletedSourceCount: 0, + failedSourceCount: 0, + } + : generationConflictResult({ refreshedSourceCount: 0, deletedSourceCount: 0 }), + ); + } + for (const [key, expected] of existingSourceKeys) { + results.set( + key, + expected.changed.every((sourceKey) => replacedSourceKeys.has(sourceKey)) && + expected.unchanged.every((sourceKey) => matchedSourceKeys.has(sourceKey)) + ? { + success: true, + refreshedSourceCount: expected.allCount, + deletedSourceCount: 0, + failedSourceCount: 0, + } + : generationConflictResult({ refreshedSourceCount: 0, deletedSourceCount: 0 }), + ); + } + for (const item of collectionItems) { + if (options.shouldContinue && !options.shouldContinue()) break; + const key = contentRefreshKey(item.collectionId, item.contentId); + if (!snapshots.has(item.contentId)) continue; + if (results.has(key)) continue; + const result = await withContentUsageLock(item.collectionSlug, item.contentId, () => + refreshContentMediaUsageUnlocked(db, item.collectionSlug, item.contentId, { + collectionId: item.collectionId, + durableWork: true, + fieldDiscovery, + observedSources, + snapshotsResult: snapshots.get(item.contentId), + }), + ); + results.set(key, result); + } + }); + } + return results; } async function refreshContentMediaUsageUnlocked( @@ -294,19 +516,18 @@ async function refreshContentMediaUsageAttempt( ): Promise { const repo = new MediaUsageRepository(db); const canonicalSourceKeys = contentSourceKeys(collectionSlug, contentId, options.collectionId); - const observedSources = await repo.findSources(canonicalSourceKeys); - const snapshotsResult = await loadContentMediaUsageSnapshots( - db, - collectionSlug, - contentId, - undefined, - options.collectionId ? { collectionId: options.collectionId, identityVersion: 1 } : undefined, - ); + const observedSources = options.observedSources ?? (await repo.findSources(canonicalSourceKeys)); + const snapshotsResult = + options.snapshotsResult ?? + (await loadContentMediaUsageSnapshots( + db, + collectionSlug, + contentId, + options.fieldDiscovery, + options.collectionId ? { collectionId: options.collectionId, identityVersion: 1 } : undefined, + )); if (!snapshotsResult.success) { if (snapshotsResult.error === "CONTENT_NOT_FOUND" && options.collectionId) { - if (!(await contentCollectionExists(db, collectionSlug, options.collectionId))) { - return generationConflictResult({ refreshedSourceCount: 0, deletedSourceCount: 0 }); - } if (!options.admissionBudget) throw new Error("Durable media usage work requires an admission budget"); const admission = await planContentMediaUsageProjectionAdmission( @@ -336,10 +557,7 @@ async function refreshContentMediaUsageAttempt( : markSnapshotFailure(db, collectionSlug, snapshotsResult); } - if (!(await contentCollectionExists(db, collectionSlug, options.collectionId))) { - if (options.collectionId) { - return generationConflictResult({ refreshedSourceCount: 0, deletedSourceCount: 0 }); - } + if (!options.collectionId && !(await contentCollectionExists(db, collectionSlug))) { const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); return { ...ZERO_RESULT, deletedSourceCount }; } @@ -381,10 +599,7 @@ async function refreshContentMediaUsageAttempt( } refreshedSourceCount++; } - if (!(await contentCollectionExists(db, collectionSlug, options.collectionId))) { - if (options.collectionId) { - return generationConflictResult({ refreshedSourceCount, deletedSourceCount: 0 }); - } + if (!options.collectionId && !(await contentCollectionExists(db, collectionSlug))) { const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); return { ...ZERO_RESULT, deletedSourceCount }; } @@ -422,6 +637,10 @@ async function refreshContentMediaUsageAttempt( }; } +export function contentRefreshKey(collectionId: string, contentId: string): string { + return `${collectionId}\u0000${contentId}`; +} + function contentSourceKeys( collectionSlug: string, contentId: string, diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts index 6757a3375c..5f6b6e7196 100644 --- a/packages/core/src/media/usage/content-snapshots.ts +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -1,5 +1,6 @@ import { sql, type Kysely } from "kysely"; +import { jsonTextValues } from "../../database/json-recordset.js"; import type { MediaUsageOccurrenceInput, MediaUsageSourceInput, @@ -63,6 +64,12 @@ export interface LoadContentMediaUsageSnapshotsOptions { identityVersion?: number; } +export interface LoadContentMediaUsageSnapshotsBatchControl { + shouldContinue?: () => boolean; + maxOccurrenceCount?: number; + maxProjectionBytes?: number; +} + export async function loadContentMediaUsageSnapshots( db: Kysely, collectionSlug: string, @@ -70,20 +77,84 @@ export async function loadContentMediaUsageSnapshots( fieldDiscovery?: ContentMediaUsageFieldDiscovery, options: LoadContentMediaUsageSnapshotsOptions = {}, ): Promise { + const results = await loadContentMediaUsageSnapshotsBatch( + db, + collectionSlug, + [contentId], + fieldDiscovery, + options, + ); + return results.get(contentId) ?? { success: false, error: "CONTENT_NOT_FOUND" }; +} + +export async function loadContentMediaUsageSnapshotsBatch( + db: Kysely, + collectionSlug: string, + contentIds: readonly string[], + fieldDiscovery?: ContentMediaUsageFieldDiscovery, + options: LoadContentMediaUsageSnapshotsOptions = {}, + control: LoadContentMediaUsageSnapshotsBatchControl = {}, +): Promise> { validateIdentifier(collectionSlug, "collection slug"); if (options.identityVersion !== undefined && !options.collectionId) { throw new Error("Canonical media usage snapshots require a collection identity"); } const discovery = fieldDiscovery ?? (await loadContentMediaUsageFields(db, collectionSlug)); - const row = await loadContentRow( + const rows = await loadContentRows( db, collectionSlug, - contentId, + contentIds, [...discovery.extractionFields.map((field) => field.slug), ...discovery.displayFieldSlugs], options.collectionId, ); + const revisionIds = Array.from(rows.values(), (row) => + readNullableString(row.draft_revision_id), + ).filter((revisionId): revisionId is string => revisionId !== null); + const revisions = await loadRevisionRows(db, revisionIds); + const results = new Map(); + let occurrenceCount = 0; + let projectionBytes = 0; + for (const contentId of new Set(contentIds)) { + if (control.shouldContinue && !control.shouldContinue()) break; + const row = rows.get(contentId); + const result: LoadContentMediaUsageSnapshotsResult = row + ? await buildContentMediaUsageSnapshots(row, collectionSlug, discovery, options, revisions) + : { success: false, error: "CONTENT_NOT_FOUND" }; + const snapshots = result.success ? result.snapshots : (result.snapshots ?? []); + const itemOccurrenceCount = snapshots.reduce( + (total, snapshot) => total + snapshot.occurrences.length, + 0, + ); + const itemProjectionBytes = snapshots.reduce( + (total, snapshot) => total + snapshot.projectionByteLength, + 0, + ); + if ( + results.size > 0 && + ((control.maxOccurrenceCount !== undefined && + occurrenceCount + itemOccurrenceCount > control.maxOccurrenceCount) || + (control.maxProjectionBytes !== undefined && + projectionBytes + itemProjectionBytes > control.maxProjectionBytes)) + ) { + break; + } + results.set(contentId, result); + occurrenceCount += itemOccurrenceCount; + projectionBytes += itemProjectionBytes; + } + return results; +} - if (!row) return { success: false, error: "CONTENT_NOT_FOUND" }; +async function buildContentMediaUsageSnapshots( + row: Record, + collectionSlug: string, + discovery: ContentMediaUsageFieldDiscovery, + options: LoadContentMediaUsageSnapshotsOptions, + revisions: ReadonlyMap< + string, + { success: true; revision: RevisionSnapshotRow } | { success: false } + >, +): Promise { const collectionId = readString(row[CONTENT_COLLECTION_ID_RESULT]); if (!collectionId) { throw new Error("Media usage snapshot query did not return a collection identity"); @@ -135,7 +206,7 @@ export async function loadContentMediaUsageSnapshots( sourceVariant: "draft_overlay", revisionId: draftRevisionId, }); - const revisionResult = await loadRevisionRow(db, draftRevisionId); + const revisionResult = revisions.get(draftRevisionId); if (!revisionResult) { return { success: false, @@ -212,52 +283,78 @@ interface RevisionSnapshotRow { data: Record; } -async function loadContentRow( +async function loadContentRows( db: Kysely, collectionSlug: string, - contentId: string, + contentIds: readonly string[], fieldSlugs: readonly string[], expectedCollectionId?: string, -): Promise | null> { +): Promise>> { const tableName = getContentTableName(collectionSlug); const columns = uniqueColumns([...CONTENT_SYSTEM_COLUMNS, ...fieldSlugs]); const columnRefs = columns.map((column) => sql.ref(`content.${column}`)); + const rows = new Map>(); + const uniqueContentIds = [...new Set(contentIds)]; + if (uniqueContentIds.length === 0) return rows; + const contentIdInput = jsonTextValues(db, uniqueContentIds); const result = await sql>` + WITH requested AS (${contentIdInput}) SELECT ${sql.join(columnRefs, sql`, `)}, collection.id AS __emdash_media_usage_collection_id FROM ${sql.ref(tableName)} AS content + INNER JOIN requested ON requested.value = content.id INNER JOIN _emdash_collections AS collection ON collection.slug = ${collectionSlug} ${expectedCollectionId ? sql`AND collection.id = ${expectedCollectionId}` : sql``} - WHERE content.id = ${contentId} - LIMIT 1 `.execute(db); - - return result.rows[0] ?? null; + for (const row of result.rows) { + const contentId = readString(row.id); + if (contentId) rows.set(contentId, row); + } + return rows; } -async function loadRevisionRow( +async function loadRevisionRows( db: Kysely, - revisionId: string, -): Promise<{ success: true; revision: RevisionSnapshotRow } | { success: false } | null> { - const row = await db - .selectFrom("revisions") - .select(["id", "collection", "entry_id", "data"]) - .where("id", "=", revisionId) - .executeTakeFirst(); - if (!row) return null; - const data = parseRevisionData(row.data); - if (!data) return { success: false }; - return { - success: true, - revision: { - id: row.id, - collection: row.collection, - entryId: row.entry_id, - data, - }, - }; + revisionIds: readonly string[], +): Promise> { + const revisions = new Map< + string, + { success: true; revision: RevisionSnapshotRow } | { success: false } + >(); + const uniqueRevisionIds = [...new Set(revisionIds)]; + if (uniqueRevisionIds.length === 0) return revisions; + const revisionIdInput = jsonTextValues(db, uniqueRevisionIds); + const rows = await sql<{ + id: string; + collection: string; + entry_id: string; + data: unknown; + }>` + WITH requested AS (${revisionIdInput}) + SELECT revision.id, revision.collection, revision.entry_id, revision.data + FROM revisions AS revision + INNER JOIN requested ON requested.value = revision.id + `.execute(db); + for (const row of rows.rows) { + const data = parseRevisionData(row.data); + revisions.set( + row.id, + data + ? { + success: true, + revision: { + id: row.id, + collection: row.collection, + entryId: row.entry_id, + data, + }, + } + : { success: false }, + ); + } + return revisions; } function buildContentSource(input: { diff --git a/packages/core/src/media/usage/maintenance-engine.ts b/packages/core/src/media/usage/maintenance-engine.ts new file mode 100644 index 0000000000..ccc31d6bb4 --- /dev/null +++ b/packages/core/src/media/usage/maintenance-engine.ts @@ -0,0 +1,166 @@ +import type { Kysely } from "kysely"; + +import type { Database } from "../../database/types.js"; +import { getRequestContext } from "../../request-context.js"; +import { + continueMediaUsageActivation, + MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION, + MediaUsageActivationVersionMismatchError, +} from "./activation.js"; +import { processDueMediaUsageCollectionDeletions } from "./collection-deletion-processor.js"; +import { processDueMediaUsageReconciliationDetailed } from "./reconciliation-processor.js"; +import { MediaUsageReconciliationRepository } from "./reconciliation.js"; +import { processDueMediaUsageWork } from "./work-processor.js"; + +type MediaUsageMaintenanceTaskClass = "entry_work" | "collection_deletion" | "reconciliation"; + +export type MediaUsageMaintenanceContinuation = + | { kind: "none" } + | { kind: "immediate" } + | { kind: "delayed"; delaySeconds: 30 }; + +export interface MediaUsageMaintenanceStepResult { + state: "inactive" | "idle" | "blocked" | "progress"; + continuation: MediaUsageMaintenanceContinuation; +} + +export const MEDIA_USAGE_MAINTENANCE_LIMITS = Object.freeze({ + eventQueryCeiling: 900, + maxStepQueries: 150, +}); + +const TASK_CLASSES: readonly MediaUsageMaintenanceTaskClass[] = [ + "collection_deletion", + "entry_work", + "reconciliation", +]; + +export async function runMediaUsageMaintenanceStep( + db: Kysely, +): Promise { + const activation = await db + .selectFrom("_emdash_media_usage_activation") + .select(["state", "runtime_generation"]) + .where("task_key", "=", "incremental_capture") + .executeTakeFirst(); + if ( + activation?.state !== "active" || + activation.runtime_generation !== MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION + ) { + return runActivationStep(db); + } + let blocked = false; + let blockedReconciliation = false; + let madeProgress = false; + + for (const taskClass of TASK_CLASSES) { + const metrics = getRequestContext()?.metrics; + if (metrics && !canStartMediaUsageMaintenanceStep(metrics)) { + return madeProgress + ? { + state: "progress", + continuation: { kind: "immediate" }, + } + : { + state: "blocked", + continuation: { kind: "immediate" }, + }; + } + const outcome = await runTaskClass(db, taskClass); + if (outcome === "inactive") return inactiveResult(); + if (outcome === "progress") madeProgress = true; + if (outcome === "blocked") { + blocked = true; + if (taskClass === "reconciliation") blockedReconciliation = true; + } + } + if (madeProgress) { + return { + state: "progress", + continuation: { kind: "immediate" }, + }; + } + + if ( + blockedReconciliation && + (await new MediaUsageReconciliationRepository(db).wakeDrainedBarrierCandidate()) + ) { + return { + state: "progress", + continuation: { kind: "immediate" }, + }; + } + + if (blocked) { + return { + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, + }; + } + + return { + state: "idle", + continuation: { kind: "none" }, + }; +} + +async function runActivationStep(db: Kysely): Promise { + try { + const result = await continueMediaUsageActivation(db); + if (result.outcome === "activating" || result.outcome === "active") { + return { + state: "progress", + continuation: { kind: "immediate" }, + }; + } + if (result.outcome === "lease_active" || result.outcome === "conflict") { + return { + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, + }; + } + return inactiveResult(); + } catch (error) { + if (error instanceof MediaUsageActivationVersionMismatchError) return inactiveResult(); + throw error; + } +} + +async function runTaskClass( + db: Kysely, + taskClass: MediaUsageMaintenanceTaskClass, +): Promise<"inactive" | "idle" | "blocked" | "progress"> { + if (taskClass === "entry_work") { + const result = await processDueMediaUsageWork(db, { + activationKnownActive: true, + }); + if (result.claimedCount > 0) return "progress"; + return result.candidateCount > 0 ? "blocked" : "idle"; + } + if (taskClass === "collection_deletion") { + const result = await processDueMediaUsageCollectionDeletions(db); + if (result.claimedCount > 0) return "progress"; + return result.candidateCount > 0 ? "blocked" : "idle"; + } + + const result = await processDueMediaUsageReconciliationDetailed(db, { + activationKnownActive: true, + }); + if (result.outcome === "inactive") return "inactive"; + if (result.consumedUnit) return "progress"; + return result.outcome === "claim_lost" || result.hasDeferredCandidate ? "blocked" : "idle"; +} + +function inactiveResult(): MediaUsageMaintenanceStepResult { + return { + state: "inactive", + continuation: { kind: "none" }, + }; +} + +function canStartMediaUsageMaintenanceStep(metrics: { dbCount: number }): boolean { + return ( + metrics.dbCount + MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries <= + MEDIA_USAGE_MAINTENANCE_LIMITS.eventQueryCeiling + ); +} diff --git a/packages/core/src/media/usage/reconciliation-processor.ts b/packages/core/src/media/usage/reconciliation-processor.ts index 3bf944f437..0f54c5d7d1 100644 --- a/packages/core/src/media/usage/reconciliation-processor.ts +++ b/packages/core/src/media/usage/reconciliation-processor.ts @@ -15,13 +15,13 @@ import { CONTENT_SOURCE_SCHEMA_VERSION } from "./types.js"; export const MEDIA_USAGE_RECONCILIATION_LIMITS = Object.freeze({ candidatesPerTick: 4, - pageSize: 50, + scanPageSize: 1_000, + sourcePageSize: 1_000, leaseDurationSeconds: 60, maxAttempts: 5, retryBaseSeconds: 30, retryMaxSeconds: 15 * 60, retryJitterRatio: 0.25, - maxQueriesPerTick: 20, }); export type MediaUsageReconciliationOutcome = @@ -34,6 +34,16 @@ export type MediaUsageReconciliationOutcome = | "retry" | "failed"; +export interface MediaUsageReconciliationDetailedResult { + outcome: MediaUsageReconciliationOutcome; + consumedUnit: boolean; + hasDeferredCandidate: boolean; +} + +export interface ProcessDueMediaUsageReconciliationOptions { + activationKnownActive?: boolean; +} + export type MediaUsageReconciliationScanOutcome = | "advanced" | "exhausted" @@ -43,24 +53,39 @@ export type MediaUsageReconciliationScanOutcome = export async function processDueMediaUsageReconciliation( db: Kysely, ): Promise { - const activation = await db - .selectFrom("_emdash_media_usage_activation") - .select("state") - .where("task_key", "=", "incremental_capture") - .executeTakeFirst(); - if (activation?.state !== "active") return "inactive"; + return (await processDueMediaUsageReconciliationDetailed(db)).outcome; +} + +export async function processDueMediaUsageReconciliationDetailed( + db: Kysely, + options: ProcessDueMediaUsageReconciliationOptions = {}, +): Promise { + if (!options.activationKnownActive) { + const activation = await db + .selectFrom("_emdash_media_usage_activation") + .select("state") + .where("task_key", "=", "incremental_capture") + .executeTakeFirst(); + if (activation?.state !== "active") { + return { outcome: "inactive", consumedUnit: false, hasDeferredCandidate: false }; + } + } const reconciliation = new MediaUsageReconciliationRepository(db); - if (await reconciliation.deleteOneObsolete()) return "completed"; + if (await reconciliation.deleteOneObsolete()) { + return { outcome: "completed", consumedUnit: true, hasDeferredCandidate: false }; + } const [failed] = await reconciliation.findFailed(1); if (failed) { if (await reconciliation.finishFailedCoverage(failed.collectionId, failed.runToken)) { - return "failed"; + return { outcome: "failed", consumedUnit: true, hasDeferredCandidate: false }; + } + if (await reconciliation.resetFailedForNewEpoch(failed)) { + return { outcome: "advanced", consumedUnit: true, hasDeferredCandidate: false }; } - if (await reconciliation.resetFailedForNewEpoch(failed)) return "advanced"; } - await reconciliation.seedNextCandidate(); + const seeded = await reconciliation.seedNextCandidate(); const candidates = await reconciliation.findDue( MEDIA_USAGE_RECONCILIATION_LIMITS.candidatesPerTick, ); @@ -73,10 +98,21 @@ export async function processDueMediaUsageReconciliation( }); if (claim) break; } - if (!claim) return candidates.length === 0 ? "not_due" : "claim_lost"; + if (!claim) { + return { + outcome: candidates.length === 0 ? "not_due" : "claim_lost", + consumedUnit: seeded, + hasDeferredCandidate: + candidates.length === 0 && (await reconciliation.hasDeferredCandidate()), + }; + } try { - return await processClaimedReconciliation(db, claim); + return { + outcome: await processClaimedReconciliation(db, claim), + consumedUnit: true, + hasDeferredCandidate: false, + }; } catch (error) { const terminal = claim.attemptCount + 1 >= MEDIA_USAGE_RECONCILIATION_LIMITS.maxAttempts; const recorded = await reconciliation.recordFailure({ @@ -87,10 +123,16 @@ export async function processDueMediaUsageReconciliation( retryDelaySeconds: retryDelaySeconds(claim.attemptCount), terminal, }); - if (!recorded) return "claim_lost"; + if (!recorded) { + return { outcome: "claim_lost", consumedUnit: true, hasDeferredCandidate: false }; + } if (terminal) await reconciliation.finishFailedCoverage(claim.collectionId, claim.runToken); console.error("[media-usage:reconciliation] Processing failed:", error); - return terminal ? "failed" : "retry"; + return { + outcome: terminal ? "failed" : "retry", + consumedUnit: true, + hasDeferredCandidate: false, + }; } } @@ -98,9 +140,11 @@ export async function processClaimedMediaUsageReconciliationScan( db: Kysely, claim: MediaUsageReconciliationClaim, options: { releaseOnExhausted?: boolean } = {}, + knownCurrent?: MediaUsageReconciliationRecord, ): Promise { const reconciliation = new MediaUsageReconciliationRepository(db); - let current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); + let current = + knownCurrent ?? (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)); if (!current || current.leaseToken !== claim.leaseToken || current.phase !== "scan") { return "deferred"; } @@ -127,8 +171,15 @@ export async function processClaimedMediaUsageReconciliationScan( ) { return "deferred"; } - current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); - if (!current) return "deferred"; + current = { + ...current, + targetEpoch, + fieldFingerprint, + scanCursor: null, + scanUpperId, + sourceCursor: null, + sourceUpperKey: null, + }; } else { fields = await loadContentMediaUsageFields(db, claim.collectionSlug, claim.collectionId); fieldFingerprint = await buildContentMediaUsageFieldFingerprint(fields); @@ -137,7 +188,10 @@ export async function processClaimedMediaUsageReconciliationScan( if (current.fieldFingerprint !== fieldFingerprint || current.targetEpoch === null) { return "restart_required"; } - const contentIds = await reconciliation.findScanPage(current, 50); + const contentIds = await reconciliation.findScanPage( + current, + MEDIA_USAGE_RECONCILIATION_LIMITS.scanPageSize, + ); if (contentIds.length === 0) { if (options.releaseOnExhausted ?? true) { await reconciliation.release({ ...claim, delaySeconds: 30 }); @@ -175,23 +229,39 @@ async function processClaimedReconciliation( claim: MediaUsageReconciliationClaim, ): Promise { const reconciliation = new MediaUsageReconciliationRepository(db); - let current = await reconciliation.findByIdentity(claim.collectionId, claim.runToken); - if (!current || current.leaseToken !== claim.leaseToken) return "claim_lost"; + let current: MediaUsageReconciliationRecord = claim; if (current.targetEpoch !== null && !(await reconciliation.ownsRun(claim, current.targetEpoch))) { return restartReconciliation(db, reconciliation, claim, current); } + if (current.targetEpoch !== null) { + await new MediaUsageWorkRepository(db).deleteObsoleteReconciliationWork({ + collectionId: claim.collectionId, + collectionSlug: claim.collectionSlug, + runToken: claim.runToken, + leaseToken: claim.leaseToken, + targetEpoch: current.targetEpoch, + }); + } if (current.phase === "scan") { - const outcome = await processClaimedMediaUsageReconciliationScan(db, claim, { - releaseOnExhausted: false, - }); + const outcome = await processClaimedMediaUsageReconciliationScan( + db, + claim, + { + releaseOnExhausted: false, + }, + current, + ); if (outcome === "restart_required") { current = (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)) ?? current; return restartReconciliation(db, reconciliation, claim, current); } if (outcome !== "exhausted") return outcome; - current = (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)) ?? current; + if (current.targetEpoch === null || current.fieldFingerprint === null) { + current = + (await reconciliation.findByIdentity(claim.collectionId, claim.runToken)) ?? current; + } const barrier = await reconciliation.findWorkBarrier(claim.collectionId); if (barrier.state === "failed") { return failReconciliation(reconciliation, claim, barrier.errorCode, true); @@ -234,9 +304,10 @@ async function processSourcePhase( const page = await reconciliation.findSourcePage( current, - MEDIA_USAGE_RECONCILIATION_LIMITS.pageSize, + MEDIA_USAGE_RECONCILIATION_LIMITS.sourcePageSize, ); if (page.length > 0) { + let enqueuedMissingContent = false; const malformed = page.some( (source) => !source.contentId || @@ -265,6 +336,7 @@ async function processSourcePhase( phase: "sources", contentIds: enqueueIds, }); + enqueuedMissingContent = true; } if ( !(await reconciliation.checkpointSources({ @@ -276,8 +348,13 @@ async function processSourcePhase( ) { return "claim_lost"; } - if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "claim_lost"; - return "advanced"; + if ( + enqueuedMissingContent || + page.length === MEDIA_USAGE_RECONCILIATION_LIMITS.sourcePageSize + ) { + if (!(await reconciliation.release({ ...claim, delaySeconds: 0 }))) return "claim_lost"; + return "advanced"; + } } const barrier = await reconciliation.findWorkBarrier(claim.collectionId); diff --git a/packages/core/src/media/usage/reconciliation.ts b/packages/core/src/media/usage/reconciliation.ts index 2fc7bcddcc..61b1d3131f 100644 --- a/packages/core/src/media/usage/reconciliation.ts +++ b/packages/core/src/media/usage/reconciliation.ts @@ -2,6 +2,7 @@ import { sql, type Kysely, type RawBuilder, type Selectable } from "kysely"; import { ulid } from "ulidx"; import { isPostgres } from "../../database/dialect-helpers.js"; +import { jsonTextValues } from "../../database/json-recordset.js"; import type { Database, MediaUsageReconciliationTable } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; @@ -147,8 +148,8 @@ export class MediaUsageReconciliationRepository { reconciliation: MediaUsageReconciliationRecord, limit: number, ): Promise { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) { - throw new Error("Reconciliation scan page limit must be from 1 to 50"); + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) { + throw new Error("Reconciliation scan page limit must be from 1 to 1000"); } if (!reconciliation.leaseToken || reconciliation.targetEpoch === null) return []; const tableName = contentTableName(reconciliation.collectionSlug); @@ -366,8 +367,8 @@ export class MediaUsageReconciliationRepository { reconciliation: MediaUsageReconciliationRecord, limit: number, ): Promise { - if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) { - throw new Error("Reconciliation source page limit must be from 1 to 50"); + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) { + throw new Error("Reconciliation source page limit must be from 1 to 1000"); } if (!reconciliation.leaseToken || reconciliation.targetEpoch === null) return []; let query = this.db @@ -400,12 +401,16 @@ export class MediaUsageReconciliationRepository { ): Promise { const unique = [...new Set(contentIds)]; if (unique.length === 0) return []; - if (unique.length > 50 || unique.some((contentId) => !contentId)) { + if (unique.length > 1_000 || unique.some((contentId) => !contentId)) { throw new Error("Reconciliation source page has invalid content identity"); } const tableName = contentTableName(collectionSlug); + const requested = jsonTextValues(this.db, unique); const existing = await sql<{ id: string }>` - SELECT id FROM ${sql.ref(tableName)} WHERE id IN (${sql.join(unique)}) + WITH requested AS (${requested}) + SELECT content.id + FROM ${sql.ref(tableName)} AS content + INNER JOIN requested ON requested.value = content.id `.execute(this.db); const present = new Set(existing.rows.map((row) => row.id)); return unique.filter((contentId) => !present.has(contentId)); @@ -482,6 +487,12 @@ export class MediaUsageReconciliationRepository { .where("status.collection_id", "=", collectionId) .where("status.status", "=", "running") .where("status.cursor", "=", runToken) + .where(sql`status.change_epoch = ( + SELECT reconciliation.target_epoch + FROM _emdash_media_usage_reconciliations AS reconciliation + WHERE reconciliation.collection_id = ${collectionId} + AND reconciliation.run_token = ${runToken} + )`) .where((eb) => eb.exists( eb @@ -751,6 +762,70 @@ export class MediaUsageReconciliationRepository { return result.rows.map(rowToRecord); } + async hasDeferredCandidate(): Promise { + const row = await this.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("state", "in", ["pending", "retry", "leased"]) + .limit(1) + .executeTakeFirst(); + return row !== undefined; + } + + async wakeDrainedBarrierCandidate(): Promise { + const now = timestampOffset(this.db, 0); + const candidateIsDue = timestampIsDue(this.db, "reconciliation.next_attempt_at"); + const targetIsDue = timestampIsDue( + this.db, + "_emdash_media_usage_reconciliations.next_attempt_at", + ); + const result = await sql<{ collection_id: string }>` + UPDATE _emdash_media_usage_reconciliations + SET next_attempt_at = ${now}, updated_at = ${now} + WHERE collection_id = ( + SELECT reconciliation.collection_id + FROM _emdash_media_usage_reconciliations AS reconciliation + INNER JOIN _emdash_media_usage_index_status AS status + ON status.collection_id = reconciliation.collection_id + AND status.scope_key = reconciliation.collection_slug + WHERE reconciliation.state = 'pending' + AND reconciliation.target_epoch IS NOT NULL + AND NOT ${candidateIsDue} + AND status.adapter_id = ${CONTENT_ADAPTER_ID} + AND status.scope_type = ${COLLECTION_SCOPE} + AND status.capture_state = 'active' + AND status.reconciliation_required = 1 + AND status.status = 'running' + AND status.cursor = reconciliation.run_token + AND status.change_epoch = reconciliation.target_epoch + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_work AS work + WHERE work.collection_id = reconciliation.collection_id + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_collection_deletions AS deletion + WHERE deletion.collection_id = reconciliation.collection_id + ) + ORDER BY reconciliation.next_attempt_at, reconciliation.updated_at, + reconciliation.collection_id + LIMIT 1 + ) + AND state = 'pending' + AND NOT ${targetIsDue} + AND EXISTS ( + SELECT 1 FROM _emdash_media_usage_activation AS activation + WHERE activation.task_key = ${ACTIVATION_KEY} + AND activation.state = 'active' + ) + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage_work AS work + WHERE work.collection_id = _emdash_media_usage_reconciliations.collection_id + ) + RETURNING collection_id + `.execute(this.db); + return result.rows.length === 1; + } + async findFailed(limit: number): Promise { assertLimit(limit); const rows = await this.db @@ -1005,7 +1080,15 @@ export class MediaUsageReconciliationRepository { .where("status.scope_type", "=", COLLECTION_SCOPE) .where("status.capture_state", "=", "active") .where("status.reconciliation_required", "=", 1) - .where("status.cursor", "is", null) + .where((status) => + status.or([ + status("status.cursor", "is", null), + status.and([ + status("status.status", "=", "running"), + status("status.cursor", "=", runToken), + ]), + ]), + ) .where("status.change_epoch", ">", targetEpoch), ), ) diff --git a/packages/core/src/media/usage/work-processor.ts b/packages/core/src/media/usage/work-processor.ts index 6ef62480c3..061161d771 100644 --- a/packages/core/src/media/usage/work-processor.ts +++ b/packages/core/src/media/usage/work-processor.ts @@ -1,26 +1,28 @@ import type { Kysely } from "kysely"; +import { isPostgres } from "../../database/dialect-helpers.js"; import { MediaUsageWorkRepository, type MediaUsageWorkRecord, } from "../../database/repositories/media-usage-work.js"; import { MediaUsageRepository } from "../../database/repositories/media-usage.js"; +import { withTransaction } from "../../database/transaction.js"; import type { Database } from "../../database/types.js"; +import { getRequestContext } from "../../request-context.js"; import { - refreshContentMediaUsageForWork, + contentRefreshKey, + refreshContentMediaUsageForWorkBatch, + type ContentMediaUsageRefreshResult, type ContentMediaUsageRefreshErrorCode, } from "./content-refresh.js"; export const MEDIA_USAGE_WORK_PROCESSING_LIMITS = Object.freeze({ - candidatesPerTick: 4, - jobsPerTick: 1, - maxTickDurationMs: 5_000, - leaseDurationSeconds: 60, + candidatesPerTick: 1_000, + leaseDurationSeconds: 20 * 60, maxAttempts: 5, retryBaseSeconds: 30, retryMaxSeconds: 15 * 60, retryJitterRatio: 0.25, - ordinaryStatementsPerJob: 20, }); export type MediaUsageWorkProcessingOutcome = @@ -47,7 +49,10 @@ export interface MediaUsageWorkTickResult { supersededCount: number; obsoleteCount: number; durationMs: number; - admissionClosed: boolean; +} + +export interface ProcessDueMediaUsageWorkOptions { + activationKnownActive?: boolean; } export async function processMediaUsageWorkAfterWrite( @@ -59,14 +64,45 @@ export async function processMediaUsageWorkAfterWrite( return { outcome: "inactive", claimed: false }; } + await new MediaUsageRepository(db).recoverIncrementalFinalizations(); const repo = new MediaUsageWorkRepository(db); const work = await repo.findWorkForContent(collectionSlug, contentId); if (!work) return { outcome: "not_due", claimed: false }; - return processCandidate(db, repo, work); + const claimed = await repo.claimWork({ + collectionId: work.collectionId, + contentId: work.contentId, + workVersion: work.workVersion, + leaseDurationSeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.leaseDurationSeconds, + }); + if (!claimed) return { outcome: "claim_lost", claimed: false }; + try { + const processed = await runClaimedBatch(db, [claimed]); + return processed.get(workResultKey(claimed)) ?? { outcome: "claim_lost", claimed: false }; + } catch (error) { + const transitioned = await repo.retryClaimedWorkBatch({ + work: [workLease(claimed)], + errorCode: "MEDIA_USAGE_PROCESSING_FAILED", + retryDelaySeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.retryBaseSeconds, + maxAttempts: MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts, + }); + const outcome = transitioned.get(workIdentityKey(claimed)) ?? "superseded"; + if (outcome === "failed") { + await new MediaUsageRepository(db).recordIncrementalFailure({ + collectionId: claimed.collectionId, + collectionSlug: claimed.collectionSlug, + contentId: claimed.contentId, + workVersion: claimed.workVersion, + errorCode: "MEDIA_USAGE_PROCESSING_FAILED", + }); + } + console.error("[media-usage:work] Immediate processing failed:", error); + return { outcome, claimed: true }; + } } export async function processDueMediaUsageWork( db: Kysely, + options: ProcessDueMediaUsageWorkOptions = {}, ): Promise { const startedAt = Date.now(); const result: MediaUsageWorkTickResult = { @@ -78,28 +114,28 @@ export async function processDueMediaUsageWork( supersededCount: 0, obsoleteCount: 0, durationMs: 0, - admissionClosed: false, }; - if (!(await isIncrementalCaptureActive(db))) { + if (!options.activationKnownActive && !(await isIncrementalCaptureActive(db))) { result.durationMs = Date.now() - startedAt; return result; } + await new MediaUsageRepository(db).recoverIncrementalFinalizations(); const repo = new MediaUsageWorkRepository(db); - const candidates = await repo.findDueWork(MEDIA_USAGE_WORK_PROCESSING_LIMITS.candidatesPerTick); - result.candidateCount = candidates.length; + const candidates = await repo.claimDueWorkBatch({ + limit: MEDIA_USAGE_WORK_PROCESSING_LIMITS.candidatesPerTick, + leaseDurationSeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.leaseDurationSeconds, + }); + result.candidateCount = + candidates.length > 0 || !(await repo.hasNonterminalWork()) ? candidates.length : 1; + const processedBatch = await processClaimedCollections(db, candidates); for (const candidate of candidates) { - if ( - result.claimedCount >= MEDIA_USAGE_WORK_PROCESSING_LIMITS.jobsPerTick || - Date.now() - startedAt >= MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxTickDurationMs - ) { - result.admissionClosed = true; - break; - } - - const processed = await processCandidate(db, repo, candidate); + const processed = processedBatch.get(workResultKey(candidate)) ?? { + outcome: "claim_lost" as const, + claimed: false, + }; if (processed.claimed) result.claimedCount++; if (processed.outcome === "completed") result.completedCount++; if (processed.outcome === "retry") result.retryCount++; @@ -112,92 +148,311 @@ export async function processDueMediaUsageWork( return result; } -async function processCandidate( +async function processClaimedCollections( db: Kysely, - repo: MediaUsageWorkRepository, - candidate: MediaUsageWorkRecord, -): Promise { - const claimed = await repo.claimWork({ - collectionId: candidate.collectionId, - contentId: candidate.contentId, - workVersion: candidate.workVersion, - leaseDurationSeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.leaseDurationSeconds, - }); - if (!claimed?.leaseToken) return { outcome: "claim_lost", claimed: false }; + candidates: readonly MediaUsageWorkRecord[], +): Promise> { + const processed = new Map(); + const byCollection = new Map(); + for (const candidate of candidates) { + const key = collectionResultKey(candidate); + const collection = byCollection.get(key) ?? []; + collection.push(candidate); + byCollection.set(key, collection); + } + for (const collection of byCollection.values()) { + try { + for (const [key, result] of await runClaimedBatch(db, collection)) { + processed.set(key, result); + } + } catch (error) { + for (const [key, result] of await retryFailedClaimedCollection(db, collection)) { + processed.set(key, result); + } + console.error("[media-usage:work] Collection processing failed:", error); + } + } + return processed; +} - const lease = { - collectionId: claimed.collectionId, - contentId: claimed.contentId, - workVersion: claimed.workVersion, - leaseToken: claimed.leaseToken, - }; - if (!(await collectionIdentityIsCurrent(db, claimed.collectionId, claimed.collectionSlug))) { - return { - outcome: (await repo.completeWork(lease)) ? "obsolete" : "superseded", +async function retryFailedClaimedCollection( + db: Kysely, + candidates: readonly MediaUsageWorkRecord[], +): Promise> { + const transitioned = await new MediaUsageWorkRepository(db).retryClaimedWorkBatch({ + work: candidates.map(workLease), + errorCode: "MEDIA_USAGE_PROCESSING_FAILED", + retryDelaySeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.retryBaseSeconds, + maxAttempts: MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts, + }); + const processed = new Map(); + const failedCollectionIds = new Set(); + for (const candidate of candidates) { + const state = transitioned.get(workIdentityKey(candidate)); + if (state === "failed") failedCollectionIds.add(candidate.collectionId); + processed.set(workResultKey(candidate), { + outcome: state ?? "superseded", claimed: true, - }; + }); } + await new MediaUsageRepository(db).recordIncrementalFailuresByCollection({ + collectionIds: [...failedCollectionIds], + errorCode: "MEDIA_USAGE_PROCESSING_FAILED", + }); + return processed; +} - const refresh = await refreshContentMediaUsageForWork( - db, - claimed.collectionId, - claimed.collectionSlug, - claimed.contentId, - ); - if (refresh.success) { - const completed = await repo.completeWork(lease); - if (completed) { - await new MediaUsageRepository(db).recordIncrementalSuccess({ - collectionId: claimed.collectionId, - collectionSlug: claimed.collectionSlug, - }); +async function runClaimedBatch( + db: Kysely, + candidates: readonly MediaUsageWorkRecord[], +): Promise> { + return isPostgres(db) + ? withTransaction(db, (trx) => + processClaimedBatch(trx, new MediaUsageWorkRepository(trx), candidates), + ) + : processClaimedBatch(db, new MediaUsageWorkRepository(db), candidates); +} + +async function processClaimedBatch( + db: Kysely, + repo: MediaUsageWorkRepository, + candidates: readonly MediaUsageWorkRecord[], +): Promise> { + const results = new Map(); + const locked = await repo.lockClaimedWorkBatch(candidates.map(workLease)); + const owned: MediaUsageWorkRecord[] = []; + for (const candidate of candidates) { + if (locked.has(workIdentityKey(candidate))) owned.push(candidate); + else { + results.set(workResultKey(candidate), { outcome: "superseded", claimed: true }); } - return { - outcome: completed ? "completed" : "superseded", - claimed: true, - }; + } + const currentCollections = await findCurrentCollectionIdentities(db, owned); + const current: MediaUsageWorkRecord[] = []; + const obsolete: MediaUsageWorkRecord[] = []; + for (const candidate of owned) { + if (currentCollections.has(collectionResultKey(candidate))) current.push(candidate); + else obsolete.push(candidate); } - if (!(await collectionIdentityIsCurrent(db, claimed.collectionId, claimed.collectionSlug))) { - return { - outcome: (await repo.completeWork(lease)) ? "obsolete" : "superseded", + const obsoleteCompleted = await repo.completeWorkBatch(obsolete.map(workLease)); + for (const candidate of obsolete) { + results.set(workResultKey(candidate), { + outcome: obsoleteCompleted.has(workIdentityKey(candidate)) ? "obsolete" : "superseded", claimed: true, - }; + }); } + if (current.length === 0) return results; - const errorCode = processingErrorCode(refresh.errorCode); - const terminal = - errorCode === "MEDIA_USAGE_RESOURCE_LIMIT" || - claimed.attemptCount + 1 >= MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts; - if (terminal) { - const failed = await repo.failWork({ ...lease, errorCode }); - if (failed) { - await new MediaUsageRepository(db).recordIncrementalFailure({ - collectionId: claimed.collectionId, - collectionSlug: claimed.collectionSlug, - contentId: claimed.contentId, - workVersion: claimed.workVersion, - errorCode, + const refreshes = await refreshContentMediaUsageForWorkBatch(db, current, { + shouldContinue: canContinueBulkWork, + }); + const successful: MediaUsageWorkRecord[] = []; + const unstarted: MediaUsageWorkRecord[] = []; + const failedRefreshes: Array<{ + candidate: MediaUsageWorkRecord; + refresh: ContentMediaUsageRefreshResult; + }> = []; + for (const candidate of current) { + const refresh = refreshes.get(contentRefreshKey(candidate.collectionId, candidate.contentId)); + if (refresh?.success) successful.push(candidate); + else if (!refresh) unstarted.push(candidate); + else failedRefreshes.push({ candidate, refresh }); + } + const terminalFailureCollections = new Map>(); + const failureGroups = new Map< + string, + { + errorCode: string; + retryDelaySeconds: number; + terminal: boolean; + candidates: MediaUsageWorkRecord[]; + } + >(); + for (const failure of failedRefreshes) { + const errorCode = processingErrorCode(failure.refresh.errorCode); + const terminal = + errorCode === "MEDIA_USAGE_RESOURCE_LIMIT" || + failure.candidate.attemptCount + 1 >= MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts; + const delay = retryDelaySeconds(failure.candidate.attemptCount); + const key = `${errorCode}\u0000${terminal ? "terminal" : String(delay)}`; + const group = failureGroups.get(key) ?? { + errorCode, + retryDelaySeconds: delay, + terminal, + candidates: [], + }; + group.candidates.push(failure.candidate); + failureGroups.set(key, group); + } + for (const group of failureGroups.values()) { + const transitioned = await repo.retryClaimedWorkBatch({ + work: group.candidates.map(workLease), + errorCode: group.errorCode, + retryDelaySeconds: group.retryDelaySeconds, + maxAttempts: group.terminal ? 1 : MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts, + }); + for (const candidate of group.candidates) { + const state = transitioned.get(workIdentityKey(candidate)); + if (state === "failed") { + const collections = terminalFailureCollections.get(group.errorCode) ?? new Set(); + collections.add(candidate.collectionId); + terminalFailureCollections.set(group.errorCode, collections); + } + results.set(workResultKey(candidate), { + outcome: state ?? "superseded", + claimed: true, }); } - return { - outcome: failed ? "failed" : "superseded", + } + for (const [errorCode, collectionIds] of terminalFailureCollections) { + await new MediaUsageRepository(db).recordIncrementalFailuresByCollection({ + collectionIds: [...collectionIds], + errorCode, + }); + } + const released = await repo.releaseClaimedWorkBatch(unstarted.map(workLease)); + for (const candidate of unstarted) { + results.set(workResultKey(candidate), { + outcome: released.has(workIdentityKey(candidate)) ? "not_due" : "superseded", claimed: true, - }; + }); } + const usage = new MediaUsageRepository(db); + const successfulByCollection = new Map(); + for (const candidate of successful) { + const key = collectionResultKey(candidate); + const collection = successfulByCollection.get(key) ?? []; + collection.push(candidate); + successfulByCollection.set(key, collection); + } + const readyToComplete: MediaUsageWorkRecord[] = []; + const deferredCompletion: MediaUsageWorkRecord[] = []; + const lostFinalization: MediaUsageWorkRecord[] = []; + let pendingFinalizationQueries = 2; + for (const collection of successfulByCollection.values()) { + const first = collection[0]; + if (!first) continue; + const metrics = getRequestContext()?.metrics; + if (metrics && metrics.dbCount + pendingFinalizationQueries + 4 > 900) { + deferredCompletion.push(...collection); + continue; + } + const finalization = await usage.prepareIncrementalFinalization({ + collectionId: first.collectionId, + collectionSlug: first.collectionSlug, + }); + if (finalization.outcome !== "lost") { + readyToComplete.push(...collection); + pendingFinalizationQueries += 2; + continue; + } + lostFinalization.push(...collection); + pendingFinalizationQueries += 2; + } + const lostTransitions = await repo.retryClaimedWorkBatch({ + work: lostFinalization.map(workLease), + errorCode: "MEDIA_USAGE_GENERATION_CONFLICT", + retryDelaySeconds: MEDIA_USAGE_WORK_PROCESSING_LIMITS.retryBaseSeconds, + maxAttempts: MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts, + }); + const lostFailedCollections = new Set(); + for (const candidate of lostFinalization) { + const state = lostTransitions.get(workIdentityKey(candidate)); + if (state === "failed") lostFailedCollections.add(candidate.collectionId); + results.set(workResultKey(candidate), { + outcome: state ?? "superseded", + claimed: true, + }); + } + await usage.recordIncrementalFailuresByCollection({ + collectionIds: [...lostFailedCollections], + errorCode: "MEDIA_USAGE_GENERATION_CONFLICT", + }); + const deferred = await repo.releaseClaimedWorkBatch(deferredCompletion.map(workLease)); + for (const candidate of deferredCompletion) { + results.set(workResultKey(candidate), { + outcome: deferred.has(workIdentityKey(candidate)) ? "not_due" : "superseded", + claimed: true, + }); + } + const completed = await repo.completeWorkBatch(readyToComplete.map(workLease)); + const completedCollections = new Map(); + for (const candidate of readyToComplete) { + const didComplete = completed.has(workIdentityKey(candidate)); + results.set(workResultKey(candidate), { + outcome: didComplete ? "completed" : "superseded", + claimed: true, + }); + if (didComplete) completedCollections.set(collectionResultKey(candidate), candidate); + } + for (const collection of completedCollections.values()) { + await usage.recordIncrementalSuccess({ + collectionId: collection.collectionId, + collectionSlug: collection.collectionSlug, + }); + } + return results; +} + +async function findCurrentCollectionIdentities( + db: Kysely, + candidates: readonly MediaUsageWorkRecord[], +): Promise> { + const identities = new Map( + candidates.map((candidate) => [collectionResultKey(candidate), candidate] as const), + ); + const current = new Set(); + for (const batch of chunkCollectionIds( + Array.from(identities.values(), (item) => item.collectionId), + )) { + const rows = await db + .selectFrom("_emdash_collections") + .select(["id", "slug"]) + .where("id", "in", batch) + .execute(); + for (const row of rows) current.add(`${row.id}\u0000${row.slug}`); + } + return current; +} + +function workLease(work: MediaUsageWorkRecord) { + if (!work.leaseToken) throw new Error("Claimed media usage work requires a lease token"); return { - outcome: (await repo.retryWork({ - ...lease, - errorCode, - retryDelaySeconds: retryDelaySeconds(claimed.attemptCount), - })) - ? "retry" - : "superseded", - claimed: true, + collectionId: work.collectionId, + contentId: work.contentId, + workVersion: work.workVersion, + leaseToken: work.leaseToken, }; } +function workIdentityKey(work: MediaUsageWorkRecord): string { + return `${work.collectionId}\u0000${work.contentId}\u0000${String(work.workVersion)}`; +} + +function workResultKey(work: MediaUsageWorkRecord): string { + return workIdentityKey(work); +} + +function collectionResultKey(work: Pick) { + return `${work.collectionId}\u0000${work.collectionSlug}`; +} + +function chunkCollectionIds(ids: readonly string[]): string[][] { + const unique = [...new Set(ids)]; + const batches: string[][] = []; + for (let index = 0; index < unique.length; index += 50) { + batches.push(unique.slice(index, index + 50)); + } + return batches; +} + +function canContinueBulkWork(): boolean { + const metrics = getRequestContext()?.metrics; + return !metrics || metrics.dbCount + 150 <= 900; +} + async function isIncrementalCaptureActive(db: Kysely): Promise { const row = await db .selectFrom("_emdash_media_usage_activation") @@ -207,20 +462,6 @@ async function isIncrementalCaptureActive(db: Kysely): Promise, - collectionId: string, - collectionSlug: string, -): Promise { - const row = await db - .selectFrom("_emdash_collections") - .select("id") - .where("id", "=", collectionId) - .where("slug", "=", collectionSlug) - .executeTakeFirst(); - return row !== undefined; -} - function retryDelaySeconds(attemptCount: number): number { const exponential = Math.min( MEDIA_USAGE_WORK_PROCESSING_LIMITS.retryMaxSeconds, diff --git a/packages/core/src/plugins/scheduler/node.ts b/packages/core/src/plugins/scheduler/node.ts index 32e63d9ded..c4486c4b3d 100644 --- a/packages/core/src/plugins/scheduler/node.ts +++ b/packages/core/src/plugins/scheduler/node.ts @@ -29,7 +29,6 @@ export class NodeCronScheduler implements CronScheduler { private timer: ReturnType | null = null; private running = false; private systemCleanup: SystemCleanupFn | null = null; - private mediaUsageMaintenance: SystemCleanupFn | null = null; constructor(private executor: CronExecutor) {} @@ -37,10 +36,6 @@ export class NodeCronScheduler implements CronScheduler { this.systemCleanup = fn; } - setMediaUsageMaintenance(fn: SystemCleanupFn): void { - this.mediaUsageMaintenance = fn; - } - start(): void { this.running = true; this.arm(); @@ -117,19 +112,12 @@ export class NodeCronScheduler implements CronScheduler { } void Promise.allSettled(tasks) - .then(async (results) => { + .then((results) => { for (const r of results) { if (r.status === "rejected") { console.error("[cron:node] Tick task failed:", r.reason); } } - if (this.mediaUsageMaintenance) { - try { - await this.mediaUsageMaintenance(); - } catch (error) { - console.error("[cron:node] Media Usage maintenance failed:", error); - } - } return undefined; }) .finally(() => { diff --git a/packages/core/src/plugins/scheduler/types.ts b/packages/core/src/plugins/scheduler/types.ts index f4ee58afbb..511f9b8fcc 100644 --- a/packages/core/src/plugins/scheduler/types.ts +++ b/packages/core/src/plugins/scheduler/types.ts @@ -18,8 +18,6 @@ export interface CronScheduler { reschedule(): void; /** Register a system cleanup function to run alongside each tick. */ setSystemCleanup(fn: SystemCleanupFn): void; - /** Register bounded Media Usage maintenance to run after the general tick settles. */ - setMediaUsageMaintenance?(fn: SystemCleanupFn): void; } /** diff --git a/packages/core/tests/integration/database/media-usage-activation.test.ts b/packages/core/tests/integration/database/media-usage-activation.test.ts index dd6d9e6f16..f149915b94 100644 --- a/packages/core/tests/integration/database/media-usage-activation.test.ts +++ b/packages/core/tests/integration/database/media-usage-activation.test.ts @@ -623,6 +623,41 @@ describeEachDialect("media usage production activation", (dialect) => { ); }); + it("rejects an excessive owned trigger set before changing trigger DDL", async () => { + if (dialect !== "sqlite") return; + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "trigger_limit", label: "Trigger limit" }); + for (let index = 0; index < 150; index++) { + const triggerName = `emdash_mu_extra_${String(index).padStart(3, "0")}`; + await sql` + CREATE TRIGGER ${sql.ref(triggerName)} + AFTER INSERT ON ${sql.ref("ec_trigger_limit")} + BEGIN + SELECT 1; + END + `.execute(ctx.db); + } + + await expect(activateMediaUsageCapture(ctx.db, { writersDrained: true })).rejects.toThrow( + /activation failed/i, + ); + + const triggers = await sql<{ name: string }>` + SELECT name FROM sqlite_master + WHERE type = 'trigger' + AND tbl_name = 'ec_trigger_limit' + AND substr(name, 1, 10) = 'emdash_mu_' + `.execute(ctx.db); + expect(triggers.rows).toHaveLength(150); + expect(await activationRow()).toEqual( + expect.objectContaining({ + state: "activating", + lease_token: null, + last_error_code: "MEDIA_USAGE_ACTIVATION_FAILED", + }), + ); + }); + it("refuses a runtime generation mismatch without changing activation state", async () => { await ctx.db .updateTable("_emdash_media_usage_activation") diff --git a/packages/core/tests/integration/database/media-usage-progress-plan.test.ts b/packages/core/tests/integration/database/media-usage-progress-plan.test.ts new file mode 100644 index 0000000000..89d7ed66d5 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-progress-plan.test.ts @@ -0,0 +1,94 @@ +import Database from "better-sqlite3"; +import { Kysely, SqliteDialect } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { runMigrations } from "../../../src/database/migrations/runner.js"; +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import type { Database as DatabaseSchema } from "../../../src/database/types.js"; + +interface CapturedQuery { + sql: string; + parameters: readonly unknown[]; +} + +let sqlite: Database.Database; +let db: Kysely; +let repo: MediaUsageRepository; +let captured: CapturedQuery[]; + +beforeEach(async () => { + captured = []; + sqlite = new Database(":memory:"); + db = new Kysely({ + dialect: new SqliteDialect({ database: sqlite }), + log(event) { + if (event.level === "query") { + captured.push({ sql: event.query.sql, parameters: event.query.parameters }); + } + }, + }); + await runMigrations(db); + repo = new MediaUsageRepository(db); +}); + +afterEach(async () => { + await db.destroy(); +}); + +it("keeps aggregate progress to one indexed metadata statement as collections grow", async () => { + for (let offset = 0; offset < 1_000; offset += 50) { + const collections = Array.from({ length: 50 }, (_, index) => { + const suffix = String(offset + index).padStart(4, "0"); + return { + id: `collection-${suffix}`, + slug: `collection_${suffix}`, + label: `Collection ${suffix}`, + has_seo: 0, + }; + }); + await db.insertInto("_emdash_collections").values(collections).execute(); + await db + .insertInto("_emdash_media_usage_index_status") + .values( + collections.map((collection) => ({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: collection.slug, + status: "complete", + schema_version: 1, + collection_id: collection.id, + reconciliation_required: 0, + capture_state: "active", + })), + ) + .execute(); + } + await db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + captured = []; + + await expect(repo.findCollectionProgress()).resolves.toEqual({ + status: "ready", + readyCollections: 1_000, + totalCollections: 1_000, + }); + + expect(captured).toHaveLength(1); + const query = captured[0]!; + expect(query.parameters.length).toBeLessThan(10); + expect(query.sql).not.toContain("ec_"); + expect(query.sql).not.toContain("_emdash_media_usage_sources"); + expect(query.sql).not.toMatch(/FROM ["`]?_emdash_media_usage["`]?\s/i); + const plan = sqlite + .prepare(`EXPLAIN QUERY PLAN ${query.sql}`) + .all(...query.parameters) + .map((row) => (row as { detail: string }).detail) + .join("\n"); + expect(plan).toContain("idx__emdash_media_usage_status_collection"); + expect(plan).toContain("idx__emdash_media_usage_work_operator"); + expect(plan).toMatch(/SEARCH reconciliation USING INDEX .*reconciliations/); + expect(plan).not.toContain("USE TEMP B-TREE"); +}); diff --git a/packages/core/tests/integration/database/media-usage-projection-admission-measurement.test.ts b/packages/core/tests/integration/database/media-usage-projection-admission-measurement.test.ts index 5b34c9724e..9413ff54f5 100644 --- a/packages/core/tests/integration/database/media-usage-projection-admission-measurement.test.ts +++ b/packages/core/tests/integration/database/media-usage-projection-admission-measurement.test.ts @@ -70,7 +70,7 @@ afterAll(async () => { await db.destroy(); }); -it("reports current projection costs at occurrence and byte boundaries", async () => { +it("reports representative projection costs below the approved boundary", async () => { const rows: MeasurementRow[] = []; for (const totalOccurrences of [0, 1, 3, 4, 6, 9, 12, 15, 18, 21, 24, 27, 30]) { for (const [liveOccurrences, draftOccurrences] of candidateSplits(totalOccurrences)) { @@ -83,8 +83,7 @@ it("reports current projection costs at occurrence and byte boundaries", async ( const result = await measure(() => processMediaUsageWorkAfterWrite(db, fixture.collectionSlug, contentId), ); - const expectedOutcome = totalOccurrences <= 12 ? "completed" : "failed"; - expect(result.value.outcome).toBe(expectedOutcome); + expect(result.value.outcome).toBe("completed"); rows.push({ path: "processor", totalOccurrences, @@ -102,8 +101,7 @@ it("reports current projection costs at occurrence and byte boundaries", async ( const result = await measure(() => processMediaUsageWorkAfterWrite(db, fixture.collectionSlug, contentId), ); - const expectedOutcome = payloadBytes < 512 * 1024 ? "completed" : "failed"; - expect(result.value.outcome).toBe(expectedOutcome); + expect(result.value.outcome).toBe("completed"); rows.push({ path: "processor-bytes", totalOccurrences: 0, @@ -127,7 +125,7 @@ it("reports current projection costs at occurrence and byte boundaries", async ( expect(created.value.success).toBe(true); if (!created.value.success) throw new Error(created.value.error.message); const createOutcome = await workOutcome(created.value.data.item.id); - expect(createOutcome).toBe(totalOccurrences <= 12 ? "completed" : "failed"); + expect(createOutcome).toBe("completed"); rows.push({ path: "runtime-create", totalOccurrences, @@ -154,9 +152,7 @@ it("reports current projection costs at occurrence and byte boundaries", async ( ); expect(updated.value.success).toBe(true); const updateOutcome = await workOutcome(initial.data.item.id); - expect(updateOutcome).toBe( - liveOccurrences <= 12 && draftOccurrences <= 12 ? "completed" : "failed", - ); + expect(updateOutcome).toBe("completed"); rows.push({ path: "runtime-update", totalOccurrences, diff --git a/packages/core/tests/integration/database/media-usage-projection-admission-runtime.test.ts b/packages/core/tests/integration/database/media-usage-projection-admission-runtime.test.ts index e26fa81e6d..13955996c9 100644 --- a/packages/core/tests/integration/database/media-usage-projection-admission-runtime.test.ts +++ b/packages/core/tests/integration/database/media-usage-projection-admission-runtime.test.ts @@ -12,8 +12,10 @@ import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import { refreshContentMediaUsage } from "../../../src/media/usage/content-refresh.js"; import { loadContentMediaUsageSnapshots } from "../../../src/media/usage/content-snapshots.js"; +import { MEDIA_USAGE_MAINTENANCE_LIMITS } from "../../../src/media/usage/maintenance-engine.js"; import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; import { processMediaUsageWorkAfterWrite } from "../../../src/media/usage/work-processor.js"; +import { createRequestMetrics, runWithContext } from "../../../src/request-context.js"; import { addMediaUsageMeasurementDraft, createMediaUsageAdmissionFixture, @@ -43,12 +45,12 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { }); it("terminally rejects an oversized replacement before publishing either variant", async () => { - await insertEntry("oversized-variants", 6); + await insertEntry("oversized-variants", 250); await addMediaUsageMeasurementDraft( ctx.db, fixture, "oversized-variants", - mediaUsageMeasurementData(7, "oversized-draft"), + mediaUsageMeasurementData(251, "oversized-draft"), ); const result = await processMediaUsageWorkAfterWrite( @@ -76,13 +78,13 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { }); it("admits the occurrence boundary and keeps an oversized exact projection as a no-op", async () => { - await insertEntry("boundary", 12); + await insertEntry("boundary", 500); expect( (await processMediaUsageWorkAfterWrite(ctx.db, fixture.collectionSlug, "boundary")).outcome, ).toBe("completed"); - expect(await currentOccurrenceCount("boundary", "columns")).toBe(12); + expect(await currentOccurrenceCount("boundary", "columns")).toBe(500); - await insertEntry("oversized-no-op", 13); + await insertEntry("oversized-no-op", 501); const snapshots = await loadSnapshots("oversized-no-op"); const snapshot = snapshots.find((candidate) => candidate.source.sourceVariant === "columns"); if (!snapshot) throw new Error("Missing columns snapshot"); @@ -98,7 +100,7 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { expect((await repo.findSource(snapshot.source.sourceKey))?.currentGeneration).toBe( stored.currentGeneration, ); - expect(await currentOccurrenceCount("oversized-no-op", "columns")).toBe(13); + expect(await currentOccurrenceCount("oversized-no-op", "columns")).toBe(501); }); it("rejects oversized source bytes without mutation", async () => { @@ -107,7 +109,7 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { fixture, "oversized-bytes", mediaUsageMeasurementData(0, "oversized-bytes"), - "é".repeat(300_000), + "é".repeat(1_050_000), ); expect( (await processMediaUsageWorkAfterWrite(ctx.db, fixture.collectionSlug, "oversized-bytes")) @@ -117,7 +119,7 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { }); it("rejects oversized absent-source cleanup without mutation", async () => { - await insertEntry("oversized-delete", 13); + await insertEntry("oversized-delete", 501); const snapshots = await loadSnapshots("oversized-delete"); const snapshot = snapshots.find((candidate) => candidate.source.sourceVariant === "columns"); if (!snapshot) throw new Error("Missing columns snapshot"); @@ -137,7 +139,7 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { expect((await repo.findSource(snapshot.source.sourceKey))?.currentGeneration).toBe( stored.currentGeneration, ); - expect(await currentOccurrenceCount("oversized-delete", "columns")).toBe(13); + expect(await currentOccurrenceCount("oversized-delete", "columns")).toBe(501); }); it("leaves synchronous backwards-compatible refresh outside admission", async () => { @@ -154,7 +156,35 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { expect(Number(count.count)).toBe(13); }); - it("defers immediately after a reserved conflict inside the 40-query envelope", async () => { + it("does not begin a bulk publication past the event query reservation", async () => { + await insertEntry("query-bound-a", 1); + await insertEntry("query-bound-b", 1); + const projections = ( + await Promise.all([loadSnapshots("query-bound-a"), loadSnapshots("query-bound-b")]) + ).flatMap((snapshots) => + snapshots.map((snapshot) => ({ + source: snapshot.source, + occurrences: snapshot.occurrences, + })), + ); + const metrics = createRequestMetrics(performance.now()); + const initialDbCount = + MEDIA_USAGE_MAINTENANCE_LIMITS.eventQueryCeiling - + MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries - + 3; + metrics.dbCount = initialDbCount; + + const inserted = await runWithContext({ editMode: false, metrics }, () => + repo.replaceNewSourcesBatch(projections), + ); + + expect(inserted).toEqual(new Set()); + expect(metrics.dbCount).toBe(initialDbCount); + expect((await repo.findSources(sourceKeys("query-bound-a"))).size).toBe(0); + expect((await repo.findSources(sourceKeys("query-bound-b"))).size).toBe(0); + }); + + it("defers immediately after a reserved conflict inside the shared step reservation", async () => { await insertEntry("conflict", 0); await addMediaUsageMeasurementDraft( ctx.db, @@ -186,7 +216,7 @@ describeEachDialect("media usage projection admission runtime", (dialect) => { "conflict", ); expect(result.outcome).toBe("retry"); - expect(counter.count).toBeLessThanOrEqual(40); + expect(counter.count).toBeLessThanOrEqual(MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries); expect( (await occurrenceCountForSource(sourceKey("conflict", "draft_overlay"))) - draftRowsBefore, ).toBe(12); diff --git a/packages/core/tests/integration/database/media-usage-projection-admission.test.ts b/packages/core/tests/integration/database/media-usage-projection-admission.test.ts index e98b744ff0..8fd57ca08b 100644 --- a/packages/core/tests/integration/database/media-usage-projection-admission.test.ts +++ b/packages/core/tests/integration/database/media-usage-projection-admission.test.ts @@ -54,24 +54,34 @@ describeEachDialect("content media usage projection admission", (dialect) => { await teardownForDialect(ctx); }); - it("admits a whole entry only when the combined variant plan fits", async () => { + it("admits 500 occurrences across both variants and rejects the next occurrence", async () => { const withinBudget = await plan([ - await snapshot("combined-12", "columns", 6), - await snapshot("combined-12", "draft_overlay", 6), + await snapshot("combined-500", "columns", 250), + await snapshot("combined-500", "draft_overlay", 250), ]); const overBudget = await plan([ - await snapshot("combined-13", "columns", 6), - await snapshot("combined-13", "draft_overlay", 7), + await snapshot("combined-501", "columns", 250), + await snapshot("combined-501", "draft_overlay", 251), ]); expect(withinBudget.outcome).toBe("admitted"); expect(overBudget.outcome).toBe("intrinsic_resource_limit"); }); + it("rejects one oversized variant before the whole claim can be admitted", async () => { + const atBoundary = await snapshot("bytes-at-boundary", "columns", 0); + atBoundary.projectionByteLength = 2_000_000; + const aboveBoundary = await snapshot("bytes-above-boundary", "columns", 0); + aboveBoundary.projectionByteLength = 2_000_001; + + expect((await plan([atBoundary])).outcome).toBe("admitted"); + expect((await plan([aboveBoundary])).outcome).toBe("intrinsic_resource_limit"); + }); + it("recomputes an oversized mixed plan after proving its replacement is unchanged", async () => { const contentId = "mixed-no-op-delete"; await insertContentIdentity(ctx, contentId, true); - const unchangedColumns = await snapshot(contentId, "columns", 0, "é".repeat(270_000)); + const unchangedColumns = await snapshot(contentId, "columns", 0, "é".repeat(1_050_000)); const absentDraft = await snapshot(contentId, "draft_overlay", 1, "Small draft"); await repo.replaceSource(unchangedColumns.source, unchangedColumns.occurrences); await repo.replaceSource(absentDraft.source, absentDraft.occurrences); @@ -101,7 +111,7 @@ describeEachDialect("content media usage projection admission", (dialect) => { it("rejects deletion when the stored source row alone exceeds the byte limit", async () => { const contentId = "oversized-source-delete"; await insertContentIdentity(ctx, contentId, true); - const absentDraft = await snapshot(contentId, "draft_overlay", 0, "é".repeat(300_000)); + const absentDraft = await snapshot(contentId, "draft_overlay", 0, "é".repeat(1_050_000)); await repo.replaceSource(absentDraft.source, []); const observedSources = await repo.findSources(canonicalSourceKeys(contentId)); const budget = createContentMediaUsageAdmissionBudget(); @@ -127,7 +137,7 @@ describeEachDialect("content media usage projection admission", (dialect) => { canonicalSourceKeys("reserved-small"), budget, ); - const oversized = [await snapshot("reserved-large", "columns", 13)]; + const oversized = [await snapshot("reserved-large", "columns", 501)]; const deferred = await planContentMediaUsageProjectionAdmission( repo, oversized, @@ -145,7 +155,7 @@ describeEachDialect("content media usage projection admission", (dialect) => { expect(first.outcome).toBe("admitted"); expect(budget.hasReservedMutation).toBe(true); - expect(budget.remainingOccurrenceMutationUnits).toBe(11); + expect(budget.remainingOccurrenceMutationUnits).toBe(499); expect(deferred.outcome).toBe("claim_budget_deferred"); expect(freshClaim.outcome).toBe("intrinsic_resource_limit"); }); diff --git a/packages/core/tests/integration/database/media-usage-read-repository.test.ts b/packages/core/tests/integration/database/media-usage-read-repository.test.ts index 51503d836b..59d4c84e74 100644 --- a/packages/core/tests/integration/database/media-usage-read-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-read-repository.test.ts @@ -357,6 +357,136 @@ describeEachDialect("MediaUsageRepository reads", (dialect) => { ]); }); + it("summarizes current collection indexing progress and terminal work", async () => { + await registerCollection(ctx, "pages"); + await registerCollection(ctx, "posts"); + await insertCollectionStatus(ctx, "pages", "complete", 0); + await insertCollectionStatus(ctx, "posts", "stale", 1); + + const progress = () => repo.findCollectionProgress(); + await expect(progress()).resolves.toBeNull(); + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + + await expect(progress()).resolves.toEqual({ + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + await ctx.db + .insertInto("_emdash_media_usage_work") + .values({ + collection_id: "collection-posts", + collection_slug: "posts", + content_id: "remaining-entry", + change_epoch: 1, + state: "pending", + next_attempt_at: "2026-08-16T00:00:00.000Z", + }) + .execute(); + await expect(progress()).resolves.toEqual({ + status: "indexing", + readyCollections: 1, + totalCollections: 2, + }); + await ctx.db + .updateTable("_emdash_media_usage_work") + .set({ state: "failed", last_error_code: "MEDIA_USAGE_PROCESSING_FAILED" }) + .where("collection_id", "=", "collection-posts") + .execute(); + await expect(progress()).resolves.toEqual({ + status: "needs_attention", + readyCollections: 1, + totalCollections: 2, + }); + + await ctx.db.deleteFrom("_emdash_media_usage_work").execute(); + await ctx.db.deleteFrom("_emdash_media_usage_reconciliations").execute(); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "complete", reconciliation_required: 0 }) + .where("collection_id", "=", "collection-posts") + .execute(); + await expect(progress()).resolves.toEqual({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + + await ctx.db + .insertInto("_emdash_media_usage_reconciliations") + .values({ + collection_id: "collection-posts", + collection_slug: "posts", + run_token: "obsolete-failed-run", + state: "failed", + next_attempt_at: "2026-08-16T00:00:00.000Z", + last_error_code: "MEDIA_USAGE_RECONCILIATION_FAILED", + }) + .execute(); + await expect(progress()).resolves.toEqual({ + status: "ready", + readyCollections: 2, + totalCollections: 2, + }); + + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: null }) + .where("collection_id", "=", "collection-pages") + .execute(); + await expect(progress()).resolves.toEqual({ + status: "needs_attention", + readyCollections: 1, + totalCollections: 2, + }); + + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ capture_state: "active" }) + .where("collection_id", "=", "collection-pages") + .execute(); + await ctx.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: "collection-posts", + collection_slug: "posts", + force_delete: 0, + state: "pending", + next_attempt_at: "2026-08-16T00:00:00.000Z", + }) + .execute(); + await expect(progress()).resolves.toEqual({ + status: "indexing", + readyCollections: 1, + totalCollections: 1, + }); + await ctx.db + .updateTable("_emdash_media_usage_collection_deletions") + .set({ state: "failed" }) + .where("collection_id", "=", "collection-posts") + .execute(); + await expect(progress()).resolves.toEqual({ + status: "needs_attention", + readyCollections: 1, + totalCollections: 1, + }); + + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ schema_version: 0 }) + .where("collection_id", "=", "collection-pages") + .execute(); + await expect(progress()).resolves.toEqual({ + status: "needs_attention", + readyCollections: 0, + totalCollections: 1, + }); + }); + it("paginates complete entry groups with nested sources and occurrences", async () => { await registerCollection(ctx, "pages"); await registerCollection(ctx, "posts"); @@ -499,6 +629,27 @@ async function registerCollection(ctx: DialectTestContext, slug: string): Promis .execute(); } +async function insertCollectionStatus( + ctx: DialectTestContext, + slug: string, + status: string, + reconciliationRequired: number, +): Promise { + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: slug, + status, + schema_version: 1, + collection_id: `collection-${slug}`, + reconciliation_required: reconciliationRequired, + capture_state: "active", + }) + .execute(); +} + async function installCanonicalContentFixture( ctx: DialectTestContext, collectionSlug: string, diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts index c3ae649d77..31e6452b6d 100644 --- a/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts +++ b/packages/core/tests/integration/database/media-usage-reconciliation-finalization.test.ts @@ -4,7 +4,10 @@ import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import { processDueMediaUsageCollectionDeletions } from "../../../src/media/usage/collection-deletion-processor.js"; import { loadContentMediaUsageSnapshots } from "../../../src/media/usage/content-snapshots.js"; -import { processDueMediaUsageReconciliation } from "../../../src/media/usage/reconciliation-processor.js"; +import { + processDueMediaUsageReconciliation, + processDueMediaUsageReconciliationDetailed, +} from "../../../src/media/usage/reconciliation-processor.js"; import { MediaUsageReconciliationRepository } from "../../../src/media/usage/reconciliation.js"; import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; import { processDueMediaUsageWork } from "../../../src/media/usage/work-processor.js"; @@ -103,6 +106,52 @@ describeEachDialect("media usage reconciliation finalization", (dialect) => { expect(await usage.findSource(quarantinedSourceKey)).not.toBeNull(); }); + it("reports a seeded or claimed reconciliation unit as consumed", async () => { + const collection = await createCollection(ctx, "consumed_reconciliation", true); + await activateCollection(ctx, collection); + + await expect(processDueMediaUsageReconciliationDetailed(ctx.db)).resolves.toMatchObject({ + consumedUnit: true, + outcome: "advanced", + }); + }); + + it("sweeps a full bulk batch of current sources in one reconciliation unit", async () => { + const collection = await createCollection(ctx, "bulk_sources", true); + for (let offset = 0; offset < 500; offset += 100) { + await ctx.db + .insertInto("ec_bulk_sources") + .values( + Array.from({ length: 100 }, (_, index) => { + const id = `entry-${String(offset + index).padStart(3, "0")}`; + return { id, slug: id }; + }), + ) + .execute(); + } + await activateCollection(ctx, collection); + + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + await expect(processDueMediaUsageWork(ctx.db)).resolves.toMatchObject({ completedCount: 500 }); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("advanced"); + await expect(processDueMediaUsageReconciliation(ctx.db)).resolves.toBe("completed"); + + expect( + await ctx.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("collection_id", "=", collection.id) + .executeTakeFirst(), + ).toBeUndefined(); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select(["status", "reconciliation_required"]) + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(), + ).toEqual({ status: "complete", reconciliation_required: 0 }); + }); + it("preserves automatic ownership until failed work reaches terminal coverage", async () => { const collection = await createCollection(ctx, "articles", true); await sql` diff --git a/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts index f5b98de0cf..9499304000 100644 --- a/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts +++ b/packages/core/tests/integration/database/media-usage-reconciliation-scan.test.ts @@ -58,7 +58,7 @@ describeEachDialect("media usage reconciliation scan", (dialect) => { expect(Number(coordinator.target_epoch)).toBe(1); expect(coordinator).toMatchObject({ scan_upper_id: "entry-050", - scan_cursor: "entry-049", + scan_cursor: "entry-050", }); expect(coordinator.field_fingerprint).toMatch(/^media-usage-fields:v1:sha256:[a-f0-9]{64}$/); const workCount = await ctx.db @@ -66,7 +66,7 @@ describeEachDialect("media usage reconciliation scan", (dialect) => { .select((eb) => eb.fn.countAll().as("count")) .where("collection_id", "=", collection.id) .executeTakeFirstOrThrow(); - expect(Number(workCount.count)).toBe(50); + expect(Number(workCount.count)).toBe(51); expect(await workState(ctx, collection.id, "entry-001")).toMatchObject({ change_epoch: 1, work_version: 2, @@ -89,7 +89,7 @@ describeEachDialect("media usage reconciliation scan", (dialect) => { await expect(claimAndScan(ctx)).resolves.toBe("advanced"); expect(await workVersions(ctx, collection.id)).toEqual(versionsBeforeReplay); - await expect(claimAndScan(ctx)).resolves.toBe("advanced"); + await expect(claimAndScan(ctx)).resolves.toBe("exhausted"); expect( await ctx.db .selectFrom("_emdash_media_usage_reconciliations") @@ -97,7 +97,6 @@ describeEachDialect("media usage reconciliation scan", (dialect) => { .where("collection_id", "=", collection.id) .executeTakeFirstOrThrow(), ).toEqual({ scan_cursor: "entry-050" }); - expect(await claimAndScan(ctx)).toBe("exhausted"); }); it("invalidates stale canonical publication after deletion or a newer version", async () => { diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index ce44eaa688..72d87dcdea 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -6,6 +6,10 @@ import { MediaUsageRepository } from "../../../src/database/repositories/media-u import { RevisionRepository } from "../../../src/database/repositories/revision.js"; import type { EmDashRuntime } from "../../../src/emdash-runtime.js"; import { setI18nConfig } from "../../../src/i18n/config.js"; +import { + activateMediaUsageCapture, + continueMediaUsageActivation, +} from "../../../src/media/usage/activation.js"; import { CONTENT_MEDIA_USAGE_ADAPTER_ID, CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, @@ -751,6 +755,38 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { ); }); + it("refreshes every i18n sibling while incremental capture is active", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "active-shared-image", { + enSharedHero: "media-active-old-en", + frSharedHero: "media-active-old-fr", + }); + await activateAllCollections(ctx.db); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { shared_hero: mediaRef("media-active-new") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-active-new")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ contentId: enId, sourceVariant: "columns" }), + }), + expect.objectContaining({ + source: expect.objectContaining({ contentId: frId, sourceVariant: "columns" }), + }), + ]), + ); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select("content_id") + .where("content_id", "in", [enId, frId]) + .execute(), + ).toEqual([]); + }); + it("does not refresh i18n siblings for translatable image updates", async () => { setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); const { enId, frId } = await createLocalizedPostsPair(runtime, "translatable-image", { @@ -914,6 +950,14 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { }); }); +async function activateAllCollections(db: DialectTestContext["db"]): Promise { + let result = await activateMediaUsageCapture(db, { writersDrained: true }); + while (result.outcome === "activating") { + result = await continueMediaUsageActivation(db); + } + if (result.outcome !== "active") throw new Error(`Activation stopped at ${result.outcome}`); +} + async function createLocalizedPostsPair( runtime: EmDashRuntime, slugPrefix: string, diff --git a/packages/core/tests/integration/database/media-usage-work-processor.test.ts b/packages/core/tests/integration/database/media-usage-work-processor.test.ts index c0e4bfa314..2efb0e3235 100644 --- a/packages/core/tests/integration/database/media-usage-work-processor.test.ts +++ b/packages/core/tests/integration/database/media-usage-work-processor.test.ts @@ -10,14 +10,17 @@ import type { import { sql } from "kysely"; import { afterEach, beforeEach, expect, it } from "vitest"; +import { MediaUsageWorkRepository } from "../../../src/database/repositories/media-usage-work.js"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import type { Database } from "../../../src/database/types.js"; import { installMediaUsageCaptureTriggers } from "../../../src/media/usage/capture-triggers.js"; +import { MEDIA_USAGE_MAINTENANCE_LIMITS } from "../../../src/media/usage/maintenance-engine.js"; import { MEDIA_USAGE_WORK_PROCESSING_LIMITS, processDueMediaUsageWork, processMediaUsageWorkAfterWrite, } from "../../../src/media/usage/work-processor.js"; +import { createRequestMetrics, runWithContext } from "../../../src/request-context.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { describeEachDialect, @@ -131,7 +134,7 @@ describeEachDialect("media usage durable work processing", (dialect) => { ); }); - it("bounds each scheduled tick and leaves the backlog durable", async () => { + it("processes all available work in one bulk batch", async () => { const fixture = await createActiveFixture(ctx, "articles"); for (let index = 0; index < 3; index++) { await insertEntry(ctx, fixture, `entry-${index}`, `media-${index}`); @@ -140,18 +143,278 @@ describeEachDialect("media usage durable work processing", (dialect) => { const result = await processDueMediaUsageWork(ctx.db); expect(result.candidateCount).toBe(3); - expect(result.claimedCount).toBe(MEDIA_USAGE_WORK_PROCESSING_LIMITS.jobsPerTick); - expect(result.completedCount).toBe(MEDIA_USAGE_WORK_PROCESSING_LIMITS.jobsPerTick); - expect(await countWork(ctx.db)).toBe(3 - MEDIA_USAGE_WORK_PROCESSING_LIMITS.jobsPerTick); + expect(result.claimedCount).toBe(3); + expect(result.completedCount).toBe(3); + expect(await countWork(ctx.db)).toBe(0); expect(await findCoverageStatus(ctx.db, fixture.collectionId)).toEqual( - expect.objectContaining({ status: "stale" }), + expect.objectContaining({ status: "complete" }), ); + }); + it("does not repeat the full projection query sequence per entry", async () => { + const fixture = await createActiveFixture(ctx, "bulk_articles"); + for (let index = 0; index < 10; index++) { + await insertEntry(ctx, fixture, `entry-${index}`, `media-${index}`); + } + const counter = new QueryCountingPlugin(); + + const result = await processDueMediaUsageWork(ctx.db.withPlugin(counter)); + + expect(result.completedCount).toBe(10); + expect(counter.count).toBeLessThan(50); + expect(await countWork(ctx.db)).toBe(0); + }); + + it("processes at most one thousand entries per bulk batch", async () => { + const fixture = await createActiveFixture(ctx, "large_batch"); + for (let index = 0; index < 1_001; index++) { + await insertEntry( + ctx, + fixture, + `entry-${String(index).padStart(4, "0")}`, + `media-${index}`, + 20, + ); + } + const counter = new QueryCountingPlugin(); + + const result = await processDueMediaUsageWork(ctx.db.withPlugin(counter)); + + expect(result.claimedCount).toBe(1_000); + expect(result.completedCount).toBe(1_000); + expect(counter.count).toBeLessThan(50); + expect(await countWork(ctx.db)).toBe(1); + }); + + it( + "releases untouched work when aggregate projection memory is full", + { timeout: 30_000 }, + async () => { + const fixture = await createActiveFixture(ctx, "memory_bound_batch"); + for (let index = 0; index < 101; index++) { + await insertEntry( + ctx, + fixture, + `entry-${String(index).padStart(3, "0")}`, + `media-${index}`, + 500, + ); + } + + const first = await processDueMediaUsageWork(ctx.db); + + expect(first.completedCount).toBeGreaterThan(0); + expect(first.completedCount).toBeLessThan(101); + expect(first.retryCount).toBe(0); + expect(first.failedCount).toBe(0); + const remaining = await ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["state", "attempt_count", "last_error_code"]) + .execute(); + expect(remaining.length).toBeGreaterThan(0); + expect( + remaining.every( + (row) => + row.state === "pending" && row.attempt_count === 0 && row.last_error_code === null, + ), + ).toBe(true); + + let completed = first.completedCount; + for (let step = 0; step < 101 && (await countWork(ctx.db)) > 0; step++) { + const next = await processDueMediaUsageWork(ctx.db); + expect(next.retryCount).toBe(0); + expect(next.failedCount).toBe(0); + expect(next.completedCount).toBeGreaterThan(0); + completed += next.completedCount; + } + + expect(completed).toBe(101); + expect(await countWork(ctx.db)).toBe(0); + }, + ); + + it("updates existing projections without one query sequence per entry", async () => { + const fixture = await createActiveFixture(ctx, "existing_batch"); + for (let index = 0; index < 100; index++) { + await insertEntry(ctx, fixture, `entry-${String(index).padStart(3, "0")}`, `media-${index}`); + } await processDueMediaUsageWork(ctx.db); - await processDueMediaUsageWork(ctx.db); + await sql` + UPDATE ${sql.ref(fixture.tableName)} + SET hero = ${JSON.stringify({ id: "updated-media", provider: "local", mimeType: "image/webp" })}, + version = version + 1, + updated_at = '2026-08-20T00:00:00.000Z' + `.execute(ctx.db); + const counter = new QueryCountingPlugin(); + + const result = await processDueMediaUsageWork(ctx.db.withPlugin(counter)); + + expect(result.completedCount).toBe(100); + expect(counter.count).toBeLessThan(50); + expect(await countWork(ctx.db)).toBe(0); + }); + + it("lets only one overlapping bulk processor own each work row", async () => { + const fixture = await createActiveFixture(ctx, "overlapping_bulk"); + for (let index = 0; index < 4; index++) { + await insertEntry(ctx, fixture, `entry-${index}`, `media-${index}`); + } + + const results = await Promise.all([ + processDueMediaUsageWork(ctx.db), + processDueMediaUsageWork(ctx.db), + ]); + + expect(results.reduce((total, result) => total + result.completedCount, 0)).toBe(4); + expect(await countWork(ctx.db)).toBe(0); + }); + + it("does not release a claimed batch based only on elapsed time", async () => { + const fixture = await createActiveFixture(ctx, "elapsed_time"); + await insertEntry(ctx, fixture, "entry-1", "media-1"); + const metrics = createRequestMetrics(performance.now() - 60 * 60 * 1_000); + + const result = await runWithContext({ editMode: false, metrics }, () => + processDueMediaUsageWork(ctx.db), + ); + + expect(result.completedCount).toBe(1); expect(await countWork(ctx.db)).toBe(0); + }); + + it("retries every incomplete row after a bulk publication failure", async () => { + const fixture = await createActiveFixture(ctx, "bulk_failure"); + await insertEntry(ctx, fixture, "entry-1", "media-1"); + await insertEntry(ctx, fixture, "entry-2", "media-2"); + await installProjectionFailureTrigger(ctx); + + const result = await processDueMediaUsageWork(ctx.db); + await removeProjectionFailureTrigger(ctx); + + expect(result.retryCount).toBe(2); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["state", "attempt_count", "last_error_code"]) + .orderBy("content_id") + .execute(), + ).toEqual([ + { state: "retry", attempt_count: 1, last_error_code: "MEDIA_USAGE_PROCESSING_FAILED" }, + { state: "retry", attempt_count: 1, last_error_code: "MEDIA_USAGE_PROCESSING_FAILED" }, + ]); + + await ctx.db + .updateTable("_emdash_media_usage_work") + .set({ + state: "pending", + attempt_count: MEDIA_USAGE_WORK_PROCESSING_LIMITS.maxAttempts - 1, + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + await installProjectionFailureTrigger(ctx); + const terminal = await processDueMediaUsageWork(ctx.db); + await removeProjectionFailureTrigger(ctx); + + expect(terminal.failedCount).toBe(2); expect(await findCoverageStatus(ctx.db, fixture.collectionId)).toEqual( - expect.objectContaining({ status: "complete" }), + expect.objectContaining({ + status: "partial", + last_error_code: "MEDIA_USAGE_PROCESSING_FAILED", + }), + ); + }); + + it("does not retry healthy collections when one collection fails", async () => { + const broken = await createActiveFixture(ctx, "broken_collection"); + const healthy = await createActiveFixture(ctx, "healthy_collection"); + await insertEntry(ctx, broken, "broken-entry", "broken-media"); + await insertEntry(ctx, healthy, "healthy-entry", "healthy-media"); + await installProjectionFailureTrigger(ctx, broken.collectionId); + + const result = await processDueMediaUsageWork(ctx.db); + await removeProjectionFailureTrigger(ctx); + + expect(result.completedCount).toBe(1); + expect(result.retryCount).toBe(1); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_work") + .select(["collection_id", "state", "attempt_count", "last_error_code"]) + .execute(), + ).toEqual([ + { + collection_id: broken.collectionId, + state: "retry", + attempt_count: 1, + last_error_code: "MEDIA_USAGE_PROCESSING_FAILED", + }, + ]); + expect( + await new MediaUsageRepository(ctx.db).findSource( + canonicalSourceKey(healthy.collectionId, "healthy-entry"), + ), + ).not.toBeNull(); + }); + + it("recovers coverage when work deletion committed before its status update", async () => { + const fixture = await createActiveFixture(ctx, "finalization_recovery"); + await insertEntry(ctx, fixture, "entry-1", "media-1"); + const pending = await findWork(ctx.db); + const work = new MediaUsageWorkRepository(ctx.db); + const claimed = await work.claimWork({ + collectionId: fixture.collectionId, + contentId: "entry-1", + workVersion: pending.work_version, + leaseDurationSeconds: 60, + }); + if (!claimed?.leaseToken) throw new Error("Expected claimed work"); + const usage = new MediaUsageRepository(ctx.db); + const marker = await usage.prepareIncrementalFinalization({ + collectionId: fixture.collectionId, + collectionSlug: fixture.collectionSlug, + }); + expect(marker.outcome).toBe("marked"); + expect( + await work.completeWorkBatch([ + { + collectionId: fixture.collectionId, + contentId: "entry-1", + workVersion: claimed.workVersion, + leaseToken: claimed.leaseToken, + }, + ]), + ).toEqual( + new Set([`${fixture.collectionId}\u0000entry-1\u0000${String(claimed.workVersion)}`]), + ); + expect( + await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select("collection_id") + .where("collection_id", "=", fixture.collectionId) + .where( + sql`cursor LIKE ('incremental-finalize:' || CAST(change_epoch AS text) || ':%')`, + ) + .executeTakeFirst(), + ).toBeDefined(); + await processDueMediaUsageWork(ctx.db); + + expect(await findCoverageStatus(ctx.db, fixture.collectionId)).toEqual( + expect.objectContaining({ status: "complete", cursor: null }), + ); + }); + + it("does not complete an unmarked stale collection with no work", async () => { + const fixture = await createActiveFixture(ctx, "unmarked_stale"); + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", completed_at: null, cursor: null }) + .where("collection_id", "=", fixture.collectionId) + .execute(); + + await processDueMediaUsageWork(ctx.db); + + expect(await findCoverageStatus(ctx.db, fixture.collectionId)).toEqual( + expect.objectContaining({ status: "stale", completed_at: null, cursor: null }), ); }); @@ -285,7 +548,7 @@ describeEachDialect("media usage durable work processing", (dialect) => { ).toBeNull(); }); - it("keeps an ordinary job inside the exported statement envelope", async () => { + it("keeps an ordinary job inside the shared step reservation", async () => { const fixture = await createActiveFixture(ctx, "measured"); await insertEntry(ctx, fixture, "entry-1", "media-1"); const counter = new QueryCountingPlugin(); @@ -298,9 +561,7 @@ describeEachDialect("media usage durable work processing", (dialect) => { expect(result.outcome).toBe("completed"); expect(counter.count).toBeGreaterThan(0); - expect(counter.count).toBeLessThanOrEqual( - MEDIA_USAGE_WORK_PROCESSING_LIMITS.ordinaryStatementsPerJob, - ); + expect(counter.count).toBeLessThanOrEqual(MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries); }); }); @@ -322,6 +583,7 @@ async function createActiveFixture(ctx: DialectTestContext, collectionSlug: stri await registry.createCollection({ slug: collectionSlug, label: collectionSlug }); await registry.createField(collectionSlug, { slug: "title", label: "Title", type: "string" }); await registry.createField(collectionSlug, { slug: "hero", label: "Hero", type: "image" }); + await registry.createField(collectionSlug, { slug: "body", label: "Body", type: "portableText" }); const collection = await registry.getCollection(collectionSlug); if (!collection) throw new Error(`Expected ${collectionSlug} collection`); @@ -364,15 +626,21 @@ async function insertEntry( fixture: Awaited>, contentId: string, mediaId: string, + referenceCount = 1, ): Promise { + const body = Array.from({ length: Math.max(0, referenceCount - 1) }, (_, index) => ({ + _type: "image", + asset: { _ref: `${mediaId}-body-${index}` }, + })); await sql` - INSERT INTO ${sql.ref(fixture.tableName)} (id, slug, status, title, hero) + INSERT INTO ${sql.ref(fixture.tableName)} (id, slug, status, title, hero, body) VALUES ( ${contentId}, ${contentId}, 'published', ${contentId}, - ${JSON.stringify({ id: mediaId, provider: "local", mimeType: "image/webp" })} + ${JSON.stringify({ id: mediaId, provider: "local", mimeType: "image/webp" })}, + ${JSON.stringify(body)} ) `.execute(ctx.db); } @@ -462,3 +730,53 @@ async function removeProjectionSupersessionTrigger(ctx: DialectTestContext): Pro } await sql`DROP TRIGGER emdash_test_supersede_media_usage_work`.execute(ctx.db); } + +async function installProjectionFailureTrigger( + ctx: DialectTestContext, + collectionId?: string, +): Promise { + const collectionMatches = collectionId + ? sql`NEW.collection_id = ${sql.lit(collectionId)}` + : sql`TRUE`; + if (ctx.dialect === "postgres") { + await sql` + CREATE OR REPLACE FUNCTION emdash_test_fail_media_usage_projection() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF ${collectionMatches} THEN + RAISE EXCEPTION 'forced media usage projection failure'; + END IF; + RETURN NEW; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER emdash_test_fail_media_usage_projection + BEFORE INSERT ON _emdash_media_usage_sources + FOR EACH ROW EXECUTE FUNCTION emdash_test_fail_media_usage_projection() + `.execute(ctx.db); + return; + } + await sql` + CREATE TRIGGER emdash_test_fail_media_usage_projection + BEFORE INSERT ON _emdash_media_usage_sources + WHEN ${collectionMatches} + BEGIN + SELECT RAISE(ABORT, 'forced media usage projection failure'); + END + `.execute(ctx.db); +} + +async function removeProjectionFailureTrigger(ctx: DialectTestContext): Promise { + if (ctx.dialect === "postgres") { + await sql` + DROP TRIGGER emdash_test_fail_media_usage_projection + ON _emdash_media_usage_sources + `.execute(ctx.db); + await sql`DROP FUNCTION emdash_test_fail_media_usage_projection()`.execute(ctx.db); + return; + } + await sql`DROP TRIGGER emdash_test_fail_media_usage_projection`.execute(ctx.db); +} diff --git a/packages/core/tests/integration/database/media-usage-work-repository.test.ts b/packages/core/tests/integration/database/media-usage-work-repository.test.ts index 5ac94fdc72..74462c7332 100644 --- a/packages/core/tests/integration/database/media-usage-work-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-work-repository.test.ts @@ -84,21 +84,14 @@ describeEachDialect("media usage work repository", (dialect) => { const replacement = await claimWork(repo, 1); expect(replacement?.leaseToken).toBeTruthy(); expect(replacement?.leaseToken).not.toBe("old-owner"); - expect(await repo.completeWork(lease("old-owner", 1))).toBe(false); + expect(await completeClaim(repo, lease("old-owner", 1))).toBe(false); expect( - await repo.retryWork({ - ...lease("old-owner", 1), - retryDelaySeconds: 1, - errorCode: "TRANSIENT_DATABASE_FAILURE", - }), - ).toBe(false); + await transitionClaim(repo, lease("old-owner", 1), "TRANSIENT_DATABASE_FAILURE"), + ).toBeUndefined(); expect( - await repo.failWork({ - ...lease("old-owner", 1), - errorCode: "INVARIANT_FAILURE", - }), - ).toBe(false); - expect(await repo.completeWork(lease(replacement!.leaseToken!, 1))).toBe(true); + await transitionClaim(repo, lease("old-owner", 1), "INVARIANT_FAILURE", 1, 0), + ).toBeUndefined(); + expect(await completeClaim(repo, lease(replacement!.leaseToken!, 1))).toBe(true); expect(await readWork(ctx)).toBeUndefined(); }); @@ -121,20 +114,13 @@ describeEachDialect("media usage work repository", (dialect) => { }) .execute(); - expect(await repo.completeWork(lease(staleClaim!.leaseToken!, 1))).toBe(false); + expect(await completeClaim(repo, lease(staleClaim!.leaseToken!, 1))).toBe(false); expect( - await repo.retryWork({ - ...lease(staleClaim!.leaseToken!, 1), - retryDelaySeconds: 1, - errorCode: "TRANSIENT_DATABASE_FAILURE", - }), - ).toBe(false); + await transitionClaim(repo, lease(staleClaim!.leaseToken!, 1), "TRANSIENT_DATABASE_FAILURE"), + ).toBeUndefined(); expect( - await repo.failWork({ - ...lease(staleClaim!.leaseToken!, 1), - errorCode: "INVARIANT_FAILURE", - }), - ).toBe(false); + await transitionClaim(repo, lease(staleClaim!.leaseToken!, 1), "INVARIANT_FAILURE", 1, 0), + ).toBeUndefined(); expect(await readWork(ctx)).toEqual( expect.objectContaining({ work_version: expect.toSatisfy((value) => Number(value) === 2), @@ -152,12 +138,8 @@ describeEachDialect("media usage work repository", (dialect) => { expect(retryClaim).not.toBeNull(); expect( - await repo.retryWork({ - ...lease(retryClaim!.leaseToken!, 1), - retryDelaySeconds: 60, - errorCode: "SNAPSHOT_FAILURE", - }), - ).toBe(true); + await transitionClaim(repo, lease(retryClaim!.leaseToken!, 1), "SNAPSHOT_FAILURE", 5, 60), + ).toBe("retry"); const retry = await readWork(ctx); expect(retry).toEqual( expect.objectContaining({ @@ -178,11 +160,8 @@ describeEachDialect("media usage work repository", (dialect) => { const failureClaim = await claimWork(repo, 1); expect(failureClaim).not.toBeNull(); expect( - await repo.failWork({ - ...lease(failureClaim!.leaseToken!, 1), - errorCode: "RESOURCE_LIMIT", - }), - ).toBe(true); + await transitionClaim(repo, lease(failureClaim!.leaseToken!, 1), "RESOURCE_LIMIT", 1, 0), + ).toBe("failed"); expect(await readWork(ctx)).toEqual( expect.objectContaining({ state: "failed", @@ -205,15 +184,9 @@ describeEachDialect("media usage work repository", (dialect) => { .execute(); const expiredLease = lease(expiredClaim!.leaseToken!, 1); - expect(await repo.completeWork(expiredLease)).toBe(false); - expect( - await repo.retryWork({ - ...expiredLease, - retryDelaySeconds: 1, - errorCode: "TRANSIENT_DATABASE_FAILURE", - }), - ).toBe(false); - expect(await repo.failWork({ ...expiredLease, errorCode: "INVARIANT_FAILURE" })).toBe(false); + expect(await completeClaim(repo, expiredLease)).toBe(false); + expect(await transitionClaim(repo, expiredLease, "TRANSIENT_DATABASE_FAILURE")).toBeUndefined(); + expect(await transitionClaim(repo, expiredLease, "INVARIANT_FAILURE", 1, 0)).toBeUndefined(); expect(await readWork(ctx)).toEqual( expect.objectContaining({ state: "leased", lease_token: expiredClaim!.leaseToken }), ); @@ -250,11 +223,11 @@ describeEachDialect("media usage work repository", (dialect) => { expect(workClaim).not.toBeNull(); await expect( - repo.retryWork({ - ...lease(workClaim!.leaseToken!, 1), - retryDelaySeconds: 1, - errorCode: "database connection failed for customer data", - }), + transitionClaim( + repo, + lease(workClaim!.leaseToken!, 1), + "database connection failed for customer data", + ), ).rejects.toThrow(/stable SCREAMING_SNAKE_CASE/i); expect(await readWork(ctx)).toEqual( expect.objectContaining({ @@ -314,13 +287,10 @@ describeEachDialect("media usage work repository", (dialect) => { nextAttemptAt: "1998-01-01T00:00:00.000Z", }); - const due = await repo.findDueWork(10); - - expect(due.map((work) => work.contentId)).toEqual([ - "lease-expired", - "retry-due", - "pending-due", - ]); + const first = await repo.claimDueWorkBatch({ limit: 2, leaseDurationSeconds: 60 }); + expect(first.map((work) => work.contentId).toSorted()).toEqual(["lease-expired", "retry-due"]); + const second = await repo.claimDueWorkBatch({ limit: 10, leaseDurationSeconds: 60 }); + expect(second.map((work) => work.contentId)).toEqual(["pending-due"]); }); }); @@ -342,6 +312,34 @@ function lease(leaseToken: string, workVersion: number) { }; } +async function completeClaim( + repo: MediaUsageWorkRepository, + work: ReturnType, +): Promise { + return (await repo.completeWorkBatch([work])).has(workIdentityKey(work)); +} + +async function transitionClaim( + repo: MediaUsageWorkRepository, + work: ReturnType, + errorCode: string, + maxAttempts = 5, + retryDelaySeconds = 1, +) { + return ( + await repo.retryClaimedWorkBatch({ + work: [work], + errorCode, + maxAttempts, + retryDelaySeconds, + }) + ).get(workIdentityKey(work)); +} + +function workIdentityKey(work: ReturnType): string { + return `${work.collectionId}\u0000${work.contentId}\u0000${String(work.workVersion)}`; +} + async function insertWork( ctx: DialectTestContext, input: { diff --git a/packages/core/tests/integration/runtime/fresh-site-media-usage.test.ts b/packages/core/tests/integration/runtime/fresh-site-media-usage.test.ts new file mode 100644 index 0000000000..e140db99a8 --- /dev/null +++ b/packages/core/tests/integration/runtime/fresh-site-media-usage.test.ts @@ -0,0 +1,130 @@ +import { randomUUID } from "node:crypto"; + +import Database from "better-sqlite3"; +import { Kysely, SqliteDialect } from "kysely"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock( + "virtual:emdash/seed", + () => ({ + seed: { + version: "1", + settings: {}, + collections: [ + { + slug: "posts", + label: "Posts", + fields: [{ slug: "hero", label: "Hero", type: "image" }], + }, + ], + content: { + posts: [ + { + id: "welcome", + slug: "welcome", + data: { + hero: { id: "media-1", provider: "local", mimeType: "image/webp" }, + }, + }, + ], + }, + }, + userSeed: null, + }), + { virtual: true }, +); + +import { runMigrations } from "../../../src/database/migrations/runner.js"; +import { OptionsRepository } from "../../../src/database/repositories/options.js"; +import type { Database as EmDashDatabase } from "../../../src/database/types.js"; +import { EmDashRuntime, type RuntimeDependencies } from "../../../src/emdash-runtime.js"; +import { verifyMediaUsageCaptureTriggers } from "../../../src/media/usage/capture-triggers.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { applySeed } from "../../../src/seed/apply.js"; +import { loadSeed } from "../../../src/seed/load.js"; + +describe("fresh-site media usage tracking", () => { + let runtime: EmDashRuntime | undefined; + let setupDb: Kysely | undefined; + + afterEach(async () => { + await runtime?.stopCron(); + await setupDb?.destroy(); + runtime = undefined; + setupDb = undefined; + }); + + it("activates capture before creating seeded collections", async () => { + runtime = await EmDashRuntime.create(createDeps()); + + const activation = await runtime.db + .selectFrom("_emdash_media_usage_activation") + .select(["state", "activated_at"]) + .where("task_key", "=", "incremental_capture") + .executeTakeFirstOrThrow(); + expect(activation).toEqual({ state: "active", activated_at: expect.any(String) }); + + const collection = await new SchemaRegistry(runtime.db).getCollection("posts"); + if (!collection) throw new Error("Expected seeded posts collection"); + expect( + await verifyMediaUsageCaptureTriggers(runtime.db, { + collectionId: collection.id, + collectionSlug: collection.slug, + }), + ).toBe(true); + + await applySeed(runtime.db, await loadSeed(), { includeContent: true, onConflict: "skip" }); + const content = await runtime.db + .selectFrom("ec_posts") + .select("id") + .where("slug", "=", "welcome") + .executeTakeFirstOrThrow(); + expect( + await runtime.db + .selectFrom("_emdash_media_usage_work") + .select(["collection_id", "content_id", "state"]) + .execute(), + ).toEqual([{ collection_id: collection.id, content_id: content.id, state: "pending" }]); + }); + + it("leaves an existing configured site inactive", async () => { + const sqlite = new Database(":memory:"); + setupDb = new Kysely({ + dialect: new SqliteDialect({ database: sqlite }), + }); + await runMigrations(setupDb); + await new SchemaRegistry(setupDb).createCollection({ slug: "articles", label: "Articles" }); + await new OptionsRepository(setupDb).set("emdash:setup_complete", true); + + runtime = await EmDashRuntime.create({ + ...createDeps(), + createDialect: () => new SqliteDialect({ database: sqlite }), + }); + + expect( + await runtime.db + .selectFrom("_emdash_media_usage_activation") + .select(["state", "activated_at"]) + .where("task_key", "=", "incremental_capture") + .executeTakeFirstOrThrow(), + ).toEqual({ state: "expanded", activated_at: null }); + }); +}); + +function createDeps(): RuntimeDependencies { + return { + config: { + database: { + entrypoint: `fresh-site-media-usage-${randomUUID()}`, + config: {}, + type: "sqlite", + }, + }, + plugins: [], + createDialect: () => new SqliteDialect({ database: new Database(":memory:") }), + createStorage: null, + sandboxEnabled: false, + sandboxedPluginEntries: [], + createSandboxRunner: null, + }; +} diff --git a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts index f58c1f43d5..b8ccf17cb1 100644 --- a/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts +++ b/packages/core/tests/integration/runtime/media-usage-scheduled-driver.test.ts @@ -4,19 +4,19 @@ import Database from "better-sqlite3"; import { sql, SqliteDialect } from "kysely"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { MediaUsageWorkRepository } from "../../../src/database/repositories/media-usage-work.js"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import { OptionsRepository } from "../../../src/database/repositories/options.js"; -import { - EmDashRuntime, - MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS, - type RuntimeDependencies, -} from "../../../src/emdash-runtime.js"; +import { EmDashRuntime, type RuntimeDependencies } from "../../../src/emdash-runtime.js"; +import { activateMediaUsageCapture } from "../../../src/media/usage/activation.js"; import { installMediaUsageCaptureTriggers } from "../../../src/media/usage/capture-triggers.js"; +import { runMediaUsageMaintenanceStep } from "../../../src/media/usage/maintenance-engine.js"; +import { processDueMediaUsageReconciliation } from "../../../src/media/usage/reconciliation-processor.js"; +import { processDueMediaUsageWork } from "../../../src/media/usage/work-processor.js"; import type { CronScheduler, SystemCleanupFn } from "../../../src/plugins/scheduler/types.js"; -import { createRequestMetrics, runWithContext } from "../../../src/request-context.js"; import { SCHEDULER_HEARTBEAT_OPTION } from "../../../src/scheduler-health.js"; -describe("media usage scheduled drivers", () => { +describe("media usage maintenance engine and Node heartbeat", () => { let runtime: EmDashRuntime | null = null; afterEach(async () => { @@ -24,44 +24,198 @@ describe("media usage scheduled drivers", () => { runtime = null; }); - it("keeps general maintenance unchanged and drains work from the dedicated lane", async () => { + it("skips idle maintenance classes and processes due entry work", async () => { runtime = await EmDashRuntime.create(createDeps(null)); - const fixture = await activateCollection(runtime, "cloudflare_posts"); + const fixture = await activateCollection(runtime, "work_conserving_posts"); await insertEntry(runtime, fixture.tableName, "entry-1"); - await runtime.runScheduledTasks(); + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "progress", + continuation: { kind: "immediate" }, + }); + expect(await countWork(runtime)).toBe(0); + }); + + it("delays one continuation when every visible claim is blocked", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "blocked_claim_posts"); + await insertEntry(runtime, fixture.tableName, "entry-1"); + await sql` + CREATE TRIGGER block_media_usage_work_claim + BEFORE UPDATE OF state ON _emdash_media_usage_work + WHEN NEW.state = 'leased' + BEGIN + SELECT RAISE(IGNORE); + END + `.execute(runtime.db); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, + }); expect(await countWork(runtime)).toBe(1); - await expect(runtime.runScheduledMediaUsageTasks()).resolves.toMatchObject({ - outcome: "processed", - taskClass: "entry_work", + }); + + it("keeps a delayed continuation while retry work is waiting", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "delayed_retry_posts"); + await insertEntry(runtime, fixture.tableName, "entry-1"); + await runtime.db + .updateTable("_emdash_media_usage_work") + .set({ state: "retry", next_attempt_at: "2100-01-01T00:00:00.000Z" }) + .execute(); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, }); + expect(await countWork(runtime)).toBe(1); + }); + it("keeps a delayed continuation while collection deletion retry is waiting", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "delayed_deletion"); + await runtime.db + .insertInto("_emdash_media_usage_collection_deletions") + .values({ + collection_id: fixture.collectionId, + collection_slug: "delayed_deletion", + force_delete: 1, + state: "retry", + phase: "sources", + next_attempt_at: "2100-01-01T00:00:00.000Z", + }) + .execute(); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, + }); + }); + + it("continues useful entry work after a reconciliation claim is lost", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const work = await activateCollection(runtime, "seed_claim_work"); + await insertEntry(runtime, work.tableName, "entry-1"); + const reconciliation = await activateCollection(runtime, "seed_claim_reconciliation"); + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", reconciliation_required: 1 }) + .where("collection_id", "=", reconciliation.collectionId) + .execute(); + await sql` + CREATE TRIGGER block_media_usage_reconciliation_claim + BEFORE UPDATE OF state ON _emdash_media_usage_reconciliations + WHEN NEW.state = 'leased' + BEGIN + SELECT RAISE(IGNORE); + END + `.execute(runtime.db); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "progress", + continuation: { kind: "immediate" }, + }); expect(await countWork(runtime)).toBe(0); - expect( - await new MediaUsageRepository(runtime.db).findSource( - canonicalSourceKey(fixture.collectionId, "entry-1"), - ), - ).not.toBeNull(); }); - it("drains bounded work from the Node timer maintenance callback", async () => { - const scheduler = new CapturingScheduler(); - runtime = await EmDashRuntime.create(createDeps(() => scheduler)); - const fixture = await activateCollection(runtime, "node_posts"); - await insertEntry(runtime, fixture.tableName, "entry-1"); + it("keeps a blocked due reconciliation on its delayed continuation", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "blocked_due_reconciliation"); + const runToken = "blocked-due-run"; + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ + status: "running", + cursor: runToken, + change_epoch: 1, + reconciliation_required: 1, + }) + .where("collection_id", "=", fixture.collectionId) + .execute(); + await runtime.db + .insertInto("_emdash_media_usage_reconciliations") + .values({ + collection_id: fixture.collectionId, + collection_slug: "blocked_due_reconciliation", + run_token: runToken, + target_epoch: 1, + field_fingerprint: "blocked-due-fields", + state: "pending", + phase: "sources", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); + await sql` + CREATE TRIGGER block_due_reconciliation_claim + BEFORE UPDATE OF state ON _emdash_media_usage_reconciliations + WHEN NEW.state = 'leased' + BEGIN + SELECT RAISE(IGNORE); + END + `.execute(runtime.db); - await scheduler.runMaintenance(); + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toMatchObject({ + state: "blocked", + continuation: { kind: "delayed", delaySeconds: 30 }, + }); + }); + + it("stops continuation only after a full idle maintenance pass", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + await activateCollection(runtime, "idle_posts"); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "idle", + continuation: { kind: "none" }, + }); + }); + it("continues reconciliation immediately after entry work drains", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + await configureExistingInactiveSite(runtime); + await runtime.schemaRegistry.createCollection({ + slug: "deferred_reconciliation", + label: "Deferred reconciliation", + }); + await runtime.schemaRegistry.createField("deferred_reconciliation", { + slug: "title", + label: "Title", + type: "string", + }); + await runtime.schemaRegistry.createField("deferred_reconciliation", { + slug: "image", + label: "Image", + type: "image", + }); + await sql` + INSERT INTO ${sql.ref("ec_deferred_reconciliation")} (id, slug, status, title) + VALUES + ('entry-1', 'entry-1', 'published', 'Entry 1'), + ('entry-2', 'entry-2', 'published', 'Entry 2') + `.execute(runtime.db); + await expect(activateMediaUsageCapture(runtime.db, { writersDrained: true })).resolves.toEqual({ + outcome: "active", + processedCollections: 1, + }); + + await expect(processDueMediaUsageReconciliation(runtime.db)).resolves.toBe("advanced"); + await expect(processDueMediaUsageReconciliation(runtime.db)).resolves.toBe("deferred"); + await processDueMediaUsageWork(runtime.db); + await processDueMediaUsageWork(runtime.db); expect(await countWork(runtime)).toBe(0); expect( - await new MediaUsageRepository(runtime.db).findSource( - canonicalSourceKey(fixture.collectionId, "entry-1"), - ), - ).not.toBeNull(); + await runtime.db.selectFrom("_emdash_media_usage_reconciliations").select("state").execute(), + ).toEqual([expect.objectContaining({ state: "pending" })]); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toMatchObject({ + state: "progress", + continuation: { kind: "immediate" }, + }); }); it("records a heartbeat from the Node timer maintenance callback", async () => { - const scheduler = new CapturingScheduler(); + const scheduler = new ContinuousCapturingScheduler(); runtime = await EmDashRuntime.create(createDeps(() => scheduler)); await scheduler.runMaintenance(); @@ -74,7 +228,7 @@ describe("media usage scheduled drivers", () => { }); it("does not fail Node maintenance when the heartbeat write fails", async () => { - const scheduler = new CapturingScheduler(); + const scheduler = new ContinuousCapturingScheduler(); runtime = await EmDashRuntime.create(createDeps(() => scheduler)); await new OptionsRepository(runtime.db).set( SCHEDULER_HEARTBEAT_OPTION, @@ -97,42 +251,117 @@ describe("media usage scheduled drivers", () => { ); }); - it("drains bounded work through a legacy Node scheduler cleanup callback", async () => { - const scheduler = new LegacyCapturingScheduler(); - runtime = await EmDashRuntime.create(createDeps(() => scheduler)); - const fixture = await activateCollection(runtime, "legacy_node_posts"); - await insertEntry(runtime, fixture.tableName, "entry-1"); - - await scheduler.runMaintenance(); + it("offers every due maintenance class one opportunity per cycle", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const work = await activateCollection(runtime, "fair_engine_work"); + await insertEntry(runtime, work.tableName, "entry-1"); + const deletion = await activateCollection(runtime, "fair_engine_delete"); + await runtime.schemaRegistry.deleteCollection("fair_engine_delete", { force: true }); + const reconciliation = await activateCollection(runtime, "fair_engine_reconciliation"); + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", reconciliation_required: 1 }) + .where("collection_id", "=", reconciliation.collectionId) + .execute(); + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toMatchObject({ + state: "progress", + continuation: { kind: "immediate" }, + }); expect(await countWork(runtime)).toBe(0); + expect(await deletionPhase(runtime, deletion.collectionId)).toBe("sources"); expect( - await new MediaUsageRepository(runtime.db).findSource( - canonicalSourceKey(fixture.collectionId, "entry-1"), - ), - ).not.toBeNull(); + await runtime.db + .selectFrom("_emdash_media_usage_reconciliations") + .select("collection_id") + .where("collection_id", "=", reconciliation.collectionId) + .executeTakeFirst(), + ).toBeDefined(); }); - it("advances bounded collection deletion from the Cloudflare scheduled entry point", async () => { + it("finishes post-Ready collection deletion before declaring Ready again", async () => { runtime = await EmDashRuntime.create(createDeps(null)); - const fixture = await activateCollection(runtime, "cloudflare_delete"); - await runtime.schemaRegistry.deleteCollection("cloudflare_delete", { force: true }); - await runtime.runScheduledMediaUsageTasks(); - await runtime.runScheduledMediaUsageTasks(); + const fixture = await activateCollection(runtime, "reusable_posts"); + await insertEntry(runtime, fixture.tableName, "entry-1"); + await processDueMediaUsageWork(runtime.db); + + await runtime.schemaRegistry.deleteCollection("reusable_posts", { force: true }); + expect(await deletionPhase(runtime, fixture.collectionId)).toBe("work"); + await expect(new MediaUsageRepository(runtime.db).findCollectionProgress()).resolves.toEqual({ + status: "indexing", + readyCollections: 0, + totalCollections: 0, + }); - expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); + for (let step = 0; step < 10 && (await deletionPhase(runtime, fixture.collectionId)); step++) { + await runMediaUsageMaintenanceStep(runtime.db); + } + + expect(await deletionPhase(runtime, fixture.collectionId)).toBeNull(); + await expect( + runtime.schemaRegistry.createCollection({ slug: "reusable_posts", label: "Reusable posts" }), + ).resolves.toMatchObject({ slug: "reusable_posts" }); }); - it("advances bounded collection deletion from the Node maintenance callback", async () => { - const scheduler = new CapturingScheduler(); - runtime = await EmDashRuntime.create(createDeps(() => scheduler)); - const fixture = await activateCollection(runtime, "node_delete"); - await runtime.schemaRegistry.deleteCollection("node_delete", { force: true }); + it("resumes historical reconciliation after failed entry work is retried", async () => { + runtime = await EmDashRuntime.create(createDeps(null)); + const fixture = await activateCollection(runtime, "retry_posts"); + await insertEntry(runtime, fixture.tableName, "entry-1"); + await sql`DELETE FROM ${sql.ref(fixture.tableName)} WHERE id = 'entry-1'`.execute(runtime.db); + await runtime.db + .updateTable("_emdash_media_usage_work") + .set({ state: "failed", attempt_count: 5, last_error_code: "MEDIA_USAGE_PROCESSING_FAILED" }) + .where("collection_id", "=", fixture.collectionId) + .execute(); + await runtime.db + .updateTable("_emdash_media_usage_index_status") + .set({ + status: "running", + completed_at: null, + cursor: "failed-run", + change_epoch: 1, + reconciliation_required: 1, + last_error_code: "MEDIA_USAGE_PROCESSING_FAILED", + }) + .where("collection_id", "=", fixture.collectionId) + .execute(); + await runtime.db + .insertInto("_emdash_media_usage_reconciliations") + .values({ + collection_id: fixture.collectionId, + collection_slug: "retry_posts", + run_token: "failed-run", + target_epoch: 1, + state: "failed", + phase: "sources", + attempt_count: 5, + last_error_code: "MEDIA_USAGE_RECONCILIATION_ENTRY_FAILED", + next_attempt_at: "2000-01-01T00:00:00.000Z", + }) + .execute(); - await scheduler.runMaintenance(); - await scheduler.runMaintenance(); + await expect( + new MediaUsageWorkRepository(runtime.db).retryOperatorWork({ + collectionId: fixture.collectionId, + contentId: "entry-1", + }), + ).resolves.toMatchObject({ outcome: "pending", changed: true }); + await expect( + new MediaUsageRepository(runtime.db).findCollectionProgress(), + ).resolves.toMatchObject({ + status: "indexing", + }); - expect(await deletionPhase(runtime, fixture.collectionId)).toBe("sources"); + for (let step = 0; step < 12; step++) { + const progress = await new MediaUsageRepository(runtime.db).findCollectionProgress(); + if (progress?.status === "ready") break; + await runMediaUsageMaintenanceStep(runtime.db); + } + await expect( + new MediaUsageRepository(runtime.db).findCollectionProgress(), + ).resolves.toMatchObject({ + status: "ready", + }); }); it("processes a trigger-created job before returning from an authenticated write", async () => { @@ -156,121 +385,85 @@ describe("media usage scheduled drivers", () => { ).not.toBeNull(); }); - it("persists a fair entry, deletion, reconciliation turn sequence", async () => { + it("continues confirmed activation one bounded collection at a time", async () => { runtime = await EmDashRuntime.create(createDeps(null)); - const fixture = await activateCollection(runtime, "fair_posts"); - await runtime.db - .updateTable("_emdash_media_usage_index_status") - .set({ status: "stale", reconciliation_required: 1 }) - .where("collection_id", "=", fixture.collectionId) - .execute(); - - const classes = []; - for (let index = 0; index < 3; index++) { - const result = await runtime.runScheduledMediaUsageTasks(); - classes.push(result.taskClass); - } - expect(classes).toEqual(["entry_work", "collection_deletion", "reconciliation"]); - expect( - await runtime.db - .selectFrom("_emdash_media_usage_reconciliations") - .select("collection_id") - .where("collection_id", "=", fixture.collectionId) - .executeTakeFirst(), - ).toBeDefined(); - }); + await configureExistingInactiveSite(runtime); + await runtime.schemaRegistry.createCollection({ slug: "activation_alpha", label: "Alpha" }); + await runtime.schemaRegistry.createCollection({ slug: "activation_beta", label: "Beta" }); + + await expect(activateMediaUsageCapture(runtime.db, { writersDrained: true })).resolves.toEqual({ + outcome: "activating", + processedCollections: 1, + collectionCursor: "activation_alpha", + }); + const confirmed = await activationState(runtime); - it("does not advance the turn or spend class queries before activation", async () => { - runtime = await EmDashRuntime.create(createDeps(null)); - const before = await maintenanceTurn(runtime); - await expect(runtime.runScheduledMediaUsageTasks()).resolves.toEqual({ - outcome: "inactive", - taskClass: null, - turn: null, + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "progress", + continuation: { kind: "immediate" }, }); - expect(await maintenanceTurn(runtime)).toBe(before); + expect(await activationState(runtime)).toEqual( + expect.objectContaining({ + state: "active", + drain_confirmed_at: confirmed.drain_confirmed_at, + attempt_count: confirmed.attempt_count + 1, + last_error_code: null, + }), + ); }); - it("reserves ten queries of headroom before changing the persisted turn", async () => { + it("does not automatically retry a stored activation failure", async () => { runtime = await EmDashRuntime.create(createDeps(null)); await runtime.db .updateTable("_emdash_media_usage_activation") - .set({ state: "active" }) - .where("task_key", "=", "incremental_capture") + .set({ + state: "activating", + drain_confirmed_at: "2026-08-18T12:00:00.000Z", + last_error_code: "MEDIA_USAGE_ACTIVATION_FAILED", + }) .execute(); - const before = await maintenanceTurn(runtime); - const metrics = createRequestMetrics(performance.now()); - metrics.dbCount = - MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling - - MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.maxClassQueries; - - const result = await runWithContext({ editMode: false, metrics }, () => - runtime!.runScheduledMediaUsageTasks(), + const before = await activationState(runtime); + + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "inactive", + continuation: { kind: "none" }, + }); + expect(await activationState(runtime)).toEqual(before); + + await expect(activateMediaUsageCapture(runtime.db, { writersDrained: true })).resolves.toEqual({ + outcome: "active", + processedCollections: 0, + }); + expect(await activationState(runtime)).toEqual( + expect.objectContaining({ + state: "active", + attempt_count: before.attempt_count + 1, + last_error_code: null, + }), ); - expect(result).toEqual({ outcome: "admission_closed", taskClass: null, turn: null }); - expect(await maintenanceTurn(runtime)).toBe(before); }); - it("measures every mutation class within its exported event reservation", async () => { + it("rejects an incompatible active generation before advancing maintenance", async () => { runtime = await EmDashRuntime.create(createDeps(null)); - const work = await activateCollection(runtime, "measure_work"); - await insertEntry(runtime, work.tableName, "entry-1"); - await activateCollection(runtime, "measure_delete"); - await runtime.schemaRegistry.deleteCollection("measure_delete", { force: true }); - const reconciliation = await activateCollection(runtime, "measure_reconcile"); await runtime.db - .updateTable("_emdash_media_usage_index_status") - .set({ status: "stale", reconciliation_required: 1 }) - .where("collection_id", "=", reconciliation.collectionId) + .updateTable("_emdash_media_usage_activation") + .set({ state: "active", runtime_generation: 2 }) .execute(); + const before = await activationState(runtime); - const expected = [ - ["entry_work", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.entryWork], - ["collection_deletion", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.collectionDeletion], - ["reconciliation", MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.reconciliation], - ] as const; - for (const [taskClass, reservation] of expected) { - const metrics = createRequestMetrics(performance.now()); - const result = await runWithContext({ editMode: false, metrics }, () => - runtime!.runScheduledMediaUsageTasks(), - ); - expect(result.taskClass).toBe(taskClass); - expect(metrics.dbCount).toBeLessThanOrEqual(1 + reservation); - expect(metrics.dbCount).toBeLessThanOrEqual( - MEDIA_USAGE_MAINTENANCE_QUERY_RESERVATIONS.eventCeiling, - ); - } + await expect(runMediaUsageMaintenanceStep(runtime.db)).resolves.toEqual({ + state: "inactive", + continuation: { kind: "none" }, + }); + expect(await activationState(runtime)).toEqual(before); }); }); -class CapturingScheduler implements CronScheduler { - private maintenance: SystemCleanupFn | null = null; - private mediaUsageMaintenance: SystemCleanupFn | null = null; - - setSystemCleanup(fn: SystemCleanupFn): void { - this.maintenance = fn; - } - setMediaUsageMaintenance(fn: SystemCleanupFn): void { - this.mediaUsageMaintenance = fn; - } - - start(): void {} - stop(): void {} - reschedule(): void {} - - async runMaintenance(): Promise { - if (!this.maintenance) throw new Error("Expected Node maintenance callback"); - await this.maintenance(); - if (!this.mediaUsageMaintenance) throw new Error("Expected Media Usage maintenance callback"); - await this.mediaUsageMaintenance(); - } -} - -class LegacyCapturingScheduler implements CronScheduler { - private maintenance: SystemCleanupFn | null = null; +class ContinuousCapturingScheduler implements CronScheduler { + private systemCleanup: SystemCleanupFn | null = null; setSystemCleanup(fn: SystemCleanupFn): void { - this.maintenance = fn; + this.systemCleanup = fn; } start(): void {} @@ -278,8 +471,8 @@ class LegacyCapturingScheduler implements CronScheduler { reschedule(): void {} async runMaintenance(): Promise { - if (!this.maintenance) throw new Error("Expected Node maintenance callback"); - await this.maintenance(); + if (!this.systemCleanup) throw new Error("Expected Node maintenance callback"); + await this.systemCleanup(); } } @@ -370,13 +563,31 @@ async function deletionPhase(runtime: EmDashRuntime, collectionId: string): Prom return row?.phase ?? null; } -async function maintenanceTurn(runtime: EmDashRuntime): Promise { - const row = await runtime.db +function activationState(runtime: EmDashRuntime) { + return runtime.db .selectFrom("_emdash_media_usage_activation") - .select("media_usage_maintenance_turn") + .selectAll() .where("task_key", "=", "incremental_capture") .executeTakeFirstOrThrow(); - return row.media_usage_maintenance_turn; +} + +async function configureExistingInactiveSite(runtime: EmDashRuntime): Promise { + await new OptionsRepository(runtime.db).set("emdash:setup_complete", true); + await runtime.db + .updateTable("_emdash_media_usage_activation") + .set({ + state: "expanded", + collection_cursor: null, + drain_confirmed_at: null, + lease_token: null, + lease_expires_at: null, + attempt_count: 0, + last_attempted_at: null, + last_error_code: null, + activated_at: null, + }) + .where("task_key", "=", "incremental_capture") + .execute(); } function canonicalSourceKey(collectionId: string, contentId: string): string { diff --git a/packages/core/tests/unit/api/media-usage-activation-route.test.ts b/packages/core/tests/unit/api/media-usage-activation-route.test.ts index 2b0bf03bd2..948dee81d7 100644 --- a/packages/core/tests/unit/api/media-usage-activation-route.test.ts +++ b/packages/core/tests/unit/api/media-usage-activation-route.test.ts @@ -49,7 +49,7 @@ describe("admin media usage activation status route", () => { "INSUFFICIENT_SCOPE", ); - const post = () => activationPost({ writersDrained: true, maintenanceReady: true }); + const post = () => activationPost({ writersDrained: true }); await expectError(await POST(routeContext(post(), null)), 401, "UNAUTHORIZED"); await expectError(await POST(routeContext(post(), Role.EDITOR)), 403, "FORBIDDEN"); await expectError( @@ -118,9 +118,7 @@ describe("admin media usage activation status route", () => { "MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH", ); await expectError( - await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ), + await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)), 409, "MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH", ); @@ -159,12 +157,7 @@ describe("admin media usage activation status route", () => { it("requires both literal confirmations without mutating activation state", async () => { const before = await activationRow(); - for (const body of [ - {}, - { writersDrained: false, maintenanceReady: true }, - { writersDrained: true, maintenanceReady: false }, - { writersDrained: true, maintenanceReady: true, extra: true }, - ]) { + for (const body of [{}, { writersDrained: false }, { writersDrained: true, extra: true }]) { await expectError( await POST(routeContext(activationPost(body), Role.ADMIN)), 400, @@ -176,9 +169,7 @@ describe("admin media usage activation status route", () => { it("advances exactly one collection per confirmed request and is idempotent when active", async () => { const first = await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN, [ - "admin", - ]), + routeContext(activationPost({ writersDrained: true }), Role.ADMIN, ["admin"]), ); expect(first.status).toBe(200); expect(await first.json()).toEqual({ @@ -189,10 +180,7 @@ describe("admin media usage activation status route", () => { activation: expect.objectContaining({ state: "activating" }), }, }); - - const second = await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ); + const second = await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)); expect(second.status).toBe(200); expect(await second.json()).toEqual({ success: true, @@ -203,9 +191,7 @@ describe("admin media usage activation status route", () => { }, }); - const third = await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ); + const third = await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)); expect(await third.json()).toEqual({ success: true, data: { @@ -227,9 +213,7 @@ describe("admin media usage activation status route", () => { .where("task_key", "=", "incremental_capture") .execute(); - const response = await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ); + const response = await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)); expect(response.status).toBe(409); const body = await response.json(); expect(body).toEqual({ @@ -257,9 +241,7 @@ describe("admin media usage activation status route", () => { `.execute(ctx!.db); await expectError( - await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ), + await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)), 409, "MEDIA_USAGE_ACTIVATION_CONFLICT", ); @@ -268,9 +250,7 @@ describe("admin media usage activation status route", () => { it("records trigger failures while returning only a stable public error", async () => { await sql`DROP TABLE ${sql.ref("ec_page")}`.execute(ctx!.db); - const response = await POST( - routeContext(activationPost({ writersDrained: true, maintenanceReady: true }), Role.ADMIN), - ); + const response = await POST(routeContext(activationPost({ writersDrained: true }), Role.ADMIN)); expect(response.status).toBe(500); const body = await response.json(); expect(body).toEqual({ diff --git a/packages/core/tests/unit/api/media-usage-progress-route.test.ts b/packages/core/tests/unit/api/media-usage-progress-route.test.ts new file mode 100644 index 0000000000..b0e96d9b93 --- /dev/null +++ b/packages/core/tests/unit/api/media-usage-progress-route.test.ts @@ -0,0 +1,323 @@ +import { Role, type RoleLevel } from "@emdash-cms/auth"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { injectCoreRoutes } from "../../../src/astro/integration/routes.js"; +import { GET, POST } from "../../../src/astro/routes/api/admin/media-usage/progress.js"; +import * as maintenanceEngine from "../../../src/media/usage/maintenance-engine.js"; +import { + setupForDialectWithCollections, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +type RouteContext = Parameters[0]; + +describe("admin media usage progress route", () => { + let ctx: DialectTestContext | undefined; + let collectionId: string; + + beforeEach(async () => { + ctx = await setupForDialectWithCollections("sqlite"); + const collections = await ctx.db + .selectFrom("_emdash_collections") + .select(["id", "slug"]) + .execute(); + collectionId = collections.find(({ slug }) => slug === "post")!.id; + for (const collection of collections) { + await ctx.db + .updateTable("_emdash_media_usage_index_status") + .set({ + collection_id: collection.id, + capture_state: "active", + status: "complete", + schema_version: 1, + reconciliation_required: 0, + }) + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", collection.slug) + .execute(); + } + await ctx.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await teardownForDialect(ctx); + ctx = undefined; + }); + + it("registers the progress route", () => { + const routes: Array<{ pattern: string; entrypoint: string }> = []; + injectCoreRoutes((route) => routes.push(route)); + + expect(routes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + pattern: "/_emdash/api/admin/media-usage/progress", + entrypoint: expect.stringContaining("api/admin/media-usage/progress"), + }), + ]), + ); + }); + + it("requires authentication, schema permission, and admin token scope", async () => { + await expectError(await GET(routeContext(null)), 401, "UNAUTHORIZED"); + await expectError(await GET(routeContext(Role.EDITOR)), 403, "FORBIDDEN"); + await expectError( + await GET(routeContext(Role.ADMIN, ["content:read"])), + 403, + "INSUFFICIENT_SCOPE", + ); + }); + + it("requires authentication, schema permission, and admin token scope to advance", async () => { + await expectError(await POST(routeContext(null, undefined, "POST")), 401, "UNAUTHORIZED"); + await expectError(await POST(routeContext(Role.EDITOR, undefined, "POST")), 403, "FORBIDDEN"); + await expectError( + await POST(routeContext(Role.ADMIN, ["content:read"], "POST")), + 403, + "INSUFFICIENT_SCOPE", + ); + }); + + it("runs one maintenance step and returns the stored state after it", async () => { + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); + const body = await response.json(); + expect(body).toEqual({ + success: true, + data: { + activation: expect.objectContaining({ state: "active" }), + progress: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + nextRequestInMs: null, + }, + }); + expect(JSON.stringify(body)).not.toContain(collectionId); + expect(JSON.stringify(body)).not.toContain("post"); + }); + + it("returns immediate continuation after exactly one activation step", async () => { + await ctx!.db + .updateTable("_emdash_media_usage_activation") + .set({ + state: "activating", + drain_confirmed_at: "2026-08-24T00:00:00.000Z", + activated_at: null, + lease_token: null, + lease_expires_at: null, + }) + .where("task_key", "=", "incremental_capture") + .execute(); + + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + const body = await response.json(); + + expect(body).toEqual({ + success: true, + data: { + activation: expect.objectContaining({ state: "activating", collectionCursor: "page" }), + progress: null, + nextRequestInMs: 0, + }, + }); + }); + + it("returns delayed continuation while another activation lease is live", async () => { + await ctx!.db + .updateTable("_emdash_media_usage_activation") + .set({ + state: "activating", + drain_confirmed_at: "2026-08-24T00:00:00.000Z", + activated_at: null, + lease_token: "other-owner", + lease_expires_at: "2999-01-01T00:00:00.000Z", + }) + .where("task_key", "=", "incremental_capture") + .execute(); + + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + const body = await response.json(); + + expect(body).toEqual({ + success: true, + data: { + activation: expect.objectContaining({ state: "activating" }), + progress: null, + nextRequestInMs: 30_000, + }, + }); + }); + + it("continues when durable progress becomes incomplete after an idle step", async () => { + vi.spyOn(maintenanceEngine, "runMediaUsageMaintenanceStep").mockResolvedValue({ + state: "idle", + continuation: { kind: "none" }, + }); + await ctx!.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "stale", schema_version: 0, reconciliation_required: 1 }) + .where("collection_id", "=", collectionId) + .execute(); + + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + const body = await response.json(); + + expect(body).toEqual({ + success: true, + data: { + activation: expect.objectContaining({ state: "active" }), + progress: expect.objectContaining({ status: "indexing" }), + nextRequestInMs: 0, + }, + }); + }); + + it("stops when stored progress needs attention", async () => { + vi.spyOn(maintenanceEngine, "runMediaUsageMaintenanceStep").mockResolvedValue({ + state: "progress", + continuation: { kind: "immediate" }, + }); + await ctx!.db + .updateTable("_emdash_media_usage_index_status") + .set({ status: "failed", last_error_code: "MEDIA_USAGE_PROCESSING_FAILED" }) + .where("collection_id", "=", collectionId) + .execute(); + + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + const body = await response.json(); + + expect(body).toEqual({ + success: true, + data: { + activation: expect.objectContaining({ state: "active" }), + progress: expect.objectContaining({ status: "needs_attention" }), + nextRequestInMs: null, + }, + }); + }); + + it("rejects untouched activation without changing it", async () => { + await ctx!.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "expanded" }) + .where("task_key", "=", "incremental_capture") + .execute(); + + await expectError( + await POST(routeContext(Role.ADMIN, ["admin"], "POST")), + 409, + "MEDIA_USAGE_PROGRESS_NOT_ACTIVE", + ); + expect( + await ctx!.db + .selectFrom("_emdash_media_usage_activation") + .select("state") + .where("task_key", "=", "incremental_capture") + .executeTakeFirstOrThrow(), + ).toEqual({ state: "expanded" }); + }); + + it("returns aggregate readiness without exposing collection or work details", async () => { + const response = await GET(routeContext(Role.ADMIN, ["admin"])); + + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); + const body = await response.json(); + expect(body).toEqual({ + success: true, + data: { + status: "ready", + readyCollections: 2, + totalCollections: 2, + }, + }); + expect(JSON.stringify(body)).not.toContain(collectionId); + expect(JSON.stringify(body)).not.toContain("post"); + }); + + it("rejects progress reads before activation is active", async () => { + await ctx!.db + .updateTable("_emdash_media_usage_activation") + .set({ state: "activating" }) + .where("task_key", "=", "incremental_capture") + .execute(); + + await expectError( + await GET(routeContext(Role.ADMIN, ["admin"])), + 409, + "MEDIA_USAGE_PROGRESS_NOT_ACTIVE", + ); + }); + + it("rejects an incompatible activation runtime generation", async () => { + await ctx!.db + .updateTable("_emdash_media_usage_activation") + .set({ runtime_generation: 2 }) + .where("task_key", "=", "incremental_capture") + .execute(); + + await expectError( + await GET(routeContext(Role.ADMIN, ["admin"])), + 409, + "MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH", + ); + await expectError( + await POST(routeContext(Role.ADMIN, ["admin"], "POST")), + 409, + "MEDIA_USAGE_ACTIVATION_VERSION_MISMATCH", + ); + }); + + it("returns a stable redacted read error", async () => { + await ctx!.db.schema.dropTable("_emdash_media_usage_index_status").execute(); + + const response = await GET(routeContext(Role.ADMIN, ["admin"])); + const body = await response.clone().json(); + + await expectError(response, 500, "MEDIA_USAGE_PROGRESS_READ_ERROR"); + expect(JSON.stringify(body)).not.toContain("_emdash_media_usage_index_status"); + }); + + it("returns a stable redacted advance error", async () => { + await ctx!.db.schema.dropTable("_emdash_media_usage_activation").execute(); + + const response = await POST(routeContext(Role.ADMIN, ["admin"], "POST")); + const body = await response.clone().json(); + + await expectError(response, 500, "MEDIA_USAGE_PROGRESS_ADVANCE_ERROR"); + expect(JSON.stringify(body)).not.toContain("_emdash_media_usage_activation"); + }); + + function routeContext( + role: RoleLevel | null, + tokenScopes?: string[], + method: "GET" | "POST" = "GET", + ): RouteContext { + return { + request: new Request("http://localhost/_emdash/api/admin/media-usage/progress", { method }), + locals: { + emdash: { db: ctx!.db }, + user: role == null ? null : { id: "user-1", role }, + tokenScopes, + }, + } as RouteContext; + } +}); + +async function expectError(response: Response, status: number, code: string): Promise { + expect(response.status).toBe(status); + const body = (await response.json()) as { error: { code: string } }; + expect(body.error.code).toBe(code); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); +} diff --git a/packages/core/tests/unit/api/openapi.test.ts b/packages/core/tests/unit/api/openapi.test.ts index 08347fe11c..f5780b1773 100644 --- a/packages/core/tests/unit/api/openapi.test.ts +++ b/packages/core/tests/unit/api/openapi.test.ts @@ -190,6 +190,7 @@ describe("OpenAPI document generation", () => { }; const post = path?.post as { operationId?: string; + description?: string; requestBody?: unknown; responses?: Record; }; @@ -206,6 +207,8 @@ describe("OpenAPI document generation", () => { ); expect(JSON.stringify(get.responses?.["200"])).toContain("MediaUsageActivationStatus"); expect(post.operationId).toBe("advanceMediaUsageActivation"); + expect(post.description).toContain("progress endpoint"); + expect(post.description).not.toContain("automatic maintenance"); expect(post.requestBody).toBeDefined(); expect(post.responses).toEqual( expect.objectContaining({ @@ -219,6 +222,41 @@ describe("OpenAPI document generation", () => { ); }); + it("documents aggregate media usage indexing progress", () => { + const doc = generateOpenApiDocument(); + const path = doc.paths?.["/_emdash/api/admin/media-usage/progress"]; + const get = path?.get as + | { operationId?: string; responses?: Record } + | undefined; + const post = path?.post as + | { operationId?: string; requestBody?: unknown; responses?: Record } + | undefined; + + expect(get?.operationId).toBe("getMediaUsageProgress"); + expect(get?.responses).toEqual( + expect.objectContaining({ + "200": expect.any(Object), + "401": expect.any(Object), + "403": expect.any(Object), + "409": expect.any(Object), + "500": expect.any(Object), + }), + ); + expect(JSON.stringify(get?.responses?.["200"])).toContain("MediaUsageProgress"); + expect(post?.operationId).toBe("advanceMediaUsageProgress"); + expect(post?.requestBody).toBeUndefined(); + expect(post?.responses).toEqual( + expect.objectContaining({ + "200": expect.any(Object), + "401": expect.any(Object), + "403": expect.any(Object), + "409": expect.any(Object), + "500": expect.any(Object), + }), + ); + expect(JSON.stringify(post?.responses?.["200"])).toContain("MediaUsageProgressAdvanceResponse"); + }); + it("includes schema paths", () => { const doc = generateOpenApiDocument(); const paths = Object.keys(doc.paths ?? {}); diff --git a/packages/core/tests/unit/astro/middleware-prerender.test.ts b/packages/core/tests/unit/astro/middleware-prerender.test.ts index 47808a9c59..e1c41650ee 100644 --- a/packages/core/tests/unit/astro/middleware-prerender.test.ts +++ b/packages/core/tests/unit/astro/middleware-prerender.test.ts @@ -36,7 +36,6 @@ const { return null; }); const handlePluginApiRoute = vi.fn(async () => publicPluginResult); - return { MOCK_RUNTIME: { storage: { getPublicUrl }, diff --git a/packages/core/tests/unit/client/client.test.ts b/packages/core/tests/unit/client/client.test.ts index c17c639dce..845fe8276c 100644 --- a/packages/core/tests/unit/client/client.test.ts +++ b/packages/core/tests/unit/client/client.test.ts @@ -1137,6 +1137,67 @@ describe("EmDashClient", () => { }); describe("media usage work operators", () => { + it("reads aggregate media usage indexing progress", async () => { + let capturedRequest: Request | undefined; + const progress = { + status: "indexing" as const, + readyCollections: 2, + totalCollections: 3, + }; + const client = new EmDashClient({ + baseUrl: "http://localhost:4321", + token: "test", + interceptors: [ + async (request) => { + capturedRequest = request; + return jsonResponse(progress); + }, + ], + }); + await expect(client.mediaGetUsageProgress()).resolves.toEqual(progress); + expect(capturedRequest?.method).toBe("GET"); + expect(new URL(capturedRequest!.url).pathname).toBe( + "/_emdash/api/admin/media-usage/progress", + ); + }); + + it("advances one media usage progress step without a request body", async () => { + let capturedRequest: Request | undefined; + const result = { + activation: { + state: "active" as const, + collectionCursor: null, + attemptCount: 1, + drainConfirmedAt: "2026-08-12T09:00:00.000Z", + lastAttemptedAt: "2026-08-12T09:00:00.000Z", + lastErrorCode: null, + leaseExpiresAt: null, + activatedAt: "2026-08-12T09:00:01.000Z", + updatedAt: "2026-08-12T09:00:01.000Z", + }, + progress: { status: "ready" as const, readyCollections: 2, totalCollections: 2 }, + nextRequestInMs: null, + }; + const client = new EmDashClient({ + baseUrl: "http://localhost:4321", + token: "test", + interceptors: [ + async (request) => { + capturedRequest = request; + return jsonResponse(result); + }, + ], + }); + + await expect(client.mediaAdvanceUsageProgress()).resolves.toEqual(result); + expect(capturedRequest?.method).toBe("POST"); + expect(capturedRequest?.headers.get("X-EmDash-Request")).toBe("1"); + expect(capturedRequest?.body).toBeNull(); + expect(new URL(capturedRequest!.url).pathname).toBe( + "/_emdash/api/admin/media-usage/progress", + ); + }); + it("reads the redacted activation status", async () => { let capturedRequest: Request | undefined; const status = { @@ -1168,7 +1229,7 @@ describe("EmDashClient", () => { ); }); - it("advances one activation batch with both explicit confirmations", async () => { + it("advances one activation batch after writer confirmation", async () => { let capturedBody: unknown; const client = new EmDashClient({ baseUrl: "http://localhost:4321", @@ -1199,9 +1260,8 @@ describe("EmDashClient", () => { await client.mediaAdvanceUsageActivation({ writersDrained: true, - maintenanceReady: true, }); - expect(capturedBody).toEqual({ writersDrained: true, maintenanceReady: true }); + expect(capturedBody).toEqual({ writersDrained: true }); }); it("serializes a bounded work-list query and returns the cursor page", async () => { diff --git a/packages/core/tests/unit/db/migration-executors.test.ts b/packages/core/tests/unit/db/migration-executors.test.ts index 04e44ba091..9cec6aaa27 100644 --- a/packages/core/tests/unit/db/migration-executors.test.ts +++ b/packages/core/tests/unit/db/migration-executors.test.ts @@ -80,7 +80,7 @@ describe("SQLite migration executor", () => { const secondReport = await secondExecutor.execute(await migrationRequest("apply")); expect(secondReport.executed).toEqual([]); expect(secondReport.knownApplied).toEqual(MIGRATION_NAMES); - }); + }, 30_000); it("uses the explicit database override instead of the manifest path", async () => { const projectRoot = await temporaryDirectory(); @@ -119,7 +119,7 @@ describe("libSQL migration executor", () => { const secondReport = await secondExecutor.execute(await migrationRequest("apply")); expect(secondReport.executed).toEqual([]); expect(secondReport.knownApplied).toEqual(MIGRATION_NAMES); - }); + }, 30_000); it("fails on a missing token before constructing an executor", async () => { const projectRoot = await temporaryDirectory(); diff --git a/packages/core/tests/unit/plugins/node-cron-scheduler.test.ts b/packages/core/tests/unit/plugins/node-cron-scheduler.test.ts new file mode 100644 index 0000000000..9c0ee595ff --- /dev/null +++ b/packages/core/tests/unit/plugins/node-cron-scheduler.test.ts @@ -0,0 +1,110 @@ +import type { Kysely } from "kysely"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { Database } from "../../../src/database/types.js"; +import { CronExecutor } from "../../../src/plugins/cron.js"; +import { NodeCronScheduler } from "../../../src/plugins/scheduler/node.js"; +import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +describe("NodeCronScheduler", () => { + let db: Kysely; + let executor: CronExecutor; + let scheduler: NodeCronScheduler; + + beforeEach(async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-17T12:00:00.000Z")); + db = await setupTestDatabase(); + executor = new CronExecutor(db, async () => {}); + vi.spyOn(executor, "getNextDueTime").mockResolvedValue(null); + vi.spyOn(executor, "tick").mockResolvedValue(0); + vi.spyOn(executor, "recoverStaleLocks").mockResolvedValue(0); + scheduler = new NodeCronScheduler(executor); + }); + + afterEach(async () => { + scheduler.stop(); + vi.useRealTimers(); + await teardownTestDatabase(db); + vi.restoreAllMocks(); + }); + + it("runs the general tick, stale-lock recovery, and cleanup together", async () => { + const cleanup = vi.fn(async () => {}); + vi.mocked(executor.getNextDueTime).mockResolvedValue(new Date(Date.now()).toISOString()); + scheduler.setSystemCleanup(cleanup); + scheduler.start(); + + await vi.advanceTimersByTimeAsync(1_000); + + expect(executor.tick).toHaveBeenCalledOnce(); + expect(executor.recoverStaleLocks).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("uses the one-minute heartbeat when no task is due", async () => { + scheduler.start(); + + await vi.advanceTimersByTimeAsync(59_999); + expect(executor.tick).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(executor.tick).toHaveBeenCalledOnce(); + }); + + it("reschedules from a newly due task", async () => { + scheduler.start(); + await vi.advanceTimersByTimeAsync(0); + vi.mocked(executor.getNextDueTime).mockResolvedValue( + new Date(Date.now() + 2_000).toISOString(), + ); + + scheduler.reschedule(); + await vi.advanceTimersByTimeAsync(1_999); + expect(executor.tick).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(executor.tick).toHaveBeenCalledOnce(); + }); + + it("cancels the pending heartbeat when stopped", async () => { + scheduler.start(); + await vi.advanceTimersByTimeAsync(0); + + scheduler.stop(); + await vi.runAllTimersAsync(); + + expect(executor.tick).not.toHaveBeenCalled(); + }); + + it("unrefs the heartbeat timer", async () => { + const timeout = vi.spyOn(globalThis, "setTimeout"); + scheduler.start(); + await vi.advanceTimersByTimeAsync(0); + + const heartbeat = timeout.mock.results.find( + (result, index) => + timeout.mock.calls[index]?.[1] === 60_000 && isUnreferencedTimer(result.value), + ); + expect(heartbeat).toBeDefined(); + }); + + it("logs failed heartbeat tasks and schedules the next heartbeat", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(executor.tick).mockRejectedValueOnce(new Error("tick failed")); + vi.mocked(executor.recoverStaleLocks).mockRejectedValueOnce(new Error("recovery failed")); + scheduler.setSystemCleanup(async () => { + throw new Error("cleanup failed"); + }); + scheduler.start(); + + await vi.advanceTimersByTimeAsync(60_000); + + expect(error).toHaveBeenCalledTimes(3); + expect(executor.getNextDueTime).toHaveBeenCalledTimes(2); + }); +}); + +function isUnreferencedTimer(value: unknown): boolean { + if (!value || typeof value !== "object" || !("hasRef" in value)) return false; + const hasRef = value.hasRef; + return typeof hasRef === "function" && hasRef.call(value) === false; +} diff --git a/packages/core/tests/workerd/media-usage-activation-d1.test.ts b/packages/core/tests/workerd/media-usage-activation-d1.test.ts index 2fac11f356..9bfa1e21ea 100644 --- a/packages/core/tests/workerd/media-usage-activation-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-activation-d1.test.ts @@ -5,6 +5,10 @@ import { afterAll, beforeAll, expect, it } from "vitest"; import { EmDashD1Dialect, RawBindingD1Dialect } from "../../../cloudflare/src/db/d1-dialect.js"; import { GET, POST } from "../../src/astro/routes/api/admin/media-usage/activation.js"; +import { + GET as GET_PROGRESS, + POST as POST_PROGRESS, +} from "../../src/astro/routes/api/admin/media-usage/progress.js"; import { runMigrations } from "../../src/database/migrations/runner.js"; import type { Database } from "../../src/database/types.js"; import { MEDIA_USAGE_ACTIVATION_RUNTIME_GENERATION } from "../../src/media/usage/activation.js"; @@ -58,6 +62,9 @@ it("keeps complete authenticated activation route costs within the D1 envelope", await record("get-activating", "session", GET, activationGet(), 200, evidence); await record("final-collection", "session", POST, activationPost(), 200, evidence); await record("get-active", "raw", GET, activationGet(), 200, evidence); + await record("progress-indexing", "session", GET_PROGRESS, progressGet(), 200, evidence); + expect(evidence.at(-1)).toEqual(expect.objectContaining({ queries: 1, rowsWritten: 0 })); + await record("progress-step", "session", POST_PROGRESS, progressPost(), 200, evidence); await record("active-idempotent", "session", POST, activationPost(), 200, evidence); await adminDb @@ -139,7 +146,18 @@ function activationPost(): Request { return new Request("http://localhost/_emdash/api/admin/media-usage/activation", { method: "POST", headers: { "Content-Type": "application/json", "X-EmDash-Request": "1" }, - body: JSON.stringify({ writersDrained: true, maintenanceReady: true }), + body: JSON.stringify({ writersDrained: true }), + }); +} + +function progressGet(): Request { + return new Request("http://localhost/_emdash/api/admin/media-usage/progress"); +} + +function progressPost(): Request { + return new Request("http://localhost/_emdash/api/admin/media-usage/progress", { + method: "POST", + headers: { "X-EmDash-Request": "1" }, }); } diff --git a/packages/core/tests/workerd/media-usage-maintenance-engine-d1.test.ts b/packages/core/tests/workerd/media-usage-maintenance-engine-d1.test.ts new file mode 100644 index 0000000000..eea7fd7b7b --- /dev/null +++ b/packages/core/tests/workerd/media-usage-maintenance-engine-d1.test.ts @@ -0,0 +1,214 @@ +import { env } from "cloudflare:test"; +import { Kysely, sql } from "kysely"; +import { afterAll, beforeAll, expect, it } from "vitest"; + +import { RawBindingD1Dialect } from "../../../cloudflare/src/db/d1-dialect.js"; +import { kyselyLogOption } from "../../src/database/instrumentation.js"; +import { runMigrations } from "../../src/database/migrations/runner.js"; +import { MediaUsageRepository } from "../../src/database/repositories/media-usage.js"; +import type { Database } from "../../src/database/types.js"; +import { + MEDIA_USAGE_MAINTENANCE_LIMITS, + runMediaUsageMaintenanceStep, +} from "../../src/media/usage/maintenance-engine.js"; +import { createRequestMetrics, runWithContext } from "../../src/request-context.js"; +import { SchemaRegistry } from "../../src/schema/registry.js"; +import { + createMediaUsageAdmissionFixture, + insertMediaUsageMeasurementEntry, + mediaUsageMeasurementData, +} from "../utils/media-usage-admission-fixture.js"; + +declare module "cloudflare:test" { + interface ProvidedEnv { + DB: D1Database; + } +} + +interface D1Measurement { + queries: number; + rowsRead: number; + rowsWritten: number; + durationMs: number; + wallDurationMs: number; + maxBinds: number; + maxSqlBytes: number; +} + +let adminDb: Kysely; + +beforeAll(async () => { + adminDb = new Kysely({ + dialect: new RawBindingD1Dialect({ database: env.DB }), + }); + await runMigrations(adminDb); +}); + +afterAll(async () => { + await adminDb.destroy(); +}); + +it("keeps full-repair source lookups within D1 value limits", async () => { + const collectionId = "collection-" + "x".repeat(40); + const sourceKeys = Array.from( + { length: 50_000 }, + (_, index) => `content:${collectionId}:${String(index).padStart(8, "0")}:draft_overlay`, + ); + + await expect(new MediaUsageRepository(adminDb).findSources(sourceKeys)).resolves.toEqual( + new Map(), + ); +}); + +it("keeps the largest entry step below the shared reservation", async () => { + const fixture = await createMediaUsageAdmissionFixture(adminDb, "d1_engine_boundary"); + await adminDb + .updateTable("_emdash_media_usage_activation") + .set({ state: "active" }) + .where("task_key", "=", "incremental_capture") + .execute(); + await insertMediaUsageMeasurementEntry( + adminDb, + fixture, + "boundary-entry", + mediaUsageMeasurementData(500, "boundary-entry"), + ); + + const measurement = emptyMeasurement(); + const db = new Kysely({ + dialect: new RawBindingD1Dialect({ database: captureD1(env.DB, measurement) }), + log: kyselyLogOption(), + }); + const startedAt = performance.now(); + const result = await runMediaUsageMaintenanceStep(db); + measurement.wallDurationMs = Number((performance.now() - startedAt).toFixed(3)); + await db.destroy(); + + expect(result).toEqual({ + state: "progress", + continuation: { kind: "immediate" }, + }); + expect(measurement.queries).toBeLessThanOrEqual(MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries); + expect(measurement.maxBinds).toBeLessThanOrEqual(100); + expect(measurement.maxSqlBytes).toBeLessThan(100 * 1024); + expect(measurement.wallDurationMs).toBeLessThan(5_000); + console.info(`PR8_D1_MAINTENANCE_STEP=${JSON.stringify(measurement)}`); +}); + +it("keeps the largest admitted activation trigger replacement inside one step", async () => { + await adminDb + .updateTable("_emdash_media_usage_activation") + .set({ + state: "expanded", + collection_cursor: null, + drain_confirmed_at: null, + lease_token: null, + lease_expires_at: null, + last_error_code: null, + }) + .where("task_key", "=", "incremental_capture") + .execute(); + const collection = await new SchemaRegistry(adminDb).createCollection({ + slug: "aaa_d1_activation_bound", + label: "Activation bound", + }); + for (let index = 0; index < 100; index++) { + const triggerName = `emdash_mu_bound_${String(index).padStart(3, "0")}`; + await sql` + CREATE TRIGGER ${sql.ref(triggerName)} + AFTER INSERT ON ${sql.ref("ec_aaa_d1_activation_bound")} + BEGIN + SELECT 1; + END + `.execute(adminDb); + } + await adminDb + .updateTable("_emdash_media_usage_activation") + .set({ + state: "activating", + drain_confirmed_at: "2026-08-18T12:00:00.000Z", + }) + .where("task_key", "=", "incremental_capture") + .execute(); + + const measurement = emptyMeasurement(); + const db = new Kysely({ + dialect: new RawBindingD1Dialect({ database: captureD1(env.DB, measurement) }), + log: kyselyLogOption(), + }); + const metrics = createRequestMetrics(performance.now()); + const result = await runWithContext({ editMode: false, metrics }, () => + runMediaUsageMaintenanceStep(db), + ); + const status = await adminDb + .selectFrom("_emdash_media_usage_index_status") + .select("capture_state") + .where("collection_id", "=", collection.id) + .executeTakeFirstOrThrow(); + await db.destroy(); + + expect(result).toEqual({ + state: "progress", + continuation: { kind: "immediate" }, + }); + expect(status.capture_state).toBe("active"); + expect(measurement.queries).toBeLessThanOrEqual(MEDIA_USAGE_MAINTENANCE_LIMITS.maxStepQueries); + expect(metrics.dbCount).toBe(measurement.queries); +}); + +function emptyMeasurement(): D1Measurement { + return { + queries: 0, + rowsRead: 0, + rowsWritten: 0, + durationMs: 0, + wallDurationMs: 0, + maxBinds: 0, + maxSqlBytes: 0, + }; +} + +function captureD1(database: D1Database, measurement: D1Measurement): D1Database { + return new Proxy(database, { + get(target, property) { + if (property === "prepare") { + return (query: string) => captureStatement(target.prepare(query), query, [], measurement); + } + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +function captureStatement( + statement: D1PreparedStatement, + query: string, + binds: unknown[], + measurement: D1Measurement, +): D1PreparedStatement { + return new Proxy(statement, { + get(target, property) { + if (property === "bind") { + return (...values: unknown[]) => + captureStatement(target.bind(...values), query, values, measurement); + } + if (property === "all") { + return async () => { + measurement.queries++; + measurement.maxBinds = Math.max(measurement.maxBinds, binds.length); + measurement.maxSqlBytes = Math.max( + measurement.maxSqlBytes, + new TextEncoder().encode(query).byteLength, + ); + const result = await target.all(); + measurement.rowsRead += result.meta.rows_read; + measurement.rowsWritten += result.meta.rows_written; + measurement.durationMs += result.meta.duration; + return result; + }; + } + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} diff --git a/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts b/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts index 6997e707a9..bf618f38c0 100644 --- a/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts +++ b/packages/core/tests/workerd/media-usage-projection-admission-d1.test.ts @@ -60,14 +60,13 @@ it("reports real D1 metadata for the approved full runtime boundary", async () = const measurement = await measure(() => runtime.handleContentUpdate(fixture.collectionSlug, created.data.item.id, { - data: mediaUsageMeasurementData(12, "boundary-d1-draft"), + data: mediaUsageMeasurementData(500, "boundary-d1-draft"), }), ); expect(measurement.value.success).toBe(true); - expect(measurement.d1Queries).toBeLessThanOrEqual(40); - // The collection/source cleanup cursors add one index write for this source - // and each of its 12 occurrences; keep the bound exact so further growth fails. - expect(measurement.rowsWritten).toBeLessThanOrEqual(135); + expect(measurement.d1Queries).toBeLessThanOrEqual(150); + expect(measurement.rowsWritten).toBeLessThanOrEqual(5_000); + expect(measurement.wallDurationMs).toBeLessThan(5_000); expect(measurement.maxBinds).toBeLessThanOrEqual(100); expect(measurement.maxSqlBytes).toBeLessThan(100 * 1024); const { value: _value, ...evidence } = measurement; diff --git a/templates/blog-cloudflare/src/worker.ts b/templates/blog-cloudflare/src/worker.ts index 75fe3e2743..670648c81b 100644 --- a/templates/blog-cloudflare/src/worker.ts +++ b/templates/blog-cloudflare/src/worker.ts @@ -5,4 +5,4 @@ export { PluginBridge }; export default { ...handler, scheduled: createScheduledHandler(), -}; +} satisfies ExportedHandler; diff --git a/templates/blog-cloudflare/tsconfig.json b/templates/blog-cloudflare/tsconfig.json index 0903753115..c2e1508a58 100644 --- a/templates/blog-cloudflare/tsconfig.json +++ b/templates/blog-cloudflare/tsconfig.json @@ -3,5 +3,5 @@ "compilerOptions": { "types": ["node"] }, - "include": ["src", ".astro/types.d.ts", "emdash-env.d.ts"] + "include": ["src", ".astro/types.d.ts", "emdash-env.d.ts", "worker-configuration.d.ts"] } diff --git a/templates/blog-cloudflare/wrangler.jsonc b/templates/blog-cloudflare/wrangler.jsonc index 23632b0e98..cd8fd49e24 100644 --- a/templates/blog-cloudflare/wrangler.jsonc +++ b/templates/blog-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // General maintenance plus the bounded Media Usage lane (see src/worker.ts) + // General maintenance cron trigger "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, } diff --git a/templates/marketing-cloudflare/src/worker.ts b/templates/marketing-cloudflare/src/worker.ts index 75fe3e2743..670648c81b 100644 --- a/templates/marketing-cloudflare/src/worker.ts +++ b/templates/marketing-cloudflare/src/worker.ts @@ -5,4 +5,4 @@ export { PluginBridge }; export default { ...handler, scheduled: createScheduledHandler(), -}; +} satisfies ExportedHandler; diff --git a/templates/marketing-cloudflare/wrangler.jsonc b/templates/marketing-cloudflare/wrangler.jsonc index e5e23e764d..fcc837d63f 100644 --- a/templates/marketing-cloudflare/wrangler.jsonc +++ b/templates/marketing-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // General maintenance plus the bounded Media Usage lane (see src/worker.ts) + // General maintenance cron trigger "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, } diff --git a/templates/portfolio-cloudflare/src/worker.ts b/templates/portfolio-cloudflare/src/worker.ts index 75fe3e2743..670648c81b 100644 --- a/templates/portfolio-cloudflare/src/worker.ts +++ b/templates/portfolio-cloudflare/src/worker.ts @@ -5,4 +5,4 @@ export { PluginBridge }; export default { ...handler, scheduled: createScheduledHandler(), -}; +} satisfies ExportedHandler; diff --git a/templates/portfolio-cloudflare/wrangler.jsonc b/templates/portfolio-cloudflare/wrangler.jsonc index 792c1fda4d..ebc2105082 100644 --- a/templates/portfolio-cloudflare/wrangler.jsonc +++ b/templates/portfolio-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // General maintenance plus the bounded Media Usage lane (see src/worker.ts) + // General maintenance cron trigger "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, } diff --git a/templates/starter-cloudflare/src/worker.ts b/templates/starter-cloudflare/src/worker.ts index 75fe3e2743..670648c81b 100644 --- a/templates/starter-cloudflare/src/worker.ts +++ b/templates/starter-cloudflare/src/worker.ts @@ -5,4 +5,4 @@ export { PluginBridge }; export default { ...handler, scheduled: createScheduledHandler(), -}; +} satisfies ExportedHandler; diff --git a/templates/starter-cloudflare/tsconfig.json b/templates/starter-cloudflare/tsconfig.json index 0903753115..c2e1508a58 100644 --- a/templates/starter-cloudflare/tsconfig.json +++ b/templates/starter-cloudflare/tsconfig.json @@ -3,5 +3,5 @@ "compilerOptions": { "types": ["node"] }, - "include": ["src", ".astro/types.d.ts", "emdash-env.d.ts"] + "include": ["src", ".astro/types.d.ts", "emdash-env.d.ts", "worker-configuration.d.ts"] } diff --git a/templates/starter-cloudflare/wrangler.jsonc b/templates/starter-cloudflare/wrangler.jsonc index 23632b0e98..cd8fd49e24 100644 --- a/templates/starter-cloudflare/wrangler.jsonc +++ b/templates/starter-cloudflare/wrangler.jsonc @@ -22,8 +22,8 @@ "binding": "LOADER", }, ], - // General maintenance plus the bounded Media Usage lane (see src/worker.ts) + // General maintenance cron trigger "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, }