From 026098727c418832109dc9cb2446707bd10a4de0 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:07:15 -0700 Subject: [PATCH 1/2] Allow reprocessing files stuck in uploaded status (#168) * Allow reprocessing files stuck in uploaded status. Operators can kick S3 uploads that never entered processing via API, MCP, and the UI. Co-authored-by: Cursor * Gate reprocess on instruments with a Lambda processor. Hide UI actions and reject API/MCP reprocess for instruments without a process_file handler. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- developer-docs/lambda.md | 8 +- .../api/v1/files/[fileId]/reprocess/route.ts | 5 +- web/app/api/v1/files/[fileId]/route.ts | 2 +- .../runs/[runId]/reprocess/route.ts | 5 +- .../runs-table/run-bulk-action-bar.tsx | 1 + .../runs-table/run-row-actions.tsx | 1 + .../runs-table/run-selection-provider.tsx | 1 + .../runs/file-selection-provider.tsx | 13 ++- web/components/runs/reprocess-runs-dialog.tsx | 7 +- web/components/runs/run-files-section.tsx | 2 + web/components/runs/run-files-table.tsx | 25 +++--- web/lib/api/file-reprocessing.ts | 31 +++++++- web/lib/api/openapi/paths/files.ts | 3 +- web/lib/api/openapi/paths/runs.ts | 2 +- web/lib/instruments/processable-ids.ts | 19 +++++ web/lib/mcp/tools/files.defs.ts | 2 +- web/lib/mcp/tools/runs.defs.ts | 2 +- web/lib/runs/reprocessable-statuses.ts | 8 ++ web/lib/runs/row-actions.ts | 11 ++- web/tests/integration/files.test.ts | 79 ++++++++++++++++++- 20 files changed, 191 insertions(+), 36 deletions(-) create mode 100644 web/lib/instruments/processable-ids.ts create mode 100644 web/lib/runs/reprocessable-statuses.ts diff --git a/developer-docs/lambda.md b/developer-docs/lambda.md index 502ad168..33cfa900 100644 --- a/developer-docs/lambda.md +++ b/developer-docs/lambda.md @@ -19,7 +19,7 @@ Slack notifications are sent by the **web app**, not the Lambda — see [Slack n When a file fails processing (or needs to be re-run), users can trigger reprocessing from the run detail page in the web app. This invokes the Lambda's Function URL instead of going through S3: -1. The user clicks **Reprocess** on a failed or completed file in the web dashboard. +1. The user clicks **Reprocess** on an uploaded, failed, or completed file in the web dashboard. 2. The web app's `POST /api/v1/files/:fileId/reprocess` endpoint transitions the file to `processing` status, clears any previous error, and sends a POST request to the Lambda Function URL. 3. The Function URL is configured with `AuthType: AWS_IAM`, so the web app SigV4-signs the request using credentials it gets via Vercel OIDC federation (the `WebAppS3Role` IAM role, which has `lambda:InvokeFunctionUrl` on this function's ARN). The body is a JSON payload containing a synthetic S3 event. 4. The Lambda handler detects the Function URL invocation (via `requestContext` in the event) and parses the S3 event from the request body. Inbound auth is enforced by AWS itself in front of the function — the handler never sees an unauthenticated request. @@ -70,9 +70,11 @@ Slack channel notifications are sent by the **web app** (`web/lib/slack.ts`), no 3. **Register the dispatch.** Add an `elif` branch in the `lambda_handler` function in `lambda/src/data_hub_lambda/handler.py` that maps the new instrument ID to your `process_file` function. -4. **Add tests.** Add unit tests in `lambda/tests/` for the new processor. +4. **Expose reprocess in the web app.** Add the instrument ID to `PROCESSABLE_INSTRUMENT_IDS` in `web/lib/instruments/processable-ids.ts` so the UI and API allow reprocessing for that instrument. -5. **Configure the S3 trigger and deploy.** See [CI and deployment → Adding an S3 trigger for a new instrument](ci-and-deployment.md#adding-an-s3-trigger-for-a-new-instrument) for the `infra/template.yaml` trigger entry and the deploy steps. +5. **Add tests.** Add unit tests in `lambda/tests/` for the new processor. + +6. **Configure the S3 trigger and deploy.** See [CI and deployment → Adding an S3 trigger for a new instrument](ci-and-deployment.md#adding-an-s3-trigger-for-a-new-instrument) for the `infra/template.yaml` trigger entry and the deploy steps. ## Local processing CLI diff --git a/web/app/api/v1/files/[fileId]/reprocess/route.ts b/web/app/api/v1/files/[fileId]/reprocess/route.ts index 1f6c9ee9..79d619f0 100644 --- a/web/app/api/v1/files/[fileId]/reprocess/route.ts +++ b/web/app/api/v1/files/[fileId]/reprocess/route.ts @@ -14,8 +14,9 @@ interface RouteContext { // --------------------------------------------------------------------------- // POST /api/v1/files/:fileId/reprocess // -// Transitions a failed or completed file back to "processing" and invokes -// the Lambda Function URL to re-run the instrument's process_file workflow. +// Transitions an uploaded, failed, or completed file back to "processing" +// and invokes the Lambda Function URL to re-run the instrument's +// process_file workflow. // The core logic lives in lib/api/file-reprocessing.ts so the MCP server // can reuse it. // --------------------------------------------------------------------------- diff --git a/web/app/api/v1/files/[fileId]/route.ts b/web/app/api/v1/files/[fileId]/route.ts index 7d034c30..9b8abdf8 100644 --- a/web/app/api/v1/files/[fileId]/route.ts +++ b/web/app/api/v1/files/[fileId]/route.ts @@ -22,7 +22,7 @@ interface RouteContext { // Enforced state machine for file status transitions: // Watcher flow: detected → [upload_requested →] uploaded → processing → completed|failed // Lambda flow: (created as "uploaded" via POST .../files) → processing → completed|failed -// Reprocessing: completed|failed → processing → completed|failed +// Reprocessing: uploaded|completed|failed → processing → completed|failed // Cancel request: upload_requested → detected (watcher gave up locating // the local file after repeated polls; clears the queue // entry) diff --git a/web/app/api/v1/instruments/[instrumentId]/runs/[runId]/reprocess/route.ts b/web/app/api/v1/instruments/[instrumentId]/runs/[runId]/reprocess/route.ts index e9590922..15876c19 100644 --- a/web/app/api/v1/instruments/[instrumentId]/runs/[runId]/reprocess/route.ts +++ b/web/app/api/v1/instruments/[instrumentId]/runs/[runId]/reprocess/route.ts @@ -10,8 +10,9 @@ interface RouteContext { // --------------------------------------------------------------------------- // POST /api/v1/instruments/:instrumentId/runs/:runId/reprocess // -// Run-level convenience endpoint that reprocesses every `completed` or -// `failed` file on the run. Used by the runs list row/bulk actions. +// Run-level convenience endpoint that reprocesses every `uploaded`, +// `completed`, or `failed` file on the run. Used by the runs list +// row/bulk actions. // --------------------------------------------------------------------------- export async function POST(request: NextRequest, { params }: RouteContext) { diff --git a/web/components/instruments/runs-table/run-bulk-action-bar.tsx b/web/components/instruments/runs-table/run-bulk-action-bar.tsx index 6f6b0c77..f0108f04 100644 --- a/web/components/instruments/runs-table/run-bulk-action-bar.tsx +++ b/web/components/instruments/runs-table/run-bulk-action-bar.tsx @@ -182,6 +182,7 @@ export function RunBulkActionBar() { runId: r.runId, filesCompleted: r.stats.filesCompleted, filesFailed: r.stats.filesFailed, + filesUploaded: r.stats.filesUploaded, })); // On desktop the expanded sidebar reserves space on the left, so anchor the diff --git a/web/components/instruments/runs-table/run-row-actions.tsx b/web/components/instruments/runs-table/run-row-actions.tsx index 43c99263..253bdad0 100644 --- a/web/components/instruments/runs-table/run-row-actions.tsx +++ b/web/components/instruments/runs-table/run-row-actions.tsx @@ -199,6 +199,7 @@ export function RunRowActions({ row }: { row: RunRow }) { runId: row.run_id, filesCompleted: row.files_completed, filesFailed: row.files_failed, + filesUploaded: row.files_uploaded, }, ]} /> diff --git a/web/components/instruments/runs-table/run-selection-provider.tsx b/web/components/instruments/runs-table/run-selection-provider.tsx index 8d934cff..baf8c6bc 100644 --- a/web/components/instruments/runs-table/run-selection-provider.tsx +++ b/web/components/instruments/runs-table/run-selection-provider.tsx @@ -16,6 +16,7 @@ export interface RunStats { fileCount: number; filesCompleted: number; filesFailed: number; + filesUploaded: number; } export interface RunRef { diff --git a/web/components/runs/file-selection-provider.tsx b/web/components/runs/file-selection-provider.tsx index 308646c4..5901bcf4 100644 --- a/web/components/runs/file-selection-provider.tsx +++ b/web/components/runs/file-selection-provider.tsx @@ -2,6 +2,8 @@ import { createContext, use, useCallback, useMemo, useState } from "react"; import type { RunFile } from "@/lib/api/instrument-runs"; +import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; // --------------------------------------------------------------------------- // File selection provider for the run files table. Mirrors RunSelectionProvider @@ -18,7 +20,7 @@ const DOWNLOADABLE_STATUSES = new Set([ "failed", ]); -const REPROCESSABLE_STATUSES = new Set(["completed", "failed"]); +const REPROCESSABLE_STATUS_SET = new Set(REPROCESSABLE_STATUSES); export interface FileCaps { dismiss: boolean; @@ -36,14 +38,19 @@ export interface FileRef { // Returns null for rows that should not participate in selection at all // (dismissed files, transient `upload_requested` rows). Caller treats null // the same as "no checkbox in this row". -export function buildFileRef(file: RunFile): FileRef | null { +export function buildFileRef( + file: RunFile, + instrumentId: string +): FileRef | null { if (file.deletedAt !== null) { return null; } const isDetected = file.status === "detected"; const canDownload = DOWNLOADABLE_STATUSES.has(file.status); const canReprocess = - REPROCESSABLE_STATUSES.has(file.status) && file.s3Key !== null; + isProcessableInstrument(instrumentId) && + REPROCESSABLE_STATUS_SET.has(file.status) && + file.s3Key !== null; if (!(isDetected || canDownload)) { return null; } diff --git a/web/components/runs/reprocess-runs-dialog.tsx b/web/components/runs/reprocess-runs-dialog.tsx index ee787fc5..6a454d3b 100644 --- a/web/components/runs/reprocess-runs-dialog.tsx +++ b/web/components/runs/reprocess-runs-dialog.tsx @@ -18,6 +18,7 @@ import { export interface ReprocessRunTarget { filesCompleted: number; filesFailed: number; + filesUploaded: number; instrumentId: string; runId: string; } @@ -76,7 +77,7 @@ export function ReprocessRunsDialog({ const runCount = runs.length; const eligibleFiles = runs.reduce( - (sum, r) => sum + r.filesCompleted + r.filesFailed, + (sum, r) => sum + r.filesCompleted + r.filesFailed + r.filesUploaded, 0 ); @@ -115,8 +116,8 @@ export function ReprocessRunsDialog({ This will re-run the processing pipeline on{" "} {eligibleFiles}{" "} {eligibleFiles === 1 ? "file" : "files"} currently in the{" "} - completed or failed state. Results will overwrite - any existing processed artifacts. + uploaded, completed, or failed state. + Results will overwrite any existing processed artifacts. diff --git a/web/components/runs/run-files-section.tsx b/web/components/runs/run-files-section.tsx index b7e4a7d5..8e8df2ec 100644 --- a/web/components/runs/run-files-section.tsx +++ b/web/components/runs/run-files-section.tsx @@ -357,6 +357,7 @@ function RunFilesSectionContent({ ) : isDeleted ? ( handleSingleReprocess(id, startTransition, router) @@ -365,6 +366,7 @@ function RunFilesSectionContent({ ) : ( handleSingleDismiss(id, startTransition, router) diff --git a/web/components/runs/run-files-table.tsx b/web/components/runs/run-files-table.tsx index e7c07fcf..8f971d2a 100644 --- a/web/components/runs/run-files-table.tsx +++ b/web/components/runs/run-files-table.tsx @@ -24,6 +24,8 @@ import { } from "@/components/ui/table"; import type { RunFile } from "@/lib/api/instrument-runs"; import { formatDateTime } from "@/lib/date"; +import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; import { cn, formatBytes } from "@/lib/utils"; import { FileSelectAllCheckbox, @@ -41,7 +43,7 @@ const DOWNLOADABLE_STATUSES = new Set([ "failed", ]); -const REPROCESSABLE_STATUSES = new Set(["completed", "failed"]); +const REPROCESSABLE_STATUS_SET = new Set(REPROCESSABLE_STATUSES); const CATEGORY_BADGE_CLASSES: Record = { raw: "bg-blue-100 text-blue-700 dark:bg-blue-950 dark:text-blue-300", @@ -212,28 +214,31 @@ function UploadDismissActions({ ); } -function canReprocess(file: RunFile): boolean { +function canReprocess(file: RunFile, instrumentId: string): boolean { return ( file.deletedAt === null && - REPROCESSABLE_STATUSES.has(file.status) && + isProcessableInstrument(instrumentId) && + REPROCESSABLE_STATUS_SET.has(file.status) && file.s3Key !== null ); } // --------------------------------------------------------------------------- // Read-only variant: no selection column, no upload/dismiss. Reprocessing is -// still allowed for completed/failed files so operators can recover report -// data without restoring the run. +// still allowed for uploaded/completed/failed files so operators can recover +// report data or kick stuck uploads without restoring the run. // --------------------------------------------------------------------------- export interface ReadOnlyRunFilesTableProps { files: RunFile[]; + instrumentId: string; isPending: boolean; onReprocess: (id: number) => void; } export function ReadOnlyRunFilesTable({ files, + instrumentId, isPending, onReprocess, }: ReadOnlyRunFilesTableProps) { @@ -255,7 +260,7 @@ export function ReadOnlyRunFilesTable({ > - {canReprocess(file) && ( + {canReprocess(file, instrumentId) ? (
- )} + ) : null} ); @@ -283,6 +288,7 @@ export function ReadOnlyRunFilesTable({ export interface EditableRunFilesTableProps { files: RunFile[]; + instrumentId: string; isPending: boolean; onDismiss: (id: number) => void; onReprocess: (id: number) => void; @@ -291,6 +297,7 @@ export interface EditableRunFilesTableProps { export function EditableRunFilesTable({ files, + instrumentId, isPending, onUpload, onDismiss, @@ -305,7 +312,7 @@ export function EditableRunFilesTable({ const visibleSelectableRefs: NonNullable>[] = []; for (const file of files) { - const ref = buildFileRef(file); + const ref = buildFileRef(file, instrumentId); refsByFileId.set(file.id, ref); if (ref) { visibleSelectableRefs.push(ref); @@ -334,7 +341,7 @@ export function EditableRunFilesTable({ const ref = refsByFileId.get(file.id) ?? null; const isSelected = ref ? meta.isSelected(ref.id) : false; const canDoUploadDismiss = !isDismissed && file.status === "detected"; - const canDoReprocess = canReprocess(file); + const canDoReprocess = canReprocess(file, instrumentId); // Reveal classes: hide per-row actions while a bulk selection is // active (the bar is the single entry point) but keep the JSX diff --git a/web/lib/api/file-reprocessing.ts b/web/lib/api/file-reprocessing.ts index 69655577..5e8bc03a 100644 --- a/web/lib/api/file-reprocessing.ts +++ b/web/lib/api/file-reprocessing.ts @@ -3,9 +3,9 @@ import { after } from "next/server"; import { lookupRunByNaturalKey } from "@/lib/api/instrument-runs"; import { db } from "@/lib/db"; import { files, instrumentRuns } from "@/lib/db/schema"; +import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; import { hasInvokeCredentials, signLambdaInvoke } from "@/lib/lambda"; - -const REPROCESSABLE_STATUSES = ["failed", "completed"] as const; +import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; function getLambdaUrl(): string | null { const url = process.env.LAMBDA_FUNCTION_URL; @@ -61,7 +61,7 @@ export async function reprocessFile(fileId: number): Promise { ok: false, status: 409, code: "CONFLICT", - message: `Cannot reprocess a file in '${file.status}' status — only 'failed' or 'completed' files can be reprocessed`, + message: `Cannot reprocess a file in '${file.status}' status — only 'uploaded', 'failed', or 'completed' files can be reprocessed`, }; } @@ -75,7 +75,10 @@ export async function reprocessFile(fileId: number): Promise { } const [parentRun] = await db - .select({ deletedAt: instrumentRuns.deletedAt }) + .select({ + deletedAt: instrumentRuns.deletedAt, + instrumentId: instrumentRuns.instrumentId, + }) .from(instrumentRuns) .where(eq(instrumentRuns.id, file.instrumentRunId)) .limit(1); @@ -89,6 +92,17 @@ export async function reprocessFile(fileId: number): Promise { }; } + if (!(parentRun && isProcessableInstrument(parentRun.instrumentId))) { + return { + ok: false, + status: 409, + code: "CONFLICT", + message: parentRun + ? `Instrument '${parentRun.instrumentId}' has no Lambda processor — cannot reprocess` + : "Cannot reprocess a file with no parent run", + }; + } + const lambdaUrl = getLambdaUrl(); if (!lambdaUrl) { return { @@ -175,6 +189,15 @@ export async function reprocessRun( instrumentId: string, runId: string ): Promise { + if (!isProcessableInstrument(instrumentId)) { + return { + ok: false, + status: 409, + code: "CONFLICT", + message: `Instrument '${instrumentId}' has no Lambda processor — cannot reprocess`, + }; + } + const run = await lookupRunByNaturalKey(instrumentId, runId); if (!run) { diff --git a/web/lib/api/openapi/paths/files.ts b/web/lib/api/openapi/paths/files.ts index 5c5b03ef..4fe44391 100644 --- a/web/lib/api/openapi/paths/files.ts +++ b/web/lib/api/openapi/paths/files.ts @@ -85,7 +85,8 @@ registry.registerPath({ path: "/files/{fileId}/reprocess", operationId: "reprocessFile", summary: "Reprocess a file", - description: "Requires scope `files:reprocess`.", + description: + "Requires scope `files:reprocess`. Eligible statuses: `uploaded`, `failed`, or `completed` (file must have an S3 location). The file's instrument must have a Lambda processor.", tags: ["Files"], security: bearerSecurity, request: { params: fileParams }, diff --git a/web/lib/api/openapi/paths/runs.ts b/web/lib/api/openapi/paths/runs.ts index dd146143..aaf2154c 100644 --- a/web/lib/api/openapi/paths/runs.ts +++ b/web/lib/api/openapi/paths/runs.ts @@ -139,7 +139,7 @@ registry.registerPath({ path: "/instruments/{instrumentId}/runs/{runId}/reprocess", operationId: "reprocessInstrumentRun", summary: "Reprocess a run", - description: scoped("runs:reprocess"), + description: `${scoped("runs:reprocess")} Reprocesses every \`uploaded\`, \`failed\`, or \`completed\` file on the run. The instrument must have a Lambda processor.`, tags: tag, security: bearerSecurity, request: { params: runParams }, diff --git a/web/lib/instruments/processable-ids.ts b/web/lib/instruments/processable-ids.ts new file mode 100644 index 00000000..fe606869 --- /dev/null +++ b/web/lib/instruments/processable-ids.ts @@ -0,0 +1,19 @@ +// Instrument IDs that have a `process_file` branch in +// `lambda/src/data_hub_lambda/handler.py`. Keep in sync when adding a +// processor — instruments without a handler (e.g. InstantRaman) stay out. +export const PROCESSABLE_INSTRUMENT_IDS = [ + "akta-fplc", + "agilent-4150-tapestation", + "azure-600-gel-doc", + "azure-cielo-qpcr", + "epson-v700-scanner", + "hina-microscope", + "spectramax-id3-plate-reader", + "spectramax-id5-plate-reader", +] as const; + +const PROCESSABLE_SET = new Set(PROCESSABLE_INSTRUMENT_IDS); + +export function isProcessableInstrument(instrumentId: string): boolean { + return PROCESSABLE_SET.has(instrumentId); +} diff --git a/web/lib/mcp/tools/files.defs.ts b/web/lib/mcp/tools/files.defs.ts index 42cc634d..91c41a22 100644 --- a/web/lib/mcp/tools/files.defs.ts +++ b/web/lib/mcp/tools/files.defs.ts @@ -41,7 +41,7 @@ export const reprocessFileTool = { name: "reprocess_file", title: "Reprocess File", description: - "Re-run the Lambda processing workflow for a failed or completed file. Transitions the file back to 'processing'. Use this to retry after a parser fix or transient Lambda failure.", + "Re-run the Lambda processing workflow for an uploaded, failed, or completed file on an instrument that has a Lambda processor. Transitions the file back to 'processing'. Use this to retry after a parser fix, transient Lambda failure, or a stuck upload that never entered processing.", group: "files", scope: "files:reprocess", inputSchema: { fileId: z.number().int().describe("Numeric file ID") }, diff --git a/web/lib/mcp/tools/runs.defs.ts b/web/lib/mcp/tools/runs.defs.ts index 7976146c..5bd8133f 100644 --- a/web/lib/mcp/tools/runs.defs.ts +++ b/web/lib/mcp/tools/runs.defs.ts @@ -250,7 +250,7 @@ export const reprocessRunTool = { scope: "runs:reprocess", title: "Reprocess Run", description: - "Re-run Lambda processing for every completed or failed file on a run. Prefer this over looping reprocess_file for bulk retries after a parser fix.", + "Re-run Lambda processing for every uploaded, completed, or failed file on a run. The instrument must have a Lambda processor. Prefer this over looping reprocess_file for bulk retries after a parser fix or to kick stuck uploads.", inputSchema: { instrumentId: z.string().describe("Instrument identifier"), runId: z.string().describe("Run identifier within the instrument"), diff --git a/web/lib/runs/reprocessable-statuses.ts b/web/lib/runs/reprocessable-statuses.ts new file mode 100644 index 00000000..8c4b96d6 --- /dev/null +++ b/web/lib/runs/reprocessable-statuses.ts @@ -0,0 +1,8 @@ +// Statuses eligible for POST /files/:id/reprocess (and run-level reprocess). +// Includes `uploaded` so stuck S3 uploads can be kicked when the Lambda +// trigger never fired. Client-safe — imported by UI tables and the API. +export const REPROCESSABLE_STATUSES = [ + "uploaded", + "failed", + "completed", +] as const; diff --git a/web/lib/runs/row-actions.ts b/web/lib/runs/row-actions.ts index e1271136..3f9d7b88 100644 --- a/web/lib/runs/row-actions.ts +++ b/web/lib/runs/row-actions.ts @@ -4,6 +4,7 @@ import type { RunRef, RunStats, } from "@/components/instruments/runs-table/run-selection-provider"; +import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; // --------------------------------------------------------------------------- // Predicates that decide which row-level / bulk actions are available for a @@ -13,7 +14,8 @@ import type { // - Upload: at least one file still waiting to be uploaded. // - Download: at least one file has made it to S3 (i.e. is not `detected` // or `upload_requested`). -// - Reprocess: at least one file is in `completed` or `failed` status. +// - Reprocess: instrument has a Lambda processor and at least one file is in +// `uploaded`, `completed`, or `failed`. // - Delete: the run isn't already soft-deleted. // --------------------------------------------------------------------------- @@ -30,7 +32,11 @@ export function canDownloadRun(row: RunRow): boolean { } export function canReprocessRun(row: RunRow): boolean { - return row.deleted_at === null && row.files_completed + row.files_failed > 0; + return ( + row.deleted_at === null && + isProcessableInstrument(row.instrument_id) && + row.files_completed + row.files_failed + row.files_uploaded > 0 + ); } export function canDeleteRun(row: RunRow): boolean { @@ -51,6 +57,7 @@ export function computeRunStats(row: RunRow): RunStats { fileCount: row.file_count, filesCompleted: row.files_completed, filesFailed: row.files_failed, + filesUploaded: row.files_uploaded, }; } diff --git a/web/tests/integration/files.test.ts b/web/tests/integration/files.test.ts index f2e96157..ae714c8f 100644 --- a/web/tests/integration/files.test.ts +++ b/web/tests/integration/files.test.ts @@ -23,7 +23,9 @@ import { // Tests drive each through its full lifecycle to verify the state machine. describe("Files API", () => { let token: string; - const instrumentId = "files-test-instrument"; + // Use a known processable instrument ID so reprocess eligibility reaches + // the Lambda-config check (see `isProcessableInstrument`). + const instrumentId = "akta-fplc"; const runId = "files-test-run"; let fileId: number; let secondFileId: number; @@ -37,7 +39,7 @@ describe("Files API", () => { const db = getTestDb(); await db.insert(instruments).values({ id: instrumentId, - displayName: "Files Test Instrument", + displayName: "Akta FPLC", status: "active", }); @@ -487,6 +489,7 @@ describe("Files API", () => { // secondFileId (sample2.csv) → detected, soft-deleted // thirdFileId (sample3.csv) → detected, not deleted // lambdaFileId (processed_output) → failed, has S3 info + // Uploaded eligibility is covered by a dedicated run/file created below. // ------------------------------------------------------------------------- it("REPROCESS returns 401 without auth", async () => { @@ -583,7 +586,7 @@ describe("Files API", () => { expect(data.error.message).toContain("parent run"); }); - // These two tests verify that both reprocessable statuses (failed and + // These tests verify that reprocessable statuses (uploaded, failed, and // completed) pass all validation guards. They return 503 because the // test server has no LAMBDA_FUNCTION_URL configured. it("REPROCESS returns 503 for failed file when Lambda is not configured", async () => { @@ -605,4 +608,74 @@ describe("Files API", () => { const data = await res.json(); expect(data.error.message).toContain("not configured"); }); + + it("REPROCESS returns 503 for uploaded file when Lambda is not configured", async () => { + const uploadedRunId = "reprocess-uploaded-run"; + await api(`/api/v1/instruments/${instrumentId}/runs`, { + method: "POST", + token, + body: { run_id: uploadedRunId, source: "lambda" }, + }); + const createFileRes = await api( + `/api/v1/instruments/${instrumentId}/runs/${uploadedRunId}/files`, + { + method: "POST", + token, + body: { + s3_bucket: "test-bucket", + s3_key: `${instrumentId}/${uploadedRunId}/stuck.csv`, + filename: "stuck.csv", + }, + } + ); + expect(createFileRes.status).toBe(201); + const createdFile = await createFileRes.json(); + expect(createdFile.status).toBe("uploaded"); + + const res = await api(`/api/v1/files/${createdFile.id}/reprocess`, { + method: "POST", + token, + }); + expect(res.status).toBe(503); + const data = await res.json(); + expect(data.error.message).toContain("not configured"); + }); + + it("REPROCESS returns 409 for instrument without a Lambda processor", async () => { + const noProcessorId = "files-no-processor-instrument"; + const db = getTestDb(); + await db.insert(instruments).values({ + id: noProcessorId, + displayName: "No Processor Instrument", + status: "active", + }); + const noProcessorRunId = "reprocess-no-processor-run"; + await api(`/api/v1/instruments/${noProcessorId}/runs`, { + method: "POST", + token, + body: { run_id: noProcessorRunId, source: "lambda" }, + }); + const createFileRes = await api( + `/api/v1/instruments/${noProcessorId}/runs/${noProcessorRunId}/files`, + { + method: "POST", + token, + body: { + s3_bucket: "test-bucket", + s3_key: `${noProcessorId}/${noProcessorRunId}/data.csv`, + filename: "data.csv", + }, + } + ); + expect(createFileRes.status).toBe(201); + const createdFile = await createFileRes.json(); + + const res = await api(`/api/v1/files/${createdFile.id}/reprocess`, { + method: "POST", + token, + }); + expect(res.status).toBe(409); + const data = await res.json(); + expect(data.error.message).toContain("no Lambda processor"); + }); }); From 6dcc80bb0b12e8b8dd2ff429d6b068824a312d75 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:07:38 -0700 Subject: [PATCH 2/2] Dispatch Lambda processing by instrument_type instead of ID allowlists (#169) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Dispatch Lambda processing by instrument_type instead of ID allowlists. Adds a type→processor registry with catch-all S3 notifications, an fplc enum value, and type-based reprocess gating so new instruments don't need per-ID infra or handler branches. Co-authored-by: Cursor * Harden type-based Lambda dispatch against drift and reprocess dead-ends. Cache instrument lookups, fail Function URL no-ops so files aren't stranded in processing, and tighten InstrumentType typing across the reprocess UI. Co-authored-by: Cursor * Trim get_instrument in-invocation retries to one short backoff. Longer API outages should rely on Lambda async retries so we don't bill 10 GB sleep time inside each invocation. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- developer-docs/ci-and-deployment.md | 20 +- developer-docs/conventions.md | 16 +- developer-docs/first-time-deployment.md | 8 +- developer-docs/lambda.md | 56 +- developer-docs/shared-library.md | 10 +- infra/template.yaml | 85 +- .../agilent_4150_tapestation/process_file.py | 16 +- .../data_hub_lambda/akta_fplc/process_file.py | 14 +- lambda/src/data_hub_lambda/api_client.py | 71 +- .../azure_600_gel_doc/process_file.py | 20 +- .../azure_cielo_qpcr/process_file.py | 18 +- lambda/src/data_hub_lambda/cli.py | 9 +- .../epson_v700_scanner/process_file.py | 24 +- lambda/src/data_hub_lambda/handler.py | 206 +- .../hina_microscope/process_file.py | 20 +- lambda/src/data_hub_lambda/models.py | 9 + lambda/src/data_hub_lambda/processors.py | 83 + .../spectramax_plate_reader/process_file.py | 5 +- .../epson_v700_scanner/test_process_file.py | 30 +- .../hina_microscope/test_process_file.py | 18 +- lambda/tests/integration/conftest.py | 14 +- lambda/tests/integration/test_lambda_api.py | 76 +- lambda/tests/test_processors.py | 406 +++ .../shared/src/data_hub_shared/constants.py | 23 +- .../shared/src/data_hub_shared/testing.py | 41 +- .../instruments/edit-instrument-dialog.tsx | 48 +- .../notification-bell-content.tsx | 1 + .../runs/file-selection-provider.tsx | 7 +- web/components/runs/run-files-section.tsx | 8 +- web/components/runs/run-files-table.tsx | 21 +- .../runs/variants/default-run-detail.tsx | 1 + .../variants/epson-scanner-run-detail.tsx | 1 + .../runs/variants/gel-doc-run-detail.tsx | 1 + .../variants/hina-microscope-run-detail.tsx | 1 + .../variants/instant-raman-run-detail.tsx | 1 + .../runs/variants/plate-reader-run-detail.tsx | 1 + .../runs/variants/qpcr-run-detail.tsx | 1 + .../runs/variants/tape-station-run-detail.tsx | 1 + web/drizzle/0034_needy_tomorrow_man.sql | 1 + web/drizzle/meta/0034_snapshot.json | 2250 +++++++++++++++++ web/drizzle/meta/_journal.json | 7 + web/lib/api/file-reprocessing.ts | 28 +- web/lib/api/instrument-runs.ts | 7 +- web/lib/api/openapi/schemas/runs.ts | 2 + web/lib/api/scope-catalog.ts | 1 + web/lib/db/schema.ts | 1 + web/lib/db/seed.ts | 2 + web/lib/instruments/processable-ids.ts | 19 - web/lib/instruments/processable-types.ts | 22 + web/lib/runs/row-actions.ts | 8 +- web/tests/integration/files.test.ts | 48 +- 51 files changed, 3396 insertions(+), 391 deletions(-) create mode 100644 lambda/src/data_hub_lambda/processors.py create mode 100644 lambda/tests/test_processors.py create mode 100644 web/drizzle/0034_needy_tomorrow_man.sql create mode 100644 web/drizzle/meta/0034_snapshot.json delete mode 100644 web/lib/instruments/processable-ids.ts create mode 100644 web/lib/instruments/processable-types.ts diff --git a/developer-docs/ci-and-deployment.md b/developer-docs/ci-and-deployment.md index e62e02a2..5715ae8a 100644 --- a/developer-docs/ci-and-deployment.md +++ b/developer-docs/ci-and-deployment.md @@ -110,7 +110,7 @@ On pushes to `staging` or `production`, the **Deploy Lambda** workflow: Secrets (`DATA_HUB_API_KEY`, etc.) are stored in GitHub environment secrets scoped to each environment. -> **Note:** The CI deploy role has intentionally narrow permissions — enough to push a new container image, update the existing CloudFormation stack, modify the data buckets' S3 event notifications (so new instrument triggers roll out via CI), and update the data buckets' CORS configuration, but _not_ enough to create the stack from scratch or to add/remove S3 buckets or Lambda functions. Initial stack creation and structural infrastructure changes must be performed by an admin with broader AWS permissions. Once the stack exists, routine image-update deploys and new-trigger rollouts through CI work without issue. +> **Note:** The CI deploy role has intentionally narrow permissions — enough to push a new container image, update the existing CloudFormation stack, modify the data buckets' S3 event notifications, and update the data buckets' CORS configuration, but _not_ enough to create the stack from scratch or to add/remove S3 buckets or Lambda functions. Initial stack creation and structural infrastructure changes must be performed by an admin with broader AWS permissions. Once the stack exists, routine image-update deploys through CI work without issue. > > The deploy that first grants `s3:PutBucketCORS` must be run by an admin via `make sam-deploy` (CI can't grant itself a permission and use it in the same changeset). CORS edits after that roll out through CI. > @@ -144,23 +144,9 @@ make docker-push-lambda ENV=staging make sam-deploy ENV=staging ``` -#### Adding an S3 trigger for a new instrument +#### S3 notifications -Instruments that support automated preprocessing need an S3 event trigger so the Lambda runs as files land. The processor code lives in `data-hub-lambda` (see [Lambda → Adding a new instrument](lambda.md#adding-a-new-instrument)); this is the infrastructure half. Add a `LambdaConfiguration` entry to the `RawDataBucket` resource's `NotificationConfiguration` in `infra/template.yaml`: - -```yaml -- Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: / - - Name: suffix - Value: .csv - Function: !GetAtt DataHubFunction.Arn -``` - -The CI deploy role has permission to roll new triggers out, so the trigger goes live on the next deploy — either the [automated workflow](#automated-deployment-deploy-lambdayml) or a manual `make sam-deploy`. No manual AWS step is needed once the code and trigger are merged. +The raw bucket uses a single catch-all `ObjectCreated:*` notification on the Lambda. New instrument types do **not** need a new `LambdaConfiguration` entry — register a processor by `instrument_type` instead (see [Lambda → Adding a new instrument / processor](lambda.md#adding-a-new-instrument--processor)). Deploy the type-dispatch handler before changing notification filters when rolling this out to an environment that still has per-ID rules. ### Watcher (PyPI) diff --git a/developer-docs/conventions.md b/developer-docs/conventions.md index 20cc02b4..db8865ac 100644 --- a/developer-docs/conventions.md +++ b/developer-docs/conventions.md @@ -10,25 +10,27 @@ All raw data files are stored in S3 with the key pattern: {instrument_id}/{run_id}/{filename} ``` -- **`instrument_id`** — kebab-case identifier matching the `Instrument` enum (e.g., `akta-fplc`). +- **`instrument_id`** — kebab-case identifier for the instrument row (e.g., `akta-fplc`). - **`run_id`** — unique identifier for the run, either extracted from the filename prefix or from a subdirectory name. - **`filename`** — the original filename. The S3 bucket name follows the template `arcadia-data-hub-raw-{environment}`, where `environment` is `staging` or `production`. -## Instrument IDs +## Instrument IDs and types Instrument IDs are kebab-case strings (lowercase letters, numbers, hyphens). They serve as: - S3 key prefixes - API resource identifiers -- Enum values in `data_hub_shared.enums.Instrument` +- Primary keys on the `instruments` table -When adding a new instrument, the ID must be registered in three places: +Lambda dispatch and web reprocess eligibility use **`instrument_type`**, not the ID. When adding a processable instrument: -1. `Instrument` enum in `packages/shared/src/data_hub_shared/enums.py` -2. `INSTRUMENT_ID_TO_NAME_MAP` in `packages/shared/src/data_hub_shared/constants.py` -3. Dispatch logic in `lambda/src/data_hub_lambda/handler.py` +1. Set (or add) the appropriate `instrument_type` on the instrument row +2. Register a processor for that type in `lambda/src/data_hub_lambda/processors.py` +3. Add the same type to `PROCESSABLE_INSTRUMENT_TYPES` in `web/lib/instruments/processable-types.ts` + +The shared `Instrument` enum in `packages/shared` is optional legacy naming for watcher/CLI display — it is not the Lambda support gate. ## Environment variables diff --git a/developer-docs/first-time-deployment.md b/developer-docs/first-time-deployment.md index 54a23391..f20b78c9 100644 --- a/developer-docs/first-time-deployment.md +++ b/developer-docs/first-time-deployment.md @@ -82,7 +82,11 @@ npm run db:migrate ### Create an API key for the Lambda -Sign in with an account listed in `ADMIN_EMAILS`, then create a personal access token under Settings. The AWS stack and the Lambda use this token as `DATA_HUB_API_KEY` to call the Data Hub API, so create it now and keep it for [step 4](#4-deploy-the-aws-infrastructure). See [Issue and revoke tokens](https://datahub.arcadiascience.com/docs/manage-tokens) for the token UI. +Sign in with an account listed in `ADMIN_EMAILS`, then create a personal access token under Settings. Use the **Lambda** scope preset (or an equivalent list that includes `instruments:read`, `runs:create`, `runs:update`, `files:create`, `files:update`, and `archive-jobs:write`). The Lambda looks up each instrument's type before dispatching, so a token without `instruments:read` will 403 on every S3 event. + +The AWS stack and the Lambda use this token as `DATA_HUB_API_KEY` to call the Data Hub API, so create it now and keep it for [step 4](#4-deploy-the-aws-infrastructure). See [Issue and revoke tokens](https://datahub.arcadiascience.com/docs/manage-tokens) for the token UI. + +If you previously minted a Lambda token from an older preset that omitted `instruments:read`, revoke it and create a new one with the updated Lambda preset, then update the `DATA_HUB_API_KEY` secret / SAM parameter for each environment. ## 3. Bootstrap AWS (once per account) @@ -96,7 +100,7 @@ make sam-bootstrap ## 4. Deploy the AWS infrastructure -The storage and processing layer — the S3 buckets and the data-processing Lambda — is defined in `infra/template.yaml` and deployed with [AWS SAM](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/). The stack creates the raw/processed S3 buckets, the Lambda function (container image, function URL), the per-instrument S3 event triggers, and the IAM roles for Lambda execution, CI deploys (OIDC), and Vercel web app S3 access (OIDC). +The storage and processing layer — the S3 buckets and the data-processing Lambda — is defined in `infra/template.yaml` and deployed with [AWS SAM](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/). The stack creates the raw/processed S3 buckets, the Lambda function (container image, function URL), a catch-all S3 `ObjectCreated:*` notification on the raw bucket, and the IAM roles for Lambda execution, CI deploys (OIDC), and Vercel web app S3 access (OIDC). **1. Get the bootstrap stack outputs.** diff --git a/developer-docs/lambda.md b/developer-docs/lambda.md index 33cfa900..1afafa1b 100644 --- a/developer-docs/lambda.md +++ b/developer-docs/lambda.md @@ -8,10 +8,11 @@ The Lambda has three invocation paths: ### S3 trigger (automatic) -1. An S3 `PutObject` event triggers the Lambda function. +1. An S3 `ObjectCreated:*` event on the raw bucket triggers the Lambda (catch-all; no per-instrument prefix/suffix filters). 2. The handler parses the S3 key to extract the instrument ID, run ID, and filename. The expected key layout is `{instrument_id}/{run_id}/{filename}`. -3. It dispatches to the appropriate instrument processor based on the instrument ID. -4. The processor downloads the raw file from S3, preprocesses it (e.g., extracting metadata), and creates/updates the run and files via the Data Hub API. +3. For S3-triggered events, a cheap union of processor filename gates runs first. Non-matching files no-op without an API call. +4. The handler fetches the instrument via `GET /instruments/:id` and looks up a processor by `instrument_type` in `data_hub_lambda.processors`. Unmapped types (including `generic`) and per-type gate failures no-op. +5. The processor downloads the raw file from S3, preprocesses it, and creates/updates the run and files via the Data Hub API using the event's `instrument_id`. Slack notifications are sent by the **web app**, not the Lambda — see [Slack notifications](#slack-notifications) below. @@ -23,7 +24,7 @@ When a file fails processing (or needs to be re-run), users can trigger reproces 2. The web app's `POST /api/v1/files/:fileId/reprocess` endpoint transitions the file to `processing` status, clears any previous error, and sends a POST request to the Lambda Function URL. 3. The Function URL is configured with `AuthType: AWS_IAM`, so the web app SigV4-signs the request using credentials it gets via Vercel OIDC federation (the `WebAppS3Role` IAM role, which has `lambda:InvokeFunctionUrl` on this function's ARN). The body is a JSON payload containing a synthetic S3 event. 4. The Lambda handler detects the Function URL invocation (via `requestContext` in the event) and parses the S3 event from the request body. Inbound auth is enforced by AWS itself in front of the function — the handler never sees an unauthenticated request. -5. From here, processing follows the same dispatch logic as the S3 trigger path (steps 2–4 above). +5. From here, processing follows the same type-based dispatch as the S3 trigger path, except **filename gates are skipped** — a user clicking Reprocess has stated intent, so the handler must not leave the file stranded in `processing`. ### Function URL (archive build) @@ -36,45 +37,48 @@ The web app's `GET /api/v1/instruments/:instrumentId/runs/:runId/download-archiv See [Run archives](run-archives.md) for the full flow, S3 bucket layout, cache semantics, and operator runbook. -## Supported instruments +## Supported instrument types -| Instrument | Module | Instrument ID | +Dispatch is by `instrument_type` (Postgres/TS enum), not instrument ID. The registry lives in `lambda/src/data_hub_lambda/processors.py`; the web reprocess gate mirrors the same keys in `web/lib/instruments/processable-types.ts`. + +| `instrument_type` | Module | Filename gate (S3 events only) | | --- | --- | --- | -| Agilent 4150 TapeStation | `agilent_4150_tapestation` | `agilent-4150-tapestation` | -| Akta FPLC | `akta_fplc` | `akta-fplc` | -| Azure 600 Gel Doc | `azure_600_gel_doc` | `azure-600-gel-doc` | -| Azure Cielo qPCR | `azure_cielo_qpcr` | `azure-cielo-qpcr` | -| Epson V700 Scanner | `epson_v700_scanner` | `epson-v700-scanner` | -| Hina Microscope | `hina_microscope` | `hina-microscope` | -| InstantRaman | _(no Lambda processor)_ | `instant-raman` | -| SpectraMax iD3 Plate Reader | `spectramax_plate_reader` | `spectramax-id3-plate-reader` | -| SpectraMax iD5 Plate Reader | `spectramax_plate_reader` | `spectramax-id5-plate-reader` | - -Each processor module exposes a `process_file()` function that accepts the run ID and filename (and instrument ID for SpectraMax readers) and reports progress back through the Data Hub API. +| `tape_station` | `agilent_4150_tapestation` | `.pdf` | +| `fplc` | `akta_fplc` | `.pdf` | +| `gel_doc` | `azure_600_gel_doc` | `.tif` / `.tiff` | +| `qpcr` | `azure_cielo_qpcr` | ends with `_cq values.csv` | +| `epson_v700_scanner` | `epson_v700_scanner` | `.tif` / `.tiff` | +| `hina_microscope` | `hina_microscope` | `.nd2` | +| `plate_reader` | `spectramax_plate_reader` | `.xls` | +| `generic`, `instant_raman` | — | — | + +**One type = one vendor's output format.** Names like `qpcr` and `fplc` sound generic, but the parsers behind them are vendor-specific (Azure Cielo, ÄKTA, …). Adding a second vendor under an existing type requires splitting the type, not reusing it. + +Seeded `jolene-fplc` stays `generic` until an operator confirms its PDFs match the ÄKTA processor and edits the type to `fplc`. Typing an unknown FPLC as `fplc` would feed non-ÄKTA files into that parser. + +Each processor module exposes `process_file(instrument_id, run_id, filename)` and reports progress through the Data Hub API. ## Slack notifications Slack channel notifications are sent by the **web app** (`web/lib/slack.ts`), not the Lambda. When the Lambda's `process_file` calls `POST /api/v1/instruments/:instrumentId/runs` to register a newly-detected run, that endpoint posts a single message per run to the incoming webhook URL configured in Settings > Notifications > Slack channel (workspace admins only). Subsequent files for the same run do not re-notify because the upsert is idempotent on `(instrument_id, run_id)`. File-level failures remain visible in the web app via the file row's `status='failed'` and `error_message` fields. -## Adding a new instrument +## Adding a new instrument / processor -1. **Register the instrument.** Add a new member to the `Instrument` enum in `packages/shared/src/data_hub_shared/enums.py` and a corresponding entry in the `INSTRUMENT_ID_TO_NAME_MAP` in `packages/shared/src/data_hub_shared/constants.py`. +1. **Add or reuse an `instrument_type`.** If this is a new vendor format, extend `instrumentTypeEnum` in `web/lib/db/schema.ts` and generate an `ALTER TYPE ... ADD VALUE` migration. Create the instrument row in the web app with that type (or edit an existing row). The shared `Instrument` enum in `packages/shared` is optional — only needed for watcher/CLI display naming, not for Lambda dispatch. -2. **Create a processor module.** Add a new module under `lambda/src/data_hub_lambda/` (e.g., `new_instrument.py`). It must expose: +2. **Create a processor module** under `lambda/src/data_hub_lambda/` that exposes: ```python - def process_file(run_id: str, filename: str) -> None: + def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a file, reporting progress via the Data Hub API.""" ... ``` -3. **Register the dispatch.** Add an `elif` branch in the `lambda_handler` function in `lambda/src/data_hub_lambda/handler.py` that maps the new instrument ID to your `process_file` function. - -4. **Expose reprocess in the web app.** Add the instrument ID to `PROCESSABLE_INSTRUMENT_IDS` in `web/lib/instruments/processable-ids.ts` so the UI and API allow reprocessing for that instrument. +3. **Register it** in `lambda/src/data_hub_lambda/processors.py` (type → `process_file` + `matches_filename`) and add the same type string to `PROCESSABLE_INSTRUMENT_TYPES` in `web/lib/instruments/processable-types.ts`. -5. **Add tests.** Add unit tests in `lambda/tests/` for the new processor. +4. **Add tests** for the processor and for the new registry gate. -6. **Configure the S3 trigger and deploy.** See [CI and deployment → Adding an S3 trigger for a new instrument](ci-and-deployment.md#adding-an-s3-trigger-for-a-new-instrument) for the `infra/template.yaml` trigger entry and the deploy steps. +5. **Deploy the Lambda image.** The raw bucket already notifies on all `ObjectCreated:*` events — no new S3 trigger entry is required. ## Local processing CLI diff --git a/developer-docs/shared-library.md b/developer-docs/shared-library.md index b6b4abff..f86c521e 100644 --- a/developer-docs/shared-library.md +++ b/developer-docs/shared-library.md @@ -6,7 +6,7 @@ ### `enums` -Defines the `Instrument` enum, whose kebab-case values are used as S3 key prefixes and as the canonical instrument identifiers throughout the system. +Defines the optional `Instrument` enum for watcher/CLI display naming. Lambda dispatch and reprocess eligibility use `instrument_type` on the web app's `instruments` table, not this enum. ```python from data_hub_shared.enums import Instrument @@ -14,7 +14,7 @@ from data_hub_shared.enums import Instrument Instrument.AKTA_FPLC.value # "akta-fplc" ``` -Currently supported instruments: +Enum members available for display naming: | Enum member | Value | | --- | --- | @@ -27,11 +27,7 @@ Currently supported instruments: ### `constants` -Maps between instrument IDs and human-readable display names: - -```python -from data_hub_shared.constants import INSTRUMENT_ID_TO_NAME_MAP, INSTRUMENT_NAME_TO_ID_MAP -``` +Reserved for cross-package constants. Instrument display names live in the web app's `instruments` table; Lambda dispatch uses `instrument_type` rather than a shared ID→name map. ### `config` diff --git a/infra/template.yaml b/infra/template.yaml index 19bd08bb..ed96301d 100644 --- a/infra/template.yaml +++ b/infra/template.yaml @@ -88,89 +88,12 @@ Resources: - Content-Length MaxAge: 3600 NotificationConfiguration: + # Catch-all: the Lambda handler resolves instrument_type via the API + # and applies per-processor filename gates. Deploy the type-dispatch + # handler before (or with) this notification change — an older + # ID-dispatched handler would process every ObjectCreated event. LambdaConfigurations: - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: agilent-4150-tapestation/ - - Name: suffix - Value: .pdf - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: akta-fplc/ - - Name: suffix - Value: .pdf - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: azure-600-gel-doc/ - - Name: suffix - Value: .tif - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: azure-cielo-qpcr/ - - Name: suffix - # S3 requires spaces in filter values to be replaced - # with `+` — this matches keys ending in `_Cq Values.csv`. - Value: _Cq+Values.csv - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: epson-v700-scanner/ - - Name: suffix - Value: .tif - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: epson-v700-scanner/ - - Name: suffix - Value: .tiff - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: hina-microscope/ - - Name: suffix - Value: .nd2 - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: spectramax-id3-plate-reader/ - - Name: suffix - Value: .xls - Function: !GetAtt DataHubFunction.Arn - - Event: s3:ObjectCreated:* - Filter: - S3Key: - Rules: - - Name: prefix - Value: spectramax-id5-plate-reader/ - - Name: suffix - Value: .xls Function: !GetAtt DataHubFunction.Arn Tags: - Key: project diff --git a/lambda/src/data_hub_lambda/agilent_4150_tapestation/process_file.py b/lambda/src/data_hub_lambda/agilent_4150_tapestation/process_file.py index 9dd9283e..26717e25 100644 --- a/lambda/src/data_hub_lambda/agilent_4150_tapestation/process_file.py +++ b/lambda/src/data_hub_lambda/agilent_4150_tapestation/process_file.py @@ -5,17 +5,15 @@ from data_hub_lambda.api_client import get_client from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.AGILENT_4150_TAPESTATION.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Agilent 4150 TapeStation file through the Data Hub API. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID (`YYYY-MM-DD - HH-MM-SS` prefix). filename: The original filename (e.g. `2026-02-18 - 18-00-04-gDNA_peakTable.csv`). """ @@ -23,12 +21,12 @@ def process_file(run_id: str, filename: str) -> None: client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - client.ensure_run(INSTRUMENT_ID, run_id) + client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -39,7 +37,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) @@ -50,7 +48,7 @@ def process_file(run_id: str, filename: str) -> None: metadata["Tape Type"] = tape_type logger.info("Parsed metadata: %s", metadata) - client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) + client.update_run(instrument_id, run_id, metadata=metadata) client.update_file(file_id, status="completed") logger.info("File %s marked as completed.", filename) diff --git a/lambda/src/data_hub_lambda/akta_fplc/process_file.py b/lambda/src/data_hub_lambda/akta_fplc/process_file.py index 0b7a67df..651f7eeb 100644 --- a/lambda/src/data_hub_lambda/akta_fplc/process_file.py +++ b/lambda/src/data_hub_lambda/akta_fplc/process_file.py @@ -4,17 +4,15 @@ from data_hub_lambda.api_client import get_client from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.AKTA_FPLC.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Akta FPLC file through the Data Hub API. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID (filename stem). filename: The original filename (e.g. `2025-09-23_test.pdf`). """ @@ -22,12 +20,12 @@ def process_file(run_id: str, filename: str) -> None: client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - client.ensure_run(INSTRUMENT_ID, run_id) + client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -38,7 +36,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) diff --git a/lambda/src/data_hub_lambda/api_client.py b/lambda/src/data_hub_lambda/api_client.py index 04be76fb..bf4a9fca 100644 --- a/lambda/src/data_hub_lambda/api_client.py +++ b/lambda/src/data_hub_lambda/api_client.py @@ -1,11 +1,17 @@ from __future__ import annotations import logging import os +import time from typing import Any import requests -from data_hub_lambda.models import ApiErrorDetail, FileResponse, RunResponse +from data_hub_lambda.models import ( + ApiErrorDetail, + FileResponse, + InstrumentResponse, + RunResponse, +) logger = logging.getLogger(__name__) @@ -29,6 +35,21 @@ def __init__( # because the runs and files endpoints perform upsert queries under the hood. DEFAULT_TIMEOUT: tuple[float, float] = (5, 30) +# One short in-invocation retry for blips. Longer outages rely on Lambda's +# async retries for S3 events — those waits are free; time.sleep here bills +# at the function's full memory size. +_GET_INSTRUMENT_ATTEMPTS = 2 +_GET_INSTRUMENT_BACKOFF_SECONDS = (0.5,) + +# Warm-container cache: multi-file runs hit the same instrument repeatedly. +_INSTRUMENT_CACHE_TTL_SECONDS = 60.0 +_instrument_cache: dict[str, tuple[float, InstrumentResponse]] = {} + + +def clear_instrument_cache() -> None: + """Drop cached instrument lookups (tests / type edits mid-invocation).""" + _instrument_cache.clear() + class DataHubClient: """HTTP client for the Data Hub API (Lambda caller).""" @@ -89,6 +110,54 @@ def _request( self._handle_error(resp) return resp + # ------------------------------------------------------------------ + # Instruments + # ------------------------------------------------------------------ + + def get_instrument(self, instrument_id: str) -> InstrumentResponse: + """Fetch an instrument by ID, with one short retry on transient errors. + + Retries connection errors, timeouts, and 5xx once after a brief sleep. + 404 and 401/403 are raised immediately so the handler can classify + them. Exhausted transient failures are re-raised so S3-triggered + invocations can use Lambda's async retries (unbilled backoff). + Successful responses are cached for ``_INSTRUMENT_CACHE_TTL_SECONDS`` + so a multi-file run does not re-fetch the same instrument per file. + """ + now = time.monotonic() + cached = _instrument_cache.get(instrument_id) + if cached is not None: + cached_at, instrument = cached + if now - cached_at < _INSTRUMENT_CACHE_TTL_SECONDS: + return instrument + + last_error: ApiError | None = None + for attempt in range(_GET_INSTRUMENT_ATTEMPTS): + try: + resp = self._request("GET", f"/instruments/{instrument_id}") + instrument = InstrumentResponse.model_validate(resp.json()) + _instrument_cache[instrument_id] = (time.monotonic(), instrument) + return instrument + except ApiError as exc: + last_error = exc + is_transient = exc.status_code == 0 or exc.status_code >= 500 + if not is_transient or attempt == _GET_INSTRUMENT_ATTEMPTS - 1: + raise + delay = _GET_INSTRUMENT_BACKOFF_SECONDS[attempt] + logger.warning( + "Transient error fetching instrument %s (attempt %d/%d): %s; retrying in %.1fs", + instrument_id, + attempt + 1, + _GET_INSTRUMENT_ATTEMPTS, + exc, + delay, + ) + time.sleep(delay) + # The loop always returns or raises; keep a real raise for -O. + if last_error is None: + raise RuntimeError("get_instrument retry loop completed without result") + raise last_error + # ------------------------------------------------------------------ # Runs # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/azure_600_gel_doc/process_file.py b/lambda/src/data_hub_lambda/azure_600_gel_doc/process_file.py index 3b6af421..35e9dd6b 100644 --- a/lambda/src/data_hub_lambda/azure_600_gel_doc/process_file.py +++ b/lambda/src/data_hub_lambda/azure_600_gel_doc/process_file.py @@ -6,14 +6,11 @@ from data_hub_lambda.azure_600_gel_doc.parse_metadata import parse_metadata from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.AZURE_600_GEL_DOC.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Azure 600 Gel Doc file through the Data Hub API. Downloads the raw TIFF, runs it through the image processing pipeline @@ -21,6 +18,7 @@ def process_file(run_id: str, filename: str) -> None: S3 bucket, and registers both files via the API. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID (filename stem). filename: The original filename (e.g. `26.04.01_16.51.59.tif`). """ @@ -28,12 +26,12 @@ def process_file(run_id: str, filename: str) -> None: client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - client.ensure_run(INSTRUMENT_ID, run_id) + client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -44,7 +42,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) @@ -54,12 +52,12 @@ def process_file(run_id: str, filename: str) -> None: png_file_path = tiff_processor.export_figure() processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET - png_s3_key = f"{INSTRUMENT_ID}/{run_id}/{png_file_path.name}" + png_s3_key = f"{instrument_id}/{run_id}/{png_file_path.name}" s3_utils.upload_file(png_file_path, f"s3://{processed_bucket}/{png_s3_key}") logger.info("Uploaded processed image to s3://%s/%s", processed_bucket, png_s3_key) processed_file = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=processed_bucket or "", s3_key=png_s3_key, @@ -75,7 +73,7 @@ def process_file(run_id: str, filename: str) -> None: metadata = parse_metadata(local_file_path) logger.info("Parsed metadata: %s", metadata) - client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) + client.update_run(instrument_id, run_id, metadata=metadata) client.update_file(file_id, status="completed") logger.info("File %s marked as completed.", filename) diff --git a/lambda/src/data_hub_lambda/azure_cielo_qpcr/process_file.py b/lambda/src/data_hub_lambda/azure_cielo_qpcr/process_file.py index 8f7a1b61..5f3d0cb9 100644 --- a/lambda/src/data_hub_lambda/azure_cielo_qpcr/process_file.py +++ b/lambda/src/data_hub_lambda/azure_cielo_qpcr/process_file.py @@ -5,33 +5,31 @@ from data_hub_lambda.azure_cielo_qpcr.parse_dye_channels import parse_dye_channels from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.AZURE_CIELO_QPCR.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Azure Cielo qPCR file through the Data Hub API. For Cq Values CSV files, the unique dye channel names are extracted from the `Fluorescence` column and stored as run-level metadata. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID (`Experiment_YYYYMMDD` prefix). - filename: The original filename (e.g. `Experiment_20260101_CqValues.csv`). + filename: The original filename (e.g. `Experiment_20260101_Cq Values.csv`). """ logger.info("Processing Azure Cielo qPCR file: %s (run: %s)", filename, run_id) client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - client.ensure_run(INSTRUMENT_ID, run_id) + client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -42,7 +40,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) @@ -53,7 +51,7 @@ def process_file(run_id: str, filename: str) -> None: metadata["dye_channels"] = dye_channels logger.info("Parsed dye channels: %s", dye_channels) - client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) + client.update_run(instrument_id, run_id, metadata=metadata) client.update_file(file_id, status="completed") logger.info("File %s marked as completed.", filename) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 6d7405f8..1b6c30e5 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -306,14 +306,9 @@ def handler( from data_hub_lambda.handler import lambda_handler from data_hub_lambda.local_s3_mirror import make_mock_context, patched_s3 - from data_hub_shared.enums import Instrument - valid_ids = sorted(member.value for member in Instrument) - if instrument_id not in valid_ids: - raise click.BadParameter( - f"Unknown instrument_id '{instrument_id}'. Valid values: {', '.join(valid_ids)}", - param_hint="INSTRUMENT_ID", - ) + # Instrument existence / type is resolved by the handler via the API — + # an unknown ID no-ops there rather than failing CLI argument validation. for var in ("DATA_HUB_API_URL", "DATA_HUB_API_KEY"): if not os.environ.get(var): diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 10c36c3f..73a76664 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -9,14 +9,11 @@ from data_hub_lambda.epson_v700_scanner.image_processing import TiffProcessor from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.EPSON_V700_SCANNER.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Epson V700 Scanner file through the Data Hub API. Downloads the raw TIFF, resizes it to a web-friendly JPEG, uploads the @@ -24,6 +21,7 @@ def process_file(run_id: str, filename: str) -> None: both files via the API. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID. filename: The original filename (e.g. ``scan_001.tif``). """ @@ -31,12 +29,12 @@ def process_file(run_id: str, filename: str) -> None: client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - client.ensure_run(INSTRUMENT_ID, run_id) + client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -47,7 +45,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) @@ -66,12 +64,12 @@ def process_file(run_id: str, filename: str) -> None: ) processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET - jpg_s3_key = f"{INSTRUMENT_ID}/{run_id}/{jpg_file_path.name}" + jpg_s3_key = f"{instrument_id}/{run_id}/{jpg_file_path.name}" s3_utils.upload_file(jpg_file_path, f"s3://{processed_bucket}/{jpg_s3_key}") logger.info("Uploaded processed image to s3://%s/%s", processed_bucket, jpg_s3_key) processed_file = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=processed_bucket or "", s3_key=jpg_s3_key, @@ -92,10 +90,10 @@ def process_file(run_id: str, filename: str) -> None: pipeline.to_dataframes(), raw_data_dir / csv_name, ) - csv_s3_key = f"{INSTRUMENT_ID}/{run_id}/{csv_name}" + csv_s3_key = f"{instrument_id}/{run_id}/{csv_name}" s3_utils.upload_file(csv_path, f"s3://{processed_bucket}/{csv_s3_key}") csv_file = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=processed_bucket or "", s3_key=csv_s3_key, @@ -112,7 +110,7 @@ def process_file(run_id: str, filename: str) -> None: logger.info("Parsed metadata: %s", metadata) - client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) + client.update_run(instrument_id, run_id, metadata=metadata) client.update_file(file_id, status="completed") logger.info("File %s marked as completed.", filename) diff --git a/lambda/src/data_hub_lambda/handler.py b/lambda/src/data_hub_lambda/handler.py index 72e9cfa7..465bd877 100644 --- a/lambda/src/data_hub_lambda/handler.py +++ b/lambda/src/data_hub_lambda/handler.py @@ -11,17 +11,9 @@ from aws_lambda_typing.context import Context from aws_lambda_typing.events.s3 import S3Event -from data_hub_lambda import ( - agilent_4150_tapestation, - akta_fplc, - archive_builder, - azure_600_gel_doc, - azure_cielo_qpcr, - epson_v700_scanner, - hina_microscope, - spectramax_plate_reader, -) -from data_hub_shared.enums import Instrument +from data_hub_lambda import archive_builder +from data_hub_lambda.api_client import ApiError, get_client +from data_hub_lambda.processors import get_processor, matches_any_processor_gate from data_hub_shared.logger import get_named_logger logger = get_named_logger(__name__) @@ -52,7 +44,7 @@ def parse_s3_event(event: S3Event) -> S3EventInfo: An `S3EventInfo` with all fields populated. Raises: - ValueError: If the event payload is malformed or the instrument is unsupported. + ValueError: If the event payload is malformed or the key shape is wrong. """ record = event["Records"][0] if not record: @@ -67,19 +59,12 @@ def parse_s3_event(event: S3Event) -> S3EventInfo: if not match: raise ValueError(f"Object key does not match expected pattern: {s3_key}") - instrument_id = match.group(1) - run_id = match.group(2) - filename = match.group(3) - - if instrument_id not in {member.value for member in Instrument}: - raise ValueError(f"This instrument is not currently supported: {instrument_id}") - return S3EventInfo( - instrument_id=instrument_id, - run_id=run_id, + instrument_id=match.group(1), + run_id=match.group(2), s3_bucket=s3_bucket, s3_key=s3_key, - filename=filename, + filename=match.group(3), ) @@ -270,19 +255,74 @@ def _post_archive_job_status( logger.exception("Failed to PATCH archive-job %s", job_id) +# ------------------------------------------------------------------ +# Reprocess failure recovery +# ------------------------------------------------------------------ + + +def _fail_reprocess_file( + instrument_id: str, + run_id: str, + filename: str, + error_message: str, +) -> None: + """PATCH the file to failed so a Function URL no-op can't leave it in processing. + + The web app transitions the file to ``processing`` before invoking the + Function URL. Resolves the row via the idempotent ``create_file`` upsert + (returns the existing record) and marks it failed. Failures here are + logged but not re-raised — the caller has already decided not to process. + """ + from data_hub_shared.config import config + + try: + client = get_client() + s3_bucket = config.AWS_S3_RAW_DATA_BUCKET or "" + s3_key = f"{instrument_id}/{run_id}/{filename}" + file_record = client.create_file( + instrument_id=instrument_id, + run_id=run_id, + s3_bucket=s3_bucket, + s3_key=s3_key, + filename=filename, + ) + client.update_file( + file_record.id, + status="failed", + error_message=error_message, + ) + except Exception: + logger.exception( + "Failed to mark reprocess file %s/%s/%s as failed", + instrument_id, + run_id, + filename, + ) + + # ------------------------------------------------------------------ # Main handler # ------------------------------------------------------------------ def lambda_handler(event: dict[str, Any], context: Context) -> dict[str, Any] | None: - """Top-level Lambda handler dispatching to instrument workflows.""" + """Top-level Lambda handler dispatching to instrument workflows. + + Dispatch is by ``instrument_type`` (fetched from the API), not instrument + ID. Filename gates filter the S3 firehose; explicit reprocess via the + Function URL bypasses those gates so user-initiated work can't strand + a file in ``processing``. + """ logger.info("Received event: %s", pformat(event)) + # Capture before unwrapping so reprocess (Function URL) can skip gates. + is_function_url = _is_function_url_event(event) + apply_filename_gates = not is_function_url + # Function URL invocations carry a requestContext with an http key. # AWS_IAM auth is enforced by Lambda before the handler runs, so we # only need to unwrap the inner JSON payload here. - if _is_function_url_event(event): + if is_function_url: payload = _parse_function_url_body(event) if payload is None: return {"statusCode": 400, "body": "Invalid JSON body"} @@ -304,67 +344,91 @@ def lambda_handler(event: dict[str, Any], context: Context) -> dict[str, Any] | instrument_id = event_info.instrument_id run_id = event_info.run_id + filename = event_info.filename logger.info("Instrument ID: '%s'", instrument_id) logger.info("Run ID: '%s'", run_id) - # Pre-cleanup: if the previous invocation on this warm container was - # SIGKILL'd (e.g. OOM), the `finally` block below didn't run and stale - # downloads may still be sitting in /tmp. Wipe them before we start. + # Pre-cleanup before the filename gate so warm containers recover from + # a prior OOM even when the catch-all notification mostly no-ops. _cleanup_tmp() try: - logger.info("Processing file %s...", event_info.filename) - - if instrument_id == Instrument.AKTA_FPLC.value: - akta_fplc.process_file( - run_id=event_info.run_id, - filename=event_info.filename, - ) - - elif instrument_id == Instrument.AGILENT_4150_TAPESTATION.value: - agilent_4150_tapestation.process_file( - run_id=event_info.run_id, - filename=event_info.filename, - ) - - elif instrument_id == Instrument.AZURE_600_GEL_DOC.value: - azure_600_gel_doc.process_file( - run_id=event_info.run_id, - filename=event_info.filename, - ) - - elif instrument_id == Instrument.AZURE_CIELO_QPCR.value: - azure_cielo_qpcr.process_file( - run_id=event_info.run_id, - filename=event_info.filename, + # Skip the API call when no processor could possibly want this filename. + if apply_filename_gates and not matches_any_processor_gate(filename): + logger.info( + "No processor filename gate matches %s; skipping.", + filename, ) + return None - elif instrument_id == Instrument.EPSON_V700_SCANNER.value: - epson_v700_scanner.process_file( - run_id=event_info.run_id, - filename=event_info.filename, + try: + instrument = get_client().get_instrument(instrument_id) + except ApiError as exc: + if exc.status_code == 404: + logger.info( + "Instrument %s not found; skipping %s.", + instrument_id, + filename, + ) + if is_function_url: + _fail_reprocess_file( + instrument_id, + run_id, + filename, + f"Instrument '{instrument_id}' not found", + ) + return None + if exc.status_code in (401, 403): + logger.error( + "Auth failure fetching instrument %s (status %d): %s. " + "Check that DATA_HUB_API_KEY includes instruments:read.", + instrument_id, + exc.status_code, + exc.message, + ) + raise + # Transient errors already retried inside get_instrument; re-raise + # so the async S3 path can retry the invocation. + logger.error( + "Failed to fetch instrument %s after retries (status %d): %s", + instrument_id, + exc.status_code, + exc.message, ) - - elif instrument_id == Instrument.HINA_MICROSCOPE.value: - hina_microscope.process_file( - run_id=event_info.run_id, - filename=event_info.filename, + raise + + processor = get_processor(instrument.instrument_type) + if processor is None: + message = f"No Lambda processor for instrument_type='{instrument.instrument_type}'" + logger.info( + "%s (instrument %s); skipping %s.", + message, + instrument_id, + filename, ) + if is_function_url: + _fail_reprocess_file(instrument_id, run_id, filename, message) + return None - elif instrument_id in ( - Instrument.SPECTRAMAX_ID3_PLATE_READER.value, - Instrument.SPECTRAMAX_ID5_PLATE_READER.value, - ): - spectramax_plate_reader.process_file( - instrument_id=event_info.instrument_id, # pyright: ignore[reportArgumentType] - run_id=event_info.run_id, - filename=event_info.filename, + if apply_filename_gates and not processor.matches_filename(filename): + logger.info( + "Filename %s does not match gate for instrument_type=%s; skipping.", + filename, + instrument.instrument_type, ) - - else: - logger.error("Unsupported instrument: %s", instrument_id) return None + logger.info( + "Processing file %s with instrument_type=%s...", + filename, + instrument.instrument_type, + ) + processor.process_file(instrument_id, run_id, filename) + + except ApiError: + # Auth / exhausted-retry failures: re-raise so Lambda retries (S3) + # or returns 500 (Function URL). Do not swallow. + raise except Exception: # Per-file failure is already PATCHed back to the web app's file row # (status='failed', error_message=...) by each instrument's diff --git a/lambda/src/data_hub_lambda/hina_microscope/process_file.py b/lambda/src/data_hub_lambda/hina_microscope/process_file.py index b027c71f..c6f3a44c 100644 --- a/lambda/src/data_hub_lambda/hina_microscope/process_file.py +++ b/lambda/src/data_hub_lambda/hina_microscope/process_file.py @@ -6,14 +6,11 @@ from data_hub_lambda.hina_microscope.parse_metadata import parse_metadata from data_hub_shared import s3_utils from data_hub_shared.config import config -from data_hub_shared.enums import Instrument logger = logging.getLogger(__name__) -INSTRUMENT_ID = Instrument.HINA_MICROSCOPE.value - -def process_file(run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single Hina microscope ND2 file through the Data Hub API. Downloads the raw ND2, runs it through the image processing pipeline to @@ -24,6 +21,7 @@ def process_file(run_id: str, filename: str) -> None: skip the metadata step. Args: + instrument_id: The instrument ID from the S3 key / event. run_id: The run ID (grouping key for files in a single imaging session). filename: The original filename (e.g. `well_A1_xy01.nd2`). """ @@ -31,12 +29,12 @@ def process_file(run_id: str, filename: str) -> None: client = get_client() s3_bucket = config.AWS_S3_RAW_DATA_BUCKET - s3_key = f"{INSTRUMENT_ID}/{run_id}/{filename}" + s3_key = f"{instrument_id}/{run_id}/{filename}" - run = client.ensure_run(INSTRUMENT_ID, run_id) + run = client.ensure_run(instrument_id, run_id) file_record = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=s3_bucket or "", s3_key=s3_key, @@ -47,7 +45,7 @@ def process_file(run_id: str, filename: str) -> None: try: client.update_file(file_id, status="processing") - raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / INSTRUMENT_ID / run_id + raw_data_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id local_file_path = raw_data_dir / filename s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) @@ -57,12 +55,12 @@ def process_file(run_id: str, filename: str) -> None: jpg_file_path = processor.export_jpg() processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET - jpg_s3_key = f"{INSTRUMENT_ID}/{run_id}/{jpg_file_path.name}" + jpg_s3_key = f"{instrument_id}/{run_id}/{jpg_file_path.name}" s3_utils.upload_file(jpg_file_path, f"s3://{processed_bucket}/{jpg_s3_key}") logger.info("Uploaded processed image to s3://%s/%s", processed_bucket, jpg_s3_key) processed_file = client.create_file( - instrument_id=INSTRUMENT_ID, + instrument_id=instrument_id, run_id=run_id, s3_bucket=processed_bucket or "", s3_key=jpg_s3_key, @@ -80,7 +78,7 @@ def process_file(run_id: str, filename: str) -> None: # parse and store it once — on the first file to arrive. if not run.metadata: metadata = parse_metadata(processor.image) - client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) + client.update_run(instrument_id, run_id, metadata=metadata) logger.info("Parsed and stored run-level metadata for %s", run_id) else: logger.info("Run %s already has metadata; skipping metadata step.", run_id) diff --git a/lambda/src/data_hub_lambda/models.py b/lambda/src/data_hub_lambda/models.py index 6a7e2a6a..33cb4015 100644 --- a/lambda/src/data_hub_lambda/models.py +++ b/lambda/src/data_hub_lambda/models.py @@ -16,6 +16,15 @@ class ApiErrorDetail(BaseModel): details: dict | None = None +class InstrumentResponse(BaseModel): + model_config = _API_MODEL_CONFIG + + id: str + display_name: str + status: str + instrument_type: str + + class RunResponse(BaseModel): model_config = _API_MODEL_CONFIG diff --git a/lambda/src/data_hub_lambda/processors.py b/lambda/src/data_hub_lambda/processors.py new file mode 100644 index 00000000..595f6b4f --- /dev/null +++ b/lambda/src/data_hub_lambda/processors.py @@ -0,0 +1,83 @@ +"""Type → processor registry for Lambda file dispatch. + +One `instrument_type` maps to one vendor-specific processor. Types that +sound generic (`qpcr`, `plate_reader`, `gel_doc`, `tape_station`, `fplc`) +still bind to a single vendor's output format — adding a second vendor +requires splitting the type, not reusing it. +""" + +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass + +from data_hub_lambda import ( + agilent_4150_tapestation, + akta_fplc, + azure_600_gel_doc, + azure_cielo_qpcr, + epson_v700_scanner, + hina_microscope, + spectramax_plate_reader, +) + +ProcessFileFn = Callable[[str, str, str], None] + + +@dataclass(frozen=True) +class ProcessorEntry: + """A processor and the filename gate that selects it for S3 events.""" + + process_file: ProcessFileFn + matches_filename: Callable[[str], bool] + + +def _ends_with_any(*suffixes: str) -> Callable[[str], bool]: + lowers = tuple(s.lower() for s in suffixes) + + def _match(filename: str) -> bool: + name = filename.lower() + return any(name.endswith(s) for s in lowers) + + return _match + + +PROCESSORS: dict[str, ProcessorEntry] = { + "qpcr": ProcessorEntry( + process_file=azure_cielo_qpcr.process_file, + matches_filename=_ends_with_any("_cq values.csv"), + ), + "plate_reader": ProcessorEntry( + process_file=spectramax_plate_reader.process_file, + matches_filename=_ends_with_any(".xls"), + ), + "gel_doc": ProcessorEntry( + process_file=azure_600_gel_doc.process_file, + matches_filename=_ends_with_any(".tif", ".tiff"), + ), + "tape_station": ProcessorEntry( + process_file=agilent_4150_tapestation.process_file, + matches_filename=_ends_with_any(".pdf"), + ), + "hina_microscope": ProcessorEntry( + process_file=hina_microscope.process_file, + matches_filename=_ends_with_any(".nd2"), + ), + "epson_v700_scanner": ProcessorEntry( + process_file=epson_v700_scanner.process_file, + matches_filename=_ends_with_any(".tif", ".tiff"), + ), + "fplc": ProcessorEntry( + process_file=akta_fplc.process_file, + matches_filename=_ends_with_any(".pdf"), + ), +} + + +def matches_any_processor_gate(filename: str) -> bool: + """True if *filename* passes at least one processor's filename gate.""" + return any(entry.matches_filename(filename) for entry in PROCESSORS.values()) + + +def get_processor(instrument_type: str) -> ProcessorEntry | None: + """Return the registry entry for *instrument_type*, or None if unmapped.""" + return PROCESSORS.get(instrument_type) diff --git a/lambda/src/data_hub_lambda/spectramax_plate_reader/process_file.py b/lambda/src/data_hub_lambda/spectramax_plate_reader/process_file.py index a3a877d2..e3579a70 100644 --- a/lambda/src/data_hub_lambda/spectramax_plate_reader/process_file.py +++ b/lambda/src/data_hub_lambda/spectramax_plate_reader/process_file.py @@ -1,6 +1,5 @@ from __future__ import annotations import logging -from typing import Literal from data_hub_lambda.api_client import get_client from data_hub_lambda.spectramax_plate_reader.utils import parse_metadata, parse_raw_well_data @@ -9,10 +8,8 @@ logger = logging.getLogger(__name__) -InstrumentType = Literal["spectramax-id3-plate-reader", "spectramax-id5-plate-reader"] - -def process_file(instrument_id: InstrumentType, run_id: str, filename: str) -> None: +def process_file(instrument_id: str, run_id: str, filename: str) -> None: """Process a single SpectraMax plate reader file through the Data Hub API. Args: diff --git a/lambda/tests/epson_v700_scanner/test_process_file.py b/lambda/tests/epson_v700_scanner/test_process_file.py index cbdc80cd..50593b36 100644 --- a/lambda/tests/epson_v700_scanner/test_process_file.py +++ b/lambda/tests/epson_v700_scanner/test_process_file.py @@ -104,7 +104,11 @@ def test_creates_run_and_files( ): from data_hub_lambda.epson_v700_scanner.process_file import process_file - process_file(run_id="run-xyz", filename="scan.tif") + process_file( + instrument_id="epson-v700-scanner", + run_id="run-xyz", + filename="scan.tif", + ) client.ensure_run.assert_called_once() assert client.create_file.call_count == 2 @@ -127,7 +131,11 @@ def test_uploads_jpg_and_registers_processed_file( ): from data_hub_lambda.epson_v700_scanner.process_file import process_file - process_file(run_id="run-xyz", filename="scan.tif") + process_file( + instrument_id="epson-v700-scanner", + run_id="run-xyz", + filename="scan.tif", + ) patched_converter.export_jpg.assert_called_once() @@ -151,7 +159,11 @@ def test_stores_metadata_on_run( ): from data_hub_lambda.epson_v700_scanner.process_file import process_file - process_file(run_id="run-xyz", filename="scan.tif") + process_file( + instrument_id="epson-v700-scanner", + run_id="run-xyz", + filename="scan.tif", + ) client.update_run.assert_called_once() _, kwargs = client.update_run.call_args @@ -173,7 +185,11 @@ def test_marks_raw_file_completed( ): from data_hub_lambda.epson_v700_scanner.process_file import process_file - process_file(run_id="run-xyz", filename="scan.tif") + process_file( + instrument_id="epson-v700-scanner", + run_id="run-xyz", + filename="scan.tif", + ) statuses = [ call.kwargs.get("status") @@ -201,7 +217,11 @@ def test_marks_raw_file_failed_on_exception(self) -> None: from data_hub_lambda.epson_v700_scanner.process_file import process_file with pytest.raises(RuntimeError, match="boom"): - process_file(run_id="run-xyz", filename="broken.tif") + process_file( + instrument_id="epson-v700-scanner", + run_id="run-xyz", + filename="broken.tif", + ) statuses = [ call.kwargs.get("status") diff --git a/lambda/tests/hina_microscope/test_process_file.py b/lambda/tests/hina_microscope/test_process_file.py index d2a7d619..ef6f8fd6 100644 --- a/lambda/tests/hina_microscope/test_process_file.py +++ b/lambda/tests/hina_microscope/test_process_file.py @@ -102,7 +102,11 @@ def test_first_file_parses_metadata_and_updates_run( ): from data_hub_lambda.hina_microscope.process_file import process_file - process_file(run_id="run-xyz", filename="sample.nd2") + process_file( + instrument_id="hina-microscope", + run_id="run-xyz", + filename="sample.nd2", + ) # First file → metadata is parsed and persisted. parse_mock.assert_called_once_with(patched_processor.image) @@ -140,7 +144,11 @@ def test_later_file_skips_metadata_but_still_generates_jpg( ): from data_hub_lambda.hina_microscope.process_file import process_file - process_file(run_id="run-xyz", filename="sample-2.nd2") + process_file( + instrument_id="hina-microscope", + run_id="run-xyz", + filename="sample-2.nd2", + ) # Second file → metadata step is skipped. parse_mock.assert_not_called() @@ -171,7 +179,11 @@ def test_marks_raw_file_failed_on_exception(self, tmp_path: Path) -> None: from data_hub_lambda.hina_microscope.process_file import process_file with pytest.raises(RuntimeError, match="boom"): - process_file(run_id="run-xyz", filename="broken.nd2") + process_file( + instrument_id="hina-microscope", + run_id="run-xyz", + filename="broken.nd2", + ) # File should be transitioned through processing → failed with the error message. statuses = [ diff --git a/lambda/tests/integration/conftest.py b/lambda/tests/integration/conftest.py index 873b2a00..b0c1737f 100644 --- a/lambda/tests/integration/conftest.py +++ b/lambda/tests/integration/conftest.py @@ -28,8 +28,8 @@ # Constants # --------------------------------------------------------------------------- -# Instruments seeded at session scope — must match the kebab-case IDs used -# by the Python `Instrument` enum and the S3 key prefix convention. +# Instruments seeded at session scope. Types must match the Lambda +# processor registry keys in `data_hub_lambda.processors`. _INSTRUMENTS: dict[str, str] = { "azure-cielo-qpcr": "Azure Cielo qPCR", "azure-600-gel-doc": "Azure 600 Gel Doc", @@ -38,6 +38,14 @@ "spectramax-id5-plate-reader": "SpectraMax iD5 Plate Reader", } +_INSTRUMENT_TYPES: dict[str, str] = { + "azure-cielo-qpcr": "qpcr", + "azure-600-gel-doc": "gel_doc", + "hina-microscope": "hina_microscope", + "spectramax-id3-plate-reader": "plate_reader", + "spectramax-id5-plate-reader": "plate_reader", +} + # --------------------------------------------------------------------------- # Lambda-specific helpers @@ -74,7 +82,7 @@ def integration_env( """Start a real Next.js server, push the DB schema, and seed auth + instruments.""" with start_test_server() as env: - seed_instruments(env.db_dsn, _INSTRUMENTS) + seed_instruments(env.db_dsn, _INSTRUMENTS, instrument_types=_INSTRUMENT_TYPES) # Set env vars for Lambda modules and reset singletons so # DataHubClient / Config pick up the test server URL. diff --git a/lambda/tests/integration/test_lambda_api.py b/lambda/tests/integration/test_lambda_api.py index 73cbb4d6..d089db75 100644 --- a/lambda/tests/integration/test_lambda_api.py +++ b/lambda/tests/integration/test_lambda_api.py @@ -56,7 +56,7 @@ def test_csv_completes_with_dye_channels( ) -> None: # Register the real fixture CSV so the patched S3 download can find it. run_id = "Experiment_20260101" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" s3_fixture_files[s3_key] = _FIXTURES_DIR / "azure_cielo_qpcr_example.csv" @@ -260,7 +260,7 @@ def test_malformed_csv_marks_file_as_failed( bad_csv.write_text("Wrong,Headers,Only\nA,B,C\n") run_id = "Experiment_20260201" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" s3_fixture_files[s3_key] = bad_csv @@ -296,7 +296,7 @@ def test_duplicate_event_creates_single_run( mock_context: MagicMock, ) -> None: run_id = "Experiment_20260301" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" s3_fixture_files[s3_key] = _FIXTURES_DIR / "azure_cielo_qpcr_example.csv" @@ -354,7 +354,7 @@ def test_duplicate_event_creates_single_file( file via completed → processing → completed. """ run_id = "Experiment_20260301" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" s3_fixture_files[s3_key] = _FIXTURES_DIR / "azure_cielo_qpcr_example.csv" @@ -401,7 +401,7 @@ def test_reprocess_clears_failed_state( cleared. """ run_id = "Experiment_20260301" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" bad_csv = tmp_path / "bad.csv" bad_csv.write_text("Wrong,Headers,Only\nA,B,C\n") @@ -490,7 +490,7 @@ def test_happy_path_processes_file( """A Function URL event processes the file identically to a direct S3 trigger.""" run_id = "Experiment_20260401" - filename = f"{run_id}_CqValues.csv" + filename = f"{run_id}_Cq Values.csv" s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" s3_fixture_files[s3_key] = _FIXTURES_DIR / "azure_cielo_qpcr_example.csv" @@ -519,9 +519,71 @@ def test_invalid_json_body_returns_400( event = make_function_url_event( "azure-cielo-qpcr", "Experiment_20260401", - "Experiment_20260401_CqValues.csv", + "Experiment_20260401_Cq Values.csv", body_override="this is not json", ) result = lambda_handler(event, mock_context) assert result == {"statusCode": 400, "body": "Invalid JSON body"} + + +# ------------------------------------------------------------------ +# Least-privilege Lambda token (matches the curated Lambda scope preset) +# ------------------------------------------------------------------ + +# Keep in sync with the `lambda` entry in `web/lib/api/scope-catalog.ts`. +_LAMBDA_PRESET_SCOPES = [ + "instruments:read", + "runs:create", + "runs:update", + "files:create", + "files:update", + "archive-jobs:write", +] + + +class TestLeastPrivilegeLambdaScopes: + """A token minted with exactly the Lambda preset scopes can process a file. + + Catches scope-preset drift: if a new Lambda → API call needs a scope + that isn't on the preset, this test fails instead of only surfacing + in production against a freshly minted least-privilege token. + """ + + def test_preset_scopes_can_process_qpcr_file( + self, + integration_env: IntegrationEnv, + make_s3_event: Callable[..., dict[str, Any]], + s3_fixture_files: dict[str, Path], + mock_context: MagicMock, + ) -> None: + import os + + import data_hub_lambda.api_client as api_module + from data_hub_shared.testing import seed_auth + + scoped_token = seed_auth(integration_env.db_dsn, scopes=_LAMBDA_PRESET_SCOPES) + previous_key = os.environ["DATA_HUB_API_KEY"] + os.environ["DATA_HUB_API_KEY"] = scoped_token + api_module._client = None + + try: + run_id = "Experiment_20260501" + filename = f"{run_id}_Cq Values.csv" + s3_key = f"azure-cielo-qpcr/{run_id}/{filename}" + s3_fixture_files[s3_key] = _FIXTURES_DIR / "azure_cielo_qpcr_example.csv" + + event = make_s3_event("azure-cielo-qpcr", run_id, filename) + lambda_handler(event, mock_context) + + run = _api_get( + integration_env.base_url, + integration_env.api_token, + f"/api/v1/instruments/azure-cielo-qpcr/runs/{run_id}", + ) + assert run["run_id"] == run_id + assert len(run["files"]) == 1 + assert run["files"][0]["status"] == "completed" + finally: + os.environ["DATA_HUB_API_KEY"] = previous_key + api_module._client = None diff --git a/lambda/tests/test_processors.py b/lambda/tests/test_processors.py new file mode 100644 index 00000000..485e492f --- /dev/null +++ b/lambda/tests/test_processors.py @@ -0,0 +1,406 @@ +"""Unit tests for the type → processor registry and filename gates.""" + +from __future__ import annotations +import re +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from data_hub_lambda.api_client import ApiError, DataHubClient, clear_instrument_cache +from data_hub_lambda.handler import lambda_handler +from data_hub_lambda.models import FileResponse, InstrumentResponse +from data_hub_lambda.processors import ( + PROCESSORS, + get_processor, + matches_any_processor_gate, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROCESSABLE_TYPES_TS = REPO_ROOT / "web" / "lib" / "instruments" / "processable-types.ts" + + +class TestFilenameGates: + @pytest.mark.parametrize( + ("instrument_type", "filename", "expected"), + [ + ("qpcr", "Experiment_20260101_Cq Values.csv", True), + ("qpcr", "Experiment_20260101_cq values.CSV", True), + ("qpcr", "Experiment_20260101_CqValues.csv", False), + ("qpcr", "Experiment_20260101_Amplification Results.csv", False), + ("plate_reader", "plate.xls", True), + ("plate_reader", "PLATE.XLS", True), + ("plate_reader", "plate.xlsx", False), + ("gel_doc", "gel.tif", True), + ("gel_doc", "gel.tiff", True), + ("gel_doc", "gel.TIF", True), + ("gel_doc", "gel.png", False), + ("tape_station", "report.pdf", True), + ("tape_station", "peaks.csv", False), + ("hina_microscope", "well.nd2", True), + ("hina_microscope", "well.tif", False), + ("epson_v700_scanner", "scan.tif", True), + ("epson_v700_scanner", "scan.tiff", True), + ("epson_v700_scanner", "scan.jpg", False), + ("fplc", "chromatogram.pdf", True), + ("fplc", "notes.txt", False), + ], + ) + def test_per_type_gate(self, instrument_type: str, filename: str, expected: bool) -> None: + entry = PROCESSORS[instrument_type] + assert entry.matches_filename(filename) is expected + + def test_union_gate_matches_any_processor(self) -> None: + assert matches_any_processor_gate("Experiment_Cq Values.csv") + assert matches_any_processor_gate("scan.TIFF") + assert matches_any_processor_gate("well.nd2") + assert not matches_any_processor_gate("readme.txt") + assert not matches_any_processor_gate("notes.md") + + def test_unmapped_types_have_no_entry(self) -> None: + assert get_processor("generic") is None + assert get_processor("instant_raman") is None + assert get_processor("unknown_type") is None + + def test_processable_types_match_web_allowlist(self) -> None: + """Python registry keys must match PROCESSABLE_INSTRUMENT_TYPES.""" + text = PROCESSABLE_TYPES_TS.read_text() + match = re.search( + r"PROCESSABLE_INSTRUMENT_TYPES\s*=\s*\[(.*?)]\s*as const", + text, + re.DOTALL, + ) + assert match is not None, "Could not find PROCESSABLE_INSTRUMENT_TYPES in TS" + ts_types = set(re.findall(r'"([^"]+)"', match.group(1))) + assert ts_types == set(PROCESSORS) + + +class TestInstrumentCache: + def setup_method(self) -> None: + clear_instrument_cache() + + def teardown_method(self) -> None: + clear_instrument_cache() + + def test_get_instrument_caches_successful_lookup(self) -> None: + client = DataHubClient(base_url="https://example.test/api/v1") + payload = { + "id": "azure-cielo-qpcr", + "display_name": "Azure Cielo qPCR", + "status": "active", + "instrument_type": "qpcr", + } + resp = MagicMock() + resp.json.return_value = payload + with patch.object(client, "_request", return_value=resp) as request: + first = client.get_instrument("azure-cielo-qpcr") + second = client.get_instrument("azure-cielo-qpcr") + + assert first.instrument_type == "qpcr" + assert second is first + request.assert_called_once_with("GET", "/instruments/azure-cielo-qpcr") + + def test_get_instrument_does_not_cache_errors(self) -> None: + client = DataHubClient(base_url="https://example.test/api/v1") + with patch.object( + client, + "_request", + side_effect=ApiError("not found", status_code=404), + ) as request: + with pytest.raises(ApiError): + client.get_instrument("missing") + with pytest.raises(ApiError): + client.get_instrument("missing") + + assert request.call_count == 2 + + def test_get_instrument_retries_transient_once(self) -> None: + client = DataHubClient(base_url="https://example.test/api/v1") + payload = { + "id": "azure-cielo-qpcr", + "display_name": "Azure Cielo qPCR", + "status": "active", + "instrument_type": "qpcr", + } + ok = MagicMock() + ok.json.return_value = payload + with ( + patch.object( + client, + "_request", + side_effect=[ApiError("boom", status_code=503), ok], + ) as request, + patch("data_hub_lambda.api_client.time.sleep") as sleep, + ): + instrument = client.get_instrument("azure-cielo-qpcr") + + assert instrument.instrument_type == "qpcr" + assert request.call_count == 2 + sleep.assert_called_once_with(0.5) + + def test_get_instrument_does_not_sleep_through_long_outages(self) -> None: + client = DataHubClient(base_url="https://example.test/api/v1") + with ( + patch.object( + client, + "_request", + side_effect=ApiError("boom", status_code=503), + ) as request, + patch("data_hub_lambda.api_client.time.sleep") as sleep, + pytest.raises(ApiError) as exc_info, + ): + client.get_instrument("azure-cielo-qpcr") + + assert exc_info.value.status_code == 503 + assert request.call_count == 2 + sleep.assert_called_once_with(0.5) + + +class TestHandlerDispatch: + """Dispatch paths that used to be ID if/elif branches.""" + + def _s3_event(self, instrument_id: str, run_id: str, filename: str) -> dict: + from urllib.parse import quote_plus + + key = quote_plus(f"{instrument_id}/{run_id}/{filename}", safe="/") + return { + "Records": [ + { + "s3": { + "bucket": {"name": "test-bucket"}, + "object": {"key": key}, + } + } + ] + } + + def _function_url_event(self, instrument_id: str, run_id: str, filename: str) -> dict: + import json + + return { + "version": "2.0", + "requestContext": {"http": {"method": "POST", "path": "/"}}, + "body": json.dumps(self._s3_event(instrument_id, run_id, filename)), + "isBase64Encoded": False, + } + + def _file_response(self, file_id: int = 42) -> FileResponse: + return FileResponse( + id=file_id, + instrument_run_id="run-uuid", + filename="file.csv", + category="raw", + status="processing", + ) + + def test_union_gate_short_circuits_without_api_call(self) -> None: + client = MagicMock() + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + ): + result = lambda_handler( + self._s3_event("azure-cielo-qpcr", "run-1", "readme.txt"), + MagicMock(), + ) + + assert result is None + client.get_instrument.assert_not_called() + + def test_unmapped_type_is_noop_on_s3(self) -> None: + client = MagicMock() + client.get_instrument.return_value = InstrumentResponse( + id="instantraman", + display_name="InstantRaman", + status="active", + instrument_type="instant_raman", + ) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + ): + result = lambda_handler( + self._s3_event("instantraman", "run-1", "scan.tif"), + MagicMock(), + ) + + assert result is None + client.get_instrument.assert_called_once_with("instantraman") + client.create_file.assert_not_called() + + def test_generic_type_is_noop_on_s3(self) -> None: + client = MagicMock() + client.get_instrument.return_value = InstrumentResponse( + id="jolene-fplc", + display_name="Jolene FPLC", + status="active", + instrument_type="generic", + ) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + ): + result = lambda_handler( + self._s3_event("jolene-fplc", "run-1", "chromatogram.pdf"), + MagicMock(), + ) + + assert result is None + client.create_file.assert_not_called() + + def test_404_is_noop_on_s3(self) -> None: + client = MagicMock() + client.get_instrument.side_effect = ApiError("not found", status_code=404) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + ): + result = lambda_handler( + self._s3_event("missing-instrument", "run-1", "scan.tif"), + MagicMock(), + ) + + assert result is None + client.create_file.assert_not_called() + + def test_reprocess_marks_failed_when_type_unmapped(self) -> None: + client = MagicMock() + client.get_instrument.return_value = InstrumentResponse( + id="instantraman", + display_name="InstantRaman", + status="active", + instrument_type="instant_raman", + ) + client.create_file.return_value = self._file_response(7) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "test-bucket", + ), + ): + result = lambda_handler( + self._function_url_event("instantraman", "run-1", "scan.tif"), + MagicMock(), + ) + + assert result is None + client.create_file.assert_called_once() + client.update_file.assert_called_once_with( + 7, + status="failed", + error_message="No Lambda processor for instrument_type='instant_raman'", + ) + + def test_reprocess_marks_failed_on_instrument_404(self) -> None: + client = MagicMock() + client.get_instrument.side_effect = ApiError("not found", status_code=404) + client.create_file.return_value = self._file_response(9) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "test-bucket", + ), + ): + result = lambda_handler( + self._function_url_event("missing-instrument", "run-1", "scan.tif"), + MagicMock(), + ) + + assert result is None + client.update_file.assert_called_once_with( + 9, + status="failed", + error_message="Instrument 'missing-instrument' not found", + ) + + def test_403_raises(self) -> None: + client = MagicMock() + client.get_instrument.side_effect = ApiError("forbidden", status_code=403) + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + pytest.raises(ApiError) as exc_info, + ): + lambda_handler( + self._s3_event("azure-cielo-qpcr", "run-1", "x_Cq Values.csv"), + MagicMock(), + ) + + assert exc_info.value.status_code == 403 + + def test_reprocess_bypasses_filename_gate(self) -> None: + client = MagicMock() + client.get_instrument.return_value = InstrumentResponse( + id="azure-cielo-qpcr", + display_name="Azure Cielo qPCR", + status="active", + instrument_type="qpcr", + ) + process_file = MagicMock() + entry = MagicMock() + entry.matches_filename.return_value = False + entry.process_file = process_file + + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + patch( + "data_hub_lambda.handler.get_processor", + return_value=entry, + ), + # Union gate would also block S3 events; reprocess must skip it. + patch( + "data_hub_lambda.handler.matches_any_processor_gate", + return_value=False, + ), + ): + result = lambda_handler( + self._function_url_event( + "azure-cielo-qpcr", + "run-1", + "Experiment_Amplification Results.csv", + ), + MagicMock(), + ) + + assert result is None + process_file.assert_called_once_with( + "azure-cielo-qpcr", + "run-1", + "Experiment_Amplification Results.csv", + ) + + def test_s3_event_applies_per_type_gate(self) -> None: + client = MagicMock() + client.get_instrument.return_value = InstrumentResponse( + id="azure-cielo-qpcr", + display_name="Azure Cielo qPCR", + status="active", + instrument_type="qpcr", + ) + process_file = MagicMock() + entry = MagicMock() + entry.matches_filename.return_value = False + entry.process_file = process_file + + with ( + patch("data_hub_lambda.handler.get_client", return_value=client), + patch("data_hub_lambda.handler._cleanup_tmp"), + patch("data_hub_lambda.handler.get_processor", return_value=entry), + # Pass the union gate (e.g. a .pdf that belongs to another type). + patch( + "data_hub_lambda.handler.matches_any_processor_gate", + return_value=True, + ), + ): + result = lambda_handler( + self._s3_event("azure-cielo-qpcr", "run-1", "notes.pdf"), + MagicMock(), + ) + + assert result is None + process_file.assert_not_called() diff --git a/packages/shared/src/data_hub_shared/constants.py b/packages/shared/src/data_hub_shared/constants.py index 86ef053f..66900fbe 100644 --- a/packages/shared/src/data_hub_shared/constants.py +++ b/packages/shared/src/data_hub_shared/constants.py @@ -1,19 +1,6 @@ -from __future__ import annotations +"""Shared constants for the Data Hub packages. -from data_hub_shared.enums import Instrument - -INSTRUMENT_ID_TO_NAME_MAP: dict[str, str] = { - Instrument.AGILENT_4150_TAPESTATION.value: "Agilent 4150 TapeStation", - Instrument.AKTA_FPLC.value: "Akta FPLC", - Instrument.AZURE_600_GEL_DOC.value: "Azure 600 Gel Doc", - Instrument.AZURE_CIELO_QPCR.value: "Azure Cielo qPCR", - Instrument.EPSON_V700_SCANNER.value: "Epson V700 Scanner", - Instrument.HINA_MICROSCOPE.value: "Hina Microscope", - Instrument.INSTANT_RAMAN.value: "InstantRaman", - Instrument.SPECTRAMAX_ID3_PLATE_READER.value: "SpectraMax iD3 Plate Reader", - Instrument.SPECTRAMAX_ID5_PLATE_READER.value: "SpectraMax iD5 Plate Reader", -} - -INSTRUMENT_NAME_TO_ID_MAP: dict[str, str] = { - name: instrument_id for instrument_id, name in INSTRUMENT_ID_TO_NAME_MAP.items() -} +Instrument display names and IDs now live in the web app's ``instruments`` +table (and the Lambda resolves processors by ``instrument_type``). This +module is reserved for any remaining cross-package constants. +""" diff --git a/packages/shared/src/data_hub_shared/testing.py b/packages/shared/src/data_hub_shared/testing.py index c0b7d6e3..d7f8b719 100644 --- a/packages/shared/src/data_hub_shared/testing.py +++ b/packages/shared/src/data_hub_shared/testing.py @@ -136,9 +136,17 @@ def _token_display_prefix(plaintext: str) -> str: # --------------------------------------------------------------------------- -def seed_auth(dsn: str) -> str: - """Insert a user and personal access token, returning the plaintext token.""" +def seed_auth(dsn: str, scopes: list[str] | None = None) -> str: + """Insert a user and personal access token, returning the plaintext token. + + Args: + dsn: Postgres DSN for the test database. + scopes: Permission scopes for the minted PAT. Defaults to ``["*"]`` + (wildcard) so existing suites keep full access. Pass an explicit + list (e.g. the Lambda preset) to exercise least-privilege paths. + """ token_plaintext = _generate_token() + token_scopes = scopes if scopes is not None else ["*"] conn = psycopg2.connect(dsn) conn.autocommit = True with conn.cursor() as cur: @@ -149,13 +157,14 @@ def seed_auth(dsn: str) -> str: ) cur.execute( """INSERT INTO personal_access_tokens - (user_id, name, token_hash, token_prefix) - VALUES (%s, %s, %s, %s)""", + (user_id, name, token_hash, token_prefix, scopes) + VALUES (%s, %s, %s, %s, %s)""", ( user_id, "integration-test-token", _hash_token(token_plaintext), _token_display_prefix(token_plaintext), + token_scopes, ), ) conn.close() @@ -194,17 +203,31 @@ def seed_watcher_release( conn.close() -def seed_instruments(dsn: str, instruments: dict[str, str]) -> None: - """Insert instrument rows (ON CONFLICT DO NOTHING).""" +def seed_instruments( + dsn: str, + instruments: dict[str, str], + *, + instrument_types: dict[str, str] | None = None, +) -> None: + """Insert instrument rows (ON CONFLICT DO NOTHING). + + Args: + dsn: Postgres DSN for the test database. + instruments: Mapping of instrument ID → display name. + instrument_types: Optional mapping of instrument ID → ``instrument_type`` + enum value. Instruments omitted default to ``generic``. + """ + types = instrument_types or {} conn = psycopg2.connect(dsn) conn.autocommit = True with conn.cursor() as cur: for inst_id, display_name in instruments.items(): + instrument_type = types.get(inst_id, "generic") cur.execute( - """INSERT INTO instruments (id, display_name) - VALUES (%s, %s) + """INSERT INTO instruments (id, display_name, instrument_type) + VALUES (%s, %s, %s) ON CONFLICT (id) DO NOTHING""", - (inst_id, display_name), + (inst_id, display_name, instrument_type), ) conn.close() diff --git a/web/components/instruments/edit-instrument-dialog.tsx b/web/components/instruments/edit-instrument-dialog.tsx index a199c443..b6c24c88 100644 --- a/web/components/instruments/edit-instrument-dialog.tsx +++ b/web/components/instruments/edit-instrument-dialog.tsx @@ -22,9 +22,10 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { VALID_INSTRUMENT_TYPES } from "@/lib/db/schema"; +import { type InstrumentType, VALID_INSTRUMENT_TYPES } from "@/lib/db/schema"; +import { isProcessableInstrumentType } from "@/lib/instruments/processable-types"; -const TYPE_LABELS: Record = { +const TYPE_LABELS: Record = { generic: "Generic", plate_reader: "Plate Reader", gel_doc: "Gel Doc", @@ -33,13 +34,29 @@ const TYPE_LABELS: Record = { hina_microscope: "Hina Microscope", epson_v700_scanner: "Epson V700 Scanner", instant_raman: "InstantRaman", + fplc: "FPLC", }; const INSTRUMENT_TYPE_OPTIONS = VALID_INSTRUMENT_TYPES.map((value) => ({ value, - label: TYPE_LABELS[value] ?? value, + label: TYPE_LABELS[value], })); +function processingBoundaryWarning( + fromType: InstrumentType, + toType: InstrumentType +): string | null { + const wasProcessable = isProcessableInstrumentType(fromType); + const willBeProcessable = isProcessableInstrumentType(toType); + if (wasProcessable && !willBeProcessable) { + return "Files will no longer be processed automatically."; + } + if (!wasProcessable && willBeProcessable) { + return `New uploads will be processed by the ${TYPE_LABELS[toType]} processor.`; + } + return null; +} + export function EditInstrumentDialog({ instrumentId, displayName, @@ -49,13 +66,13 @@ export function EditInstrumentDialog({ }: { instrumentId: string; displayName: string; - instrumentType: string; + instrumentType: InstrumentType; open: boolean; onOpenChange: (open: boolean) => void; }) { const router = useRouter(); const [name, setName] = useState(displayName); - const [type, setType] = useState(instrumentType); + const [type, setType] = useState(instrumentType); const [isPending, startTransition] = useTransition(); // Re-sync form state from props each time the dialog opens so it reflects @@ -72,6 +89,11 @@ export function EditInstrumentDialog({ const isUnchanged = name.trim() === displayName.trim() && type === instrumentType; + const typeWarning = + type === instrumentType + ? null + : processingBoundaryWarning(instrumentType, type); + function handleSave() { startTransition(async () => { const res = await fetch(`/api/v1/instruments/${instrumentId}`, { @@ -118,7 +140,10 @@ export function EditInstrumentDialog({
- setType(value as InstrumentType)} + value={type} + > @@ -131,8 +156,17 @@ export function EditInstrumentDialog({

- Controls the run detail page layout. + Controls the run detail page layout and which Lambda processor + handles new uploads.

+ {typeWarning ? ( +

+ {typeWarning} +

+ ) : null}
diff --git a/web/components/notifications/notification-bell-content.tsx b/web/components/notifications/notification-bell-content.tsx index 08fa40cf..3273773e 100644 --- a/web/components/notifications/notification-bell-content.tsx +++ b/web/components/notifications/notification-bell-content.tsx @@ -47,6 +47,7 @@ const INSTRUMENT_TYPE_ICON: Record = { hina_microscope: Microscope, epson_v700_scanner: ScanLine, instant_raman: Radar, + fplc: FlaskConical, }; // Bucket labels live alongside the buckets themselves so the section diff --git a/web/components/runs/file-selection-provider.tsx b/web/components/runs/file-selection-provider.tsx index 5901bcf4..ee513fc2 100644 --- a/web/components/runs/file-selection-provider.tsx +++ b/web/components/runs/file-selection-provider.tsx @@ -2,7 +2,8 @@ import { createContext, use, useCallback, useMemo, useState } from "react"; import type { RunFile } from "@/lib/api/instrument-runs"; -import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import type { InstrumentType } from "@/lib/db/schema"; +import { isProcessableInstrumentType } from "@/lib/instruments/processable-types"; import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; // --------------------------------------------------------------------------- @@ -40,7 +41,7 @@ export interface FileRef { // the same as "no checkbox in this row". export function buildFileRef( file: RunFile, - instrumentId: string + instrumentType: InstrumentType ): FileRef | null { if (file.deletedAt !== null) { return null; @@ -48,7 +49,7 @@ export function buildFileRef( const isDetected = file.status === "detected"; const canDownload = DOWNLOADABLE_STATUSES.has(file.status); const canReprocess = - isProcessableInstrument(instrumentId) && + isProcessableInstrumentType(instrumentType) && REPROCESSABLE_STATUS_SET.has(file.status) && file.s3Key !== null; if (!(isDetected || canDownload)) { diff --git a/web/components/runs/run-files-section.tsx b/web/components/runs/run-files-section.tsx index 8e8df2ec..bad59128 100644 --- a/web/components/runs/run-files-section.tsx +++ b/web/components/runs/run-files-section.tsx @@ -29,6 +29,7 @@ import type { RunFileStats, RunFilesPage, } from "@/lib/api/instrument-runs"; +import type { InstrumentType } from "@/lib/db/schema"; import { runDetailSearchParams } from "@/lib/search-params"; import { FileBulkActionBar } from "./file-bulk-action-bar"; import { @@ -82,6 +83,8 @@ interface RunFilesSectionProps { // Downloadable files matching the active table filters (S3-backed). filteredDownloadableCount: number; instrumentId: string; + // Selects the Lambda processor / reprocess gate (not the instrument ID). + instrumentType: InstrumentType; isDeleted: boolean; pagination: RunFilesPage["pagination"]; runId: string; @@ -131,6 +134,7 @@ function RunFilesSectionContent({ pagination, stats, instrumentId, + instrumentType, runId, isDeleted, }: RunFilesSectionProps) { @@ -357,7 +361,7 @@ function RunFilesSectionContent({ ) : isDeleted ? ( handleSingleReprocess(id, startTransition, router) @@ -366,7 +370,7 @@ function RunFilesSectionContent({ ) : ( handleSingleDismiss(id, startTransition, router) diff --git a/web/components/runs/run-files-table.tsx b/web/components/runs/run-files-table.tsx index 8f971d2a..f87d1e45 100644 --- a/web/components/runs/run-files-table.tsx +++ b/web/components/runs/run-files-table.tsx @@ -24,7 +24,8 @@ import { } from "@/components/ui/table"; import type { RunFile } from "@/lib/api/instrument-runs"; import { formatDateTime } from "@/lib/date"; -import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import type { InstrumentType } from "@/lib/db/schema"; +import { isProcessableInstrumentType } from "@/lib/instruments/processable-types"; import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; import { cn, formatBytes } from "@/lib/utils"; import { @@ -214,10 +215,10 @@ function UploadDismissActions({ ); } -function canReprocess(file: RunFile, instrumentId: string): boolean { +function canReprocess(file: RunFile, instrumentType: InstrumentType): boolean { return ( file.deletedAt === null && - isProcessableInstrument(instrumentId) && + isProcessableInstrumentType(instrumentType) && REPROCESSABLE_STATUS_SET.has(file.status) && file.s3Key !== null ); @@ -231,14 +232,14 @@ function canReprocess(file: RunFile, instrumentId: string): boolean { export interface ReadOnlyRunFilesTableProps { files: RunFile[]; - instrumentId: string; + instrumentType: InstrumentType; isPending: boolean; onReprocess: (id: number) => void; } export function ReadOnlyRunFilesTable({ files, - instrumentId, + instrumentType, isPending, onReprocess, }: ReadOnlyRunFilesTableProps) { @@ -260,7 +261,7 @@ export function ReadOnlyRunFilesTable({ > - {canReprocess(file, instrumentId) ? ( + {canReprocess(file, instrumentType) ? (
void; onReprocess: (id: number) => void; @@ -297,7 +298,7 @@ export interface EditableRunFilesTableProps { export function EditableRunFilesTable({ files, - instrumentId, + instrumentType, isPending, onUpload, onDismiss, @@ -312,7 +313,7 @@ export function EditableRunFilesTable({ const visibleSelectableRefs: NonNullable>[] = []; for (const file of files) { - const ref = buildFileRef(file, instrumentId); + const ref = buildFileRef(file, instrumentType); refsByFileId.set(file.id, ref); if (ref) { visibleSelectableRefs.push(ref); @@ -341,7 +342,7 @@ export function EditableRunFilesTable({ const ref = refsByFileId.get(file.id) ?? null; const isSelected = ref ? meta.isSelected(ref.id) : false; const canDoUploadDismiss = !isDismissed && file.status === "detected"; - const canDoReprocess = canReprocess(file, instrumentId); + const canDoReprocess = canReprocess(file, instrumentType); // Reveal classes: hide per-row actions while a bulk selection is // active (the bar is the single entry point) but keep the JSX diff --git a/web/components/runs/variants/default-run-detail.tsx b/web/components/runs/variants/default-run-detail.tsx index 2041c61b..be1ca7fd 100644 --- a/web/components/runs/variants/default-run-detail.tsx +++ b/web/components/runs/variants/default-run-detail.tsx @@ -51,6 +51,7 @@ export function DefaultRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/epson-scanner-run-detail.tsx b/web/components/runs/variants/epson-scanner-run-detail.tsx index c506a974..95e1f214 100644 --- a/web/components/runs/variants/epson-scanner-run-detail.tsx +++ b/web/components/runs/variants/epson-scanner-run-detail.tsx @@ -51,6 +51,7 @@ export function EpsonScannerRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/gel-doc-run-detail.tsx b/web/components/runs/variants/gel-doc-run-detail.tsx index da15200b..0231fedc 100644 --- a/web/components/runs/variants/gel-doc-run-detail.tsx +++ b/web/components/runs/variants/gel-doc-run-detail.tsx @@ -51,6 +51,7 @@ export function GelDocRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/hina-microscope-run-detail.tsx b/web/components/runs/variants/hina-microscope-run-detail.tsx index 349a48aa..0542f6af 100644 --- a/web/components/runs/variants/hina-microscope-run-detail.tsx +++ b/web/components/runs/variants/hina-microscope-run-detail.tsx @@ -49,6 +49,7 @@ export function HinaMicroscopeRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/instant-raman-run-detail.tsx b/web/components/runs/variants/instant-raman-run-detail.tsx index 29d83eab..cc3d2625 100644 --- a/web/components/runs/variants/instant-raman-run-detail.tsx +++ b/web/components/runs/variants/instant-raman-run-detail.tsx @@ -52,6 +52,7 @@ export function InstantRamanRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/plate-reader-run-detail.tsx b/web/components/runs/variants/plate-reader-run-detail.tsx index 65516710..d0c70bf3 100644 --- a/web/components/runs/variants/plate-reader-run-detail.tsx +++ b/web/components/runs/variants/plate-reader-run-detail.tsx @@ -305,6 +305,7 @@ export function PlateReaderRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/qpcr-run-detail.tsx b/web/components/runs/variants/qpcr-run-detail.tsx index 0ab812ca..833d0625 100644 --- a/web/components/runs/variants/qpcr-run-detail.tsx +++ b/web/components/runs/variants/qpcr-run-detail.tsx @@ -49,6 +49,7 @@ export function QpcrRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/components/runs/variants/tape-station-run-detail.tsx b/web/components/runs/variants/tape-station-run-detail.tsx index f3416510..ec472911 100644 --- a/web/components/runs/variants/tape-station-run-detail.tsx +++ b/web/components/runs/variants/tape-station-run-detail.tsx @@ -54,6 +54,7 @@ export function TapeStationRunDetail({ files={files} filteredDownloadableCount={filesDownloadableCount} instrumentId={instrumentId} + instrumentType={run.instrumentType} isDeleted={isDeleted} pagination={filesPagination} runId={runId} diff --git a/web/drizzle/0034_needy_tomorrow_man.sql b/web/drizzle/0034_needy_tomorrow_man.sql new file mode 100644 index 00000000..df634692 --- /dev/null +++ b/web/drizzle/0034_needy_tomorrow_man.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."instrument_type" ADD VALUE 'fplc'; \ No newline at end of file diff --git a/web/drizzle/meta/0034_snapshot.json b/web/drizzle/meta/0034_snapshot.json new file mode 100644 index 00000000..ebf74bf4 --- /dev/null +++ b/web/drizzle/meta/0034_snapshot.json @@ -0,0 +1,2250 @@ +{ + "id": "033645eb-80bb-4f11-a451-9925dc96f7c7", + "prevId": "4d1e27f4-8927-43ba-a12c-b9db72e703a6", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "name": "account_provider_providerAccountId_pk", + "columns": ["provider", "providerAccountId"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.archive_jobs": { + "name": "archive_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_bucket": { + "name": "archive_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "archive_job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_archive_jobs_inflight": { + "name": "uq_archive_jobs_inflight", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"archive_jobs\".\"status\" in ('pending', 'building')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_archive_jobs_run_fingerprint_status": { + "name": "idx_archive_jobs_run_fingerprint_status", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archive_jobs_instrument_run_id_instrument_runs_id_fk": { + "name": "archive_jobs_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "archive_jobs_created_by_user_id_fk": { + "name": "archive_jobs_created_by_user_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_key": { + "name": "s3_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "file_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'raw'" + }, + "status": { + "name": "status", + "type": "file_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_at": { + "name": "detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "upload_requested_at": { + "name": "upload_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "file_created_at": { + "name": "file_created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_files_instrument_run_id_relative_path": { + "name": "uq_files_instrument_run_id_relative_path", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "relative_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"relative_path\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_active_instrument_run_id_filename": { + "name": "uq_files_active_instrument_run_id_filename", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_s3_key": { + "name": "uq_files_s3_key", + "columns": [ + { + "expression": "s3_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"s3_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_instrument_run_id": { + "name": "idx_files_instrument_run_id", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_status_instrument_run_id": { + "name": "idx_files_status_instrument_run_id", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_upload_queue": { + "name": "idx_files_upload_queue", + "columns": [ + { + "expression": "upload_requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"upload_requested_at\" is not null and \"files\".\"uploaded_at\" is null and \"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_metadata_gin": { + "name": "idx_files_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_files_filename_trgm": { + "name": "idx_files_filename_trgm", + "columns": [ + { + "expression": "\"filename\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "files_instrument_run_id_instrument_runs_id_fk": { + "name": "files_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "files", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_notification_subscriptions": { + "name": "instrument_notification_subscriptions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_instrument_notification_subscriptions_user_id": { + "name": "idx_instrument_notification_subscriptions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "instrument_notification_subscriptions_user_id_user_id_fk": { + "name": "instrument_notification_subscriptions_user_id_user_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "instrument_notification_subscriptions_instrument_id_instruments_id_fk": { + "name": "instrument_notification_subscriptions_instrument_id_instruments_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "instrument_notification_subscriptions_user_id_instrument_id_pk": { + "name": "instrument_notification_subscriptions_user_id_instrument_id_pk", + "columns": ["user_id", "instrument_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_runs": { + "name": "instrument_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "instrument_run_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'lambda'" + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_instrument_runs_instrument_id_created_at": { + "name": "idx_instrument_runs_instrument_id_created_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active": { + "name": "idx_instrument_runs_active", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active_acquired_at": { + "name": "idx_instrument_runs_active_acquired_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"acquired_at\", \"created_at\") desc", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_metadata_gin": { + "name": "idx_instrument_runs_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_instrument_runs_run_id_trgm": { + "name": "idx_instrument_runs_run_id_trgm", + "columns": [ + { + "expression": "\"run_id\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "instrument_runs_instrument_id_instruments_id_fk": { + "name": "instrument_runs_instrument_id_instruments_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_watcher_id_watchers_id_fk": { + "name": "instrument_runs_watcher_id_watchers_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_deleted_by_user_id_fk": { + "name": "instrument_runs_deleted_by_user_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "user", + "columnsFrom": ["deleted_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_instrument_runs_instrument_id_run_id": { + "name": "uq_instrument_runs_instrument_id_run_id", + "nullsNotDistinct": false, + "columns": ["instrument_id", "run_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instruments": { + "name": "instruments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "instrument_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "instrument_type": { + "name": "instrument_type", + "type": "instrument_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'generic'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "retired_by": { + "name": "retired_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_instruments_display_name_trgm": { + "name": "idx_instruments_display_name_trgm", + "columns": [ + { + "expression": "\"display_name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "instruments_retired_by_user_id_fk": { + "name": "instruments_retired_by_user_id_fk", + "tableFrom": "instruments", + "tableTo": "user", + "columnsFrom": ["retired_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_preferences": { + "name": "notification_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "runs_all_muted": { + "name": "runs_all_muted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "comments_attributed_enabled": { + "name": "comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "comments_participated_enabled": { + "name": "comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "slack_runs_enabled": { + "name": "slack_runs_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_attributed_enabled": { + "name": "slack_comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_participated_enabled": { + "name": "slack_comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "notification_preferences_user_id_user_id_fk": { + "name": "notification_preferences_user_id_user_id_fk", + "tableFrom": "notification_preferences", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notifications": { + "name": "notifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "comment_id": { + "name": "comment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "read_at": { + "name": "read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_notifications_user_id_created_at": { + "name": "idx_notifications_user_id_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notifications_user_id_unread": { + "name": "idx_notifications_user_id_unread", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"notifications\".\"read_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notifications_user_id_user_id_fk": { + "name": "notifications_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_run_id_instrument_runs_id_fk": { + "name": "notifications_run_id_instrument_runs_id_fk", + "tableFrom": "notifications", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_comment_id_run_comments_id_fk": { + "name": "notifications_comment_id_run_comments_id_fk", + "tableFrom": "notifications", + "tableTo": "run_comments", + "columnsFrom": ["comment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_actor_user_id_user_id_fk": { + "name": "notifications_actor_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["actor_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_access_tokens": { + "name": "personal_access_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "ARRAY['*']::text[]" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_personal_access_tokens_user_id": { + "name": "idx_personal_access_tokens_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_access_tokens_user_id_user_id_fk": { + "name": "personal_access_tokens_user_id_user_id_fk", + "tableFrom": "personal_access_tokens", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personal_access_tokens_token_hash_unique": { + "name": "personal_access_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": ["token_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_attributions": { + "name": "run_attributions", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_run_attributions_run_id": { + "name": "idx_run_attributions_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_attributions_user_id": { + "name": "idx_run_attributions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_attributions_run_id_instrument_runs_id_fk": { + "name": "run_attributions_run_id_instrument_runs_id_fk", + "tableFrom": "run_attributions", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_attributions_user_id_user_id_fk": { + "name": "run_attributions_user_id_user_id_fk", + "tableFrom": "run_attributions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "run_attributions_run_id_user_id_pk": { + "name": "run_attributions_run_id_user_id_pk", + "columns": ["run_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_comments": { + "name": "run_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_run_comments_run_id_created_at": { + "name": "idx_run_comments_run_id_created_at", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_comments_user_id": { + "name": "idx_run_comments_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_comments_body_trgm": { + "name": "idx_run_comments_body_trgm", + "columns": [ + { + "expression": "\"body\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"run_comments\".\"deleted_at\" is null", + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "run_comments_run_id_instrument_runs_id_fk": { + "name": "run_comments_run_id_instrument_runs_id_fk", + "tableFrom": "run_comments", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_comments_user_id_user_id_fk": { + "name": "run_comments_user_id_user_id_fk", + "tableFrom": "run_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_channel_config": { + "name": "slack_channel_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_channel_config_updated_by_user_id_fk": { + "name": "slack_channel_config_updated_by_user_id_fk", + "tableFrom": "slack_channel_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "slack_channel_config_singleton": { + "name": "slack_channel_config_singleton", + "value": "\"slack_channel_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.slack_connections": { + "name": "slack_connections", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slack_user_id": { + "name": "slack_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_id": { + "name": "slack_team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_name": { + "name": "slack_team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_connections_user_id_user_id_fk": { + "name": "slack_connections_user_id_user_id_fk", + "tableFrom": "slack_connections", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_events": { + "name": "watcher_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "watcher_event_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_events_watcher_id_timestamp": { + "name": "idx_watcher_events_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_watcher_events_watcher_id_event_type": { + "name": "idx_watcher_events_watcher_id_event_type", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_events_watcher_id_watchers_id_fk": { + "name": "watcher_events_watcher_id_watchers_id_fk", + "tableFrom": "watcher_events", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_heartbeats": { + "name": "watcher_heartbeats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upload_mode": { + "name": "upload_mode", + "type": "upload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "files_uploaded_since_last": { + "name": "files_uploaded_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "runs_reported_since_last": { + "name": "runs_reported_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "errors_since_last": { + "name": "errors_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "uptime_seconds": { + "name": "uptime_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_heartbeats_watcher_id_timestamp": { + "name": "idx_watcher_heartbeats_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_heartbeats_watcher_id_watchers_id_fk": { + "name": "watcher_heartbeats_watcher_id_watchers_id_fk", + "tableFrom": "watcher_heartbeats", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_release_config": { + "name": "watcher_release_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "latest_version": { + "name": "latest_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "min_supported_version": { + "name": "min_supported_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mandatory": { + "name": "mandatory", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "watcher_release_config_updated_by_user_id_fk": { + "name": "watcher_release_config_updated_by_user_id_fk", + "tableFrom": "watcher_release_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "watcher_release_config_singleton": { + "name": "watcher_release_config_singleton", + "value": "\"watcher_release_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.watchers": { + "name": "watchers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "os_info": { + "name": "os_info", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "watcher_version": { + "name": "watcher_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_checksum": { + "name": "config_checksum", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "watcher_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'registered'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deregistered_by": { + "name": "deregistered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registered_by_token": { + "name": "registered_by_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_watchers_active_instrument_id": { + "name": "uq_watchers_active_instrument_id", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"watchers\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watchers_instrument_id_instruments_id_fk": { + "name": "watchers_instrument_id_instruments_id_fk", + "tableFrom": "watchers", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "watchers_deregistered_by_user_id_fk": { + "name": "watchers_deregistered_by_user_id_fk", + "tableFrom": "watchers", + "tableTo": "user", + "columnsFrom": ["deregistered_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "watchers_registered_by_token_personal_access_tokens_id_fk": { + "name": "watchers_registered_by_token_personal_access_tokens_id_fk", + "tableFrom": "watchers", + "tableTo": "personal_access_tokens", + "columnsFrom": ["registered_by_token"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.archive_job_status": { + "name": "archive_job_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.file_category": { + "name": "file_category", + "schema": "public", + "values": ["raw", "processed"] + }, + "public.file_status": { + "name": "file_status", + "schema": "public", + "values": [ + "detected", + "upload_requested", + "uploaded", + "processing", + "completed", + "failed" + ] + }, + "public.instrument_run_source": { + "name": "instrument_run_source", + "schema": "public", + "values": ["lambda", "watcher"] + }, + "public.instrument_status": { + "name": "instrument_status", + "schema": "public", + "values": ["pending", "active", "inactive"] + }, + "public.instrument_type": { + "name": "instrument_type", + "schema": "public", + "values": [ + "generic", + "plate_reader", + "gel_doc", + "qpcr", + "tape_station", + "hina_microscope", + "epson_v700_scanner", + "instant_raman", + "fplc" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": ["run_created", "comment_attributed", "comment_participated"] + }, + "public.upload_mode": { + "name": "upload_mode", + "schema": "public", + "values": ["auto", "manual"] + }, + "public.watcher_event_type": { + "name": "watcher_event_type", + "schema": "public", + "values": [ + "watcher_started", + "watcher_stopped", + "file_uploaded", + "upload_failed", + "run_reported", + "config_synced", + "error", + "update_started", + "update_succeeded", + "update_failed" + ] + }, + "public.watcher_status": { + "name": "watcher_status", + "schema": "public", + "values": ["registered", "watching", "stopped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index 0f268757..1c58bbe9 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -239,6 +239,13 @@ "when": 1784320580169, "tag": "0033_steady_wasp", "breakpoints": true + }, + { + "idx": 34, + "version": "7", + "when": 1785189229393, + "tag": "0034_needy_tomorrow_man", + "breakpoints": true } ] } diff --git a/web/lib/api/file-reprocessing.ts b/web/lib/api/file-reprocessing.ts index 5e8bc03a..adcecc98 100644 --- a/web/lib/api/file-reprocessing.ts +++ b/web/lib/api/file-reprocessing.ts @@ -2,8 +2,8 @@ import { and, eq, inArray, isNull } from "drizzle-orm"; import { after } from "next/server"; import { lookupRunByNaturalKey } from "@/lib/api/instrument-runs"; import { db } from "@/lib/db"; -import { files, instrumentRuns } from "@/lib/db/schema"; -import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import { files, instrumentRuns, instruments } from "@/lib/db/schema"; +import { isProcessableInstrumentType } from "@/lib/instruments/processable-types"; import { hasInvokeCredentials, signLambdaInvoke } from "@/lib/lambda"; import { REPROCESSABLE_STATUSES } from "@/lib/runs/reprocessable-statuses"; @@ -78,8 +78,10 @@ export async function reprocessFile(fileId: number): Promise { .select({ deletedAt: instrumentRuns.deletedAt, instrumentId: instrumentRuns.instrumentId, + instrumentType: instruments.instrumentType, }) .from(instrumentRuns) + .innerJoin(instruments, eq(instrumentRuns.instrumentId, instruments.id)) .where(eq(instrumentRuns.id, file.instrumentRunId)) .limit(1); @@ -92,13 +94,13 @@ export async function reprocessFile(fileId: number): Promise { }; } - if (!(parentRun && isProcessableInstrument(parentRun.instrumentId))) { + if (!(parentRun && isProcessableInstrumentType(parentRun.instrumentType))) { return { ok: false, status: 409, code: "CONFLICT", message: parentRun - ? `Instrument '${parentRun.instrumentId}' has no Lambda processor — cannot reprocess` + ? `Instrument type '${parentRun.instrumentType}' has no Lambda processor — cannot reprocess` : "Cannot reprocess a file with no parent run", }; } @@ -189,15 +191,6 @@ export async function reprocessRun( instrumentId: string, runId: string ): Promise { - if (!isProcessableInstrument(instrumentId)) { - return { - ok: false, - status: 409, - code: "CONFLICT", - message: `Instrument '${instrumentId}' has no Lambda processor — cannot reprocess`, - }; - } - const run = await lookupRunByNaturalKey(instrumentId, runId); if (!run) { @@ -218,6 +211,15 @@ export async function reprocessRun( }; } + if (!isProcessableInstrumentType(run.instrumentType)) { + return { + ok: false, + status: 409, + code: "CONFLICT", + message: `Instrument type '${run.instrumentType}' has no Lambda processor — cannot reprocess`, + }; + } + const eligible = await db .select({ id: files.id }) .from(files) diff --git a/web/lib/api/instrument-runs.ts b/web/lib/api/instrument-runs.ts index 77d15ee0..c7aef16e 100644 --- a/web/lib/api/instrument-runs.ts +++ b/web/lib/api/instrument-runs.ts @@ -566,6 +566,7 @@ export async function buildRunListQuery(filters: RunListFilters) { id: instrumentRuns.id, instrument_id: instrumentRuns.instrumentId, instrument_display_name: instruments.displayName, + instrument_type: instruments.instrumentType, run_id: instrumentRuns.runId, source: instrumentRuns.source, metadata: instrumentRuns.metadata, @@ -592,7 +593,11 @@ export async function buildRunListQuery(filters: RunListFilters) { ) ) .where(where) - .groupBy(instrumentRuns.id, instruments.displayName) + .groupBy( + instrumentRuns.id, + instruments.displayName, + instruments.instrumentType + ) .orderBy(orderFn(sortCol)) .limit(perPage) .offset(offset); diff --git a/web/lib/api/openapi/schemas/runs.ts b/web/lib/api/openapi/schemas/runs.ts index fb693514..8df82aef 100644 --- a/web/lib/api/openapi/schemas/runs.ts +++ b/web/lib/api/openapi/schemas/runs.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { fileCategorySchema, fileStatusSchema, + instrumentTypeSchema, isoDateTime, paginationSchema, runSourceSchema, @@ -161,6 +162,7 @@ export const runListItem = z id: z.string().uuid(), instrument_id: z.string(), instrument_display_name: z.string().nullable(), + instrument_type: instrumentTypeSchema, run_id: z.string(), source: runSourceSchema, metadata: metadataObject.nullable(), diff --git a/web/lib/api/scope-catalog.ts b/web/lib/api/scope-catalog.ts index 88c056fe..d7a6e56c 100644 --- a/web/lib/api/scope-catalog.ts +++ b/web/lib/api/scope-catalog.ts @@ -177,6 +177,7 @@ export const SCOPE_PRESETS: ScopePreset[] = [ label: "Lambda", description: "For the processing Lambda writing runs and file results.", scopes: [ + "instruments:read", "runs:create", "runs:update", "files:create", diff --git a/web/lib/db/schema.ts b/web/lib/db/schema.ts index f8c45970..038d7f61 100644 --- a/web/lib/db/schema.ts +++ b/web/lib/db/schema.ts @@ -61,6 +61,7 @@ export const instrumentTypeEnum = pgEnum("instrument_type", [ "hina_microscope", "epson_v700_scanner", "instant_raman", + "fplc", ]); export const VALID_INSTRUMENT_TYPES = instrumentTypeEnum.enumValues; diff --git a/web/lib/db/seed.ts b/web/lib/db/seed.ts index 4908cfb9..4eab37d5 100644 --- a/web/lib/db/seed.ts +++ b/web/lib/db/seed.ts @@ -391,6 +391,8 @@ export const SEED_INSTRUMENTS: readonly SeededInstrument[] = [ { id: "jolene-fplc", displayName: "Jolene FPLC", + // Stays generic until an operator confirms PDFs match the ÄKTA `fplc` + // processor; typing it `fplc` without that check would mis-route files. instrumentType: "generic", status: "active", hostname: "DESKTOP-30488S0", diff --git a/web/lib/instruments/processable-ids.ts b/web/lib/instruments/processable-ids.ts deleted file mode 100644 index fe606869..00000000 --- a/web/lib/instruments/processable-ids.ts +++ /dev/null @@ -1,19 +0,0 @@ -// Instrument IDs that have a `process_file` branch in -// `lambda/src/data_hub_lambda/handler.py`. Keep in sync when adding a -// processor — instruments without a handler (e.g. InstantRaman) stay out. -export const PROCESSABLE_INSTRUMENT_IDS = [ - "akta-fplc", - "agilent-4150-tapestation", - "azure-600-gel-doc", - "azure-cielo-qpcr", - "epson-v700-scanner", - "hina-microscope", - "spectramax-id3-plate-reader", - "spectramax-id5-plate-reader", -] as const; - -const PROCESSABLE_SET = new Set(PROCESSABLE_INSTRUMENT_IDS); - -export function isProcessableInstrument(instrumentId: string): boolean { - return PROCESSABLE_SET.has(instrumentId); -} diff --git a/web/lib/instruments/processable-types.ts b/web/lib/instruments/processable-types.ts new file mode 100644 index 00000000..20e542ea --- /dev/null +++ b/web/lib/instruments/processable-types.ts @@ -0,0 +1,22 @@ +import type { InstrumentType } from "@/lib/db/schema"; + +// Instrument types that have a processor entry in +// `lambda/src/data_hub_lambda/processors.py`. Keep in sync when adding a +// processor — types without a handler (e.g. `generic`, `instant_raman`) stay out. +export const PROCESSABLE_INSTRUMENT_TYPES = [ + "qpcr", + "plate_reader", + "gel_doc", + "tape_station", + "hina_microscope", + "epson_v700_scanner", + "fplc", +] as const satisfies readonly InstrumentType[]; + +const PROCESSABLE_SET = new Set(PROCESSABLE_INSTRUMENT_TYPES); + +export function isProcessableInstrumentType( + instrumentType: InstrumentType +): boolean { + return PROCESSABLE_SET.has(instrumentType); +} diff --git a/web/lib/runs/row-actions.ts b/web/lib/runs/row-actions.ts index 3f9d7b88..8d44aaec 100644 --- a/web/lib/runs/row-actions.ts +++ b/web/lib/runs/row-actions.ts @@ -4,7 +4,7 @@ import type { RunRef, RunStats, } from "@/components/instruments/runs-table/run-selection-provider"; -import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; +import { isProcessableInstrumentType } from "@/lib/instruments/processable-types"; // --------------------------------------------------------------------------- // Predicates that decide which row-level / bulk actions are available for a @@ -14,8 +14,8 @@ import { isProcessableInstrument } from "@/lib/instruments/processable-ids"; // - Upload: at least one file still waiting to be uploaded. // - Download: at least one file has made it to S3 (i.e. is not `detected` // or `upload_requested`). -// - Reprocess: instrument has a Lambda processor and at least one file is in -// `uploaded`, `completed`, or `failed`. +// - Reprocess: instrument_type has a Lambda processor and at least one file +// is in `uploaded`, `completed`, or `failed`. // - Delete: the run isn't already soft-deleted. // --------------------------------------------------------------------------- @@ -34,7 +34,7 @@ export function canDownloadRun(row: RunRow): boolean { export function canReprocessRun(row: RunRow): boolean { return ( row.deleted_at === null && - isProcessableInstrument(row.instrument_id) && + isProcessableInstrumentType(row.instrument_type) && row.files_completed + row.files_failed + row.files_uploaded > 0 ); } diff --git a/web/tests/integration/files.test.ts b/web/tests/integration/files.test.ts index ae714c8f..140f56ca 100644 --- a/web/tests/integration/files.test.ts +++ b/web/tests/integration/files.test.ts @@ -23,8 +23,8 @@ import { // Tests drive each through its full lifecycle to verify the state machine. describe("Files API", () => { let token: string; - // Use a known processable instrument ID so reprocess eligibility reaches - // the Lambda-config check (see `isProcessableInstrument`). + // Use a processable instrument_type so reprocess eligibility reaches + // the Lambda-config check (see `isProcessableInstrumentType`). const instrumentId = "akta-fplc"; const runId = "files-test-run"; let fileId: number; @@ -41,6 +41,7 @@ describe("Files API", () => { id: instrumentId, displayName: "Akta FPLC", status: "active", + instrumentType: "fplc", }); // Create a run with detected files (watcher path) @@ -642,6 +643,7 @@ describe("Files API", () => { }); it("REPROCESS returns 409 for instrument without a Lambda processor", async () => { + // Defaults to instrument_type=generic, which has no processor. const noProcessorId = "files-no-processor-instrument"; const db = getTestDb(); await db.insert(instruments).values({ @@ -678,4 +680,46 @@ describe("Files API", () => { const data = await res.json(); expect(data.error.message).toContain("no Lambda processor"); }); + + it("REPROCESS returns 409 for a generic-typed instrument even with a known ID", async () => { + // Same ID shape as a formerly allowlisted FPLC, but typed generic so + // type-based gating must refuse reprocess (mirrors jolene-fplc in seed). + const genericFplcId = "files-generic-fplc"; + const db = getTestDb(); + await db.insert(instruments).values({ + id: genericFplcId, + displayName: "Generic FPLC", + status: "active", + instrumentType: "generic", + }); + const genericRunId = "reprocess-generic-fplc-run"; + await api(`/api/v1/instruments/${genericFplcId}/runs`, { + method: "POST", + token, + body: { run_id: genericRunId, source: "lambda" }, + }); + const createFileRes = await api( + `/api/v1/instruments/${genericFplcId}/runs/${genericRunId}/files`, + { + method: "POST", + token, + body: { + s3_bucket: "test-bucket", + s3_key: `${genericFplcId}/${genericRunId}/chromatogram.pdf`, + filename: "chromatogram.pdf", + }, + } + ); + expect(createFileRes.status).toBe(201); + const createdFile = await createFileRes.json(); + + const res = await api(`/api/v1/files/${createdFile.id}/reprocess`, { + method: "POST", + token, + }); + expect(res.status).toBe(409); + const data = await res.json(); + expect(data.error.message).toContain("generic"); + expect(data.error.message).toContain("no Lambda processor"); + }); });