Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ db-process-fixtures:
fe-build:
cd web && npm run build

.PHONY: openapi-generate
openapi-generate:
cd web && npm run openapi:generate

# Formatting, linting, and type checking.
.PHONY: py-check
py-check:
Expand Down
2 changes: 1 addition & 1 deletion developer-docs/run-archives.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ The "Download all" actions on the run detail page and the runs table deliver eve

Each archive can mix files from the raw bucket and the processed bucket in a single zip. This matters for instruments that produce processed artifacts via Lambda preprocessing (SpectraMax raw `.xls` → processed CSV; Hina `.nd2` → processed JPG; Azure 600 Gel Doc `.tif` → processed PNG): the run's file rows reference both buckets, and "Download all" zips them together.

This page covers the end-to-end flow, the cache + dedup model, and the on-call runbook. For the Lambda invocation contract, see [Lambda → Function URL (archive build)](lambda.md#function-url-archive-build). For the HTTP endpoints, see [REST API → Archive jobs](https://datahub.arcadiascience.com/docs/api-reference#archive-jobs).
This page covers the end-to-end flow, the cache + dedup model, and the on-call runbook. For the Lambda invocation contract, see [Lambda → Function URL (archive build)](lambda.md#function-url-archive-build). For the HTTP endpoints, see the [generated API docs](https://datahub.arcadiascience.com/docs/api) (Archive tag).

## Flow

Expand Down
4 changes: 4 additions & 0 deletions web/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,7 @@ pnpm-debug.log*
*.tsbuildinfo
next-env.d.ts
.vercel

# Local OpenAPI dump from `npm run openapi:generate` (not committed;
# production serves the schema from GET /api/v1/openapi.json).
/openapi.json
45 changes: 10 additions & 35 deletions web/app/api/v1/archive-jobs/[id]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { eq } from "drizzle-orm";
import type { NextRequest } from "next/server";
import { authorize } from "@/lib/api/auth";
import { apiError, NOT_FOUND, VALIDATION_ERROR } from "@/lib/api/errors";
import { patchArchiveJobBody, readJsonBody } from "@/lib/api/openapi";
import { isValidUUID } from "@/lib/api/validators";
import { db } from "@/lib/db";
import { archiveJobs } from "@/lib/db/schema";
Expand Down Expand Up @@ -31,14 +32,6 @@ interface RouteContext {

const TERMINAL_STATUSES = new Set(["ready", "failed"]);

interface PatchBody {
archive_bucket?: unknown;
archive_key?: unknown;
error_message?: unknown;
size_bytes?: unknown;
status?: unknown;
}

export async function PATCH(request: NextRequest, { params }: RouteContext) {
const authResult = await authorize(request, "archive-jobs:write");
if (authResult instanceof Response) {
Expand All @@ -50,31 +43,13 @@ export async function PATCH(request: NextRequest, { params }: RouteContext) {
return apiError(400, VALIDATION_ERROR, "Invalid job ID format");
}

let body: PatchBody;
try {
body = (await request.json()) as PatchBody;
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
const body = await readJsonBody(request, patchArchiveJobBody);
if (body instanceof Response) {
return body;
}
const status = body.status;

if (
typeof body.status !== "string" ||
!["pending", "building", "ready", "failed"].includes(body.status)
) {
return apiError(
400,
VALIDATION_ERROR,
"status must be one of pending|building|ready|failed"
);
}

const status = body.status as "pending" | "building" | "ready" | "failed";

if (
status === "ready" &&
(typeof body.archive_bucket !== "string" ||
typeof body.archive_key !== "string")
) {
if (status === "ready" && !(body.archive_bucket && body.archive_key)) {
return apiError(
400,
VALIDATION_ERROR,
Expand All @@ -83,16 +58,16 @@ export async function PATCH(request: NextRequest, { params }: RouteContext) {
}

const update: Partial<typeof archiveJobs.$inferInsert> = { status };
if (typeof body.archive_bucket === "string") {
if (body.archive_bucket !== undefined) {
update.archiveBucket = body.archive_bucket;
}
if (typeof body.archive_key === "string") {
if (body.archive_key !== undefined) {
update.archiveKey = body.archive_key;
}
if (typeof body.size_bytes === "number") {
if (body.size_bytes !== undefined) {
update.sizeBytes = body.size_bytes;
}
if (typeof body.error_message === "string") {
if (body.error_message !== undefined) {
update.errorMessage = body.error_message;
}
if (TERMINAL_STATUSES.has(status)) {
Expand Down
28 changes: 11 additions & 17 deletions web/app/api/v1/files/[fileId]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
VALIDATION_ERROR,
} from "@/lib/api/errors";
import { dismissFile } from "@/lib/api/files";
import { patchFileBody, readJsonBody } from "@/lib/api/openapi";
import { db } from "@/lib/db";
import { files, instrumentRuns } from "@/lib/db/schema";

Expand Down Expand Up @@ -84,18 +85,16 @@ export async function PATCH(request: NextRequest, { params }: RouteContext) {
);
}

let body: Record<string, unknown>;
try {
body = await request.json();
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
const body = await readJsonBody(request, patchFileBody);
if (body instanceof Response) {
return body;
}

const updates: Record<string, unknown> = {};
const now = new Date();

// Status transition validation.
if ("status" in body && typeof body.status === "string") {
if (body.status !== undefined) {
const allowed = VALID_TRANSITIONS[file.status];
if (!allowed?.includes(body.status)) {
return apiError(
Expand Down Expand Up @@ -130,31 +129,26 @@ export async function PATCH(request: NextRequest, { params }: RouteContext) {
}

// S3 info — set when transitioning to "uploaded" (watcher path).
if (typeof body.s3_bucket === "string") {
if (body.s3_bucket !== undefined) {
updates.s3Bucket = body.s3_bucket;
}
if (typeof body.s3_key === "string") {
if (body.s3_key !== undefined) {
updates.s3Key = body.s3_key;
}
if (typeof body.content_type === "string") {
if (body.content_type !== undefined) {
updates.contentType = body.content_type;
}
if (typeof body.size_bytes === "number") {
if (body.size_bytes !== undefined) {
updates.sizeBytes = body.size_bytes;
}

// Metadata — flat JSON object set by the Lambda after processing.
if (
"metadata" in body &&
typeof body.metadata === "object" &&
body.metadata !== null &&
!Array.isArray(body.metadata)
) {
if (body.metadata !== undefined) {
updates.metadata = body.metadata;
}

// Error message — set when status transitions to "failed".
if (typeof body.error_message === "string") {
if (body.error_message !== undefined) {
updates.errorMessage = body.error_message;
}

Expand Down
63 changes: 8 additions & 55 deletions web/app/api/v1/instruments/[instrumentId]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,10 @@ import { and, count, eq, isNull } from "drizzle-orm";
import type { NextRequest } from "next/server";
import { authorize, requireAdminForSession } from "@/lib/api/auth";
import { apiError, NOT_FOUND, VALIDATION_ERROR } from "@/lib/api/errors";
import { patchInstrumentBody, readJsonBody } from "@/lib/api/openapi";
import { deregisterInstrumentWatchers } from "@/lib/api/watchers";
import { db } from "@/lib/db";
import {
instrumentRuns,
instruments,
VALID_INSTRUMENT_TYPES,
watchers,
} from "@/lib/db/schema";
import { instrumentRuns, instruments, watchers } from "@/lib/db/schema";

export async function GET(
request: NextRequest,
Expand Down Expand Up @@ -63,12 +59,6 @@ export async function GET(
});
}

const ALLOWED_PATCH_FIELDS = new Set([
"status",
"display_name",
"instrument_type",
]);

export async function PATCH(
request: NextRequest,
{ params }: { params: Promise<{ instrumentId: string }> }
Expand All @@ -89,21 +79,9 @@ export async function PATCH(

const { instrumentId } = await params;

let body: Record<string, unknown>;
try {
body = await request.json();
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
}

const unknownKeys = Object.keys(body).filter(
(k) => !ALLOWED_PATCH_FIELDS.has(k)
);
if (unknownKeys.length > 0) {
return apiError(400, VALIDATION_ERROR, "Unknown fields", {
unknown_fields: unknownKeys,
allowed_fields: [...ALLOWED_PATCH_FIELDS],
});
const body = await readJsonBody(request, patchInstrumentBody);
if (body instanceof Response) {
return body;
}

const [existing] = await db
Expand All @@ -116,33 +94,8 @@ export async function PATCH(
return apiError(404, NOT_FOUND, `Instrument '${instrumentId}' not found`);
}

const VALID_INSTRUMENT_STATUSES = ["pending", "active", "inactive"];
if (
"status" in body &&
!VALID_INSTRUMENT_STATUSES.includes(body.status as string)
) {
return apiError(
400,
VALIDATION_ERROR,
`Invalid status — must be one of: ${VALID_INSTRUMENT_STATUSES.join(", ")}`
);
}

if (
"instrument_type" in body &&
!(VALID_INSTRUMENT_TYPES as readonly string[]).includes(
body.instrument_type as string
)
) {
return apiError(
400,
VALIDATION_ERROR,
`Invalid instrument_type — must be one of: ${VALID_INSTRUMENT_TYPES.join(", ")}`
);
}

const updates: Record<string, unknown> = {};
if ("status" in body) {
if (body.status !== undefined) {
updates.status = body.status;
// Keep the retirement audit fields in lockstep with the status: only an
// `inactive` instrument has a retirer.
Expand All @@ -154,10 +107,10 @@ export async function PATCH(
updates.retiredBy = null;
}
}
if ("display_name" in body) {
if (body.display_name !== undefined) {
updates.displayName = body.display_name;
}
if ("instrument_type" in body) {
if (body.instrument_type !== undefined) {
updates.instrumentType = body.instrument_type;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
VALIDATION_ERROR,
} from "@/lib/api/errors";
import { lookupRunByNaturalKey } from "@/lib/api/instrument-runs";
import { commentBody, readJsonBody } from "@/lib/api/openapi";
import {
getCommentForAuthorCheck,
softDeleteComment,
Expand Down Expand Up @@ -103,15 +104,9 @@ export async function PATCH(request: NextRequest, { params }: RouteContext) {
return pre.response;
}

let payload: Record<string, unknown>;
try {
payload = await request.json();
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
}

if (typeof payload.body !== "string") {
return apiError(400, VALIDATION_ERROR, "body must be a string");
const payload = await readJsonBody(request, commentBody);
if (payload instanceof Response) {
return payload;
}

const validated = validateCommentBody(payload.body);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
VALIDATION_ERROR,
} from "@/lib/api/errors";
import { lookupRunByNaturalKey } from "@/lib/api/instrument-runs";
import { commentBody, readJsonBody } from "@/lib/api/openapi";
import {
createCommentAndNotify,
listCommentsForRun,
Expand Down Expand Up @@ -72,15 +73,9 @@ export async function POST(request: NextRequest, { params }: RouteContext) {
return apiError(409, CONFLICT, "Cannot comment on a soft-deleted run");
}

let payload: Record<string, unknown>;
try {
payload = await request.json();
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
}

if (typeof payload.body !== "string") {
return apiError(400, VALIDATION_ERROR, "body must be a string");
const payload = await readJsonBody(request, commentBody);
if (payload instanceof Response) {
return payload;
}

const validated = validateCommentBody(payload.body);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
import { and, eq, isNull } from "drizzle-orm";
import type { NextRequest } from "next/server";
import { authorize } from "@/lib/api/auth";
import {
apiError,
CONFLICT,
NOT_FOUND,
VALIDATION_ERROR,
} from "@/lib/api/errors";
import { apiError, CONFLICT, NOT_FOUND } from "@/lib/api/errors";
import { lookupRunByNaturalKey } from "@/lib/api/instrument-runs";
import { createFileBody, readJsonBody } from "@/lib/api/openapi";
import { db } from "@/lib/db";
import { files } from "@/lib/db/schema";

Expand Down Expand Up @@ -52,33 +48,17 @@ export async function POST(request: NextRequest, { params }: RouteContext) {
return apiError(409, CONFLICT, "Cannot add files to a soft-deleted run");
}

let body: Record<string, unknown>;
try {
body = await request.json();
} catch {
return apiError(400, VALIDATION_ERROR, "Invalid JSON body");
const body = await readJsonBody(request, createFileBody);
if (body instanceof Response) {
return body;
}

const s3Bucket =
typeof body.s3_bucket === "string" ? body.s3_bucket.trim() : "";
const s3Key = typeof body.s3_key === "string" ? body.s3_key.trim() : "";
const filename =
typeof body.filename === "string" ? body.filename.trim() : "";

if (!(s3Bucket && s3Key && filename)) {
return apiError(
400,
VALIDATION_ERROR,
"s3_bucket, s3_key, and filename are required"
);
}

const contentType =
typeof body.content_type === "string" ? body.content_type : null;
const sizeBytes =
typeof body.size_bytes === "number" ? body.size_bytes : null;
const category =
body.category === "processed" ? ("processed" as const) : ("raw" as const);
const s3Bucket = body.s3_bucket;
const s3Key = body.s3_key;
const filename = body.filename;
const contentType = body.content_type ?? null;
const sizeBytes = body.size_bytes ?? null;
const category = body.category ?? "raw";

const now = new Date();

Expand Down
Loading