From ab13651190d07e50a61bbfdb80ee2c13c77fe135 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:20:00 -0700 Subject: [PATCH] Play DishCam runs as video in the app (#196) * Add a DishCam processor that encodes TIFF stacks into in-app MP4 previews. Co-authored-by: Cursor * Harden DishCam encode against deadlocks, sibling races, and local playback gaps. Co-authored-by: Cursor * Stop guessing DishCam bit depth from uint16 peaks. Co-authored-by: Cursor * Hide DishCam sidecar fields from the run summary tags. Co-authored-by: Cursor * Encode every DishCam TIFF in a run and seek the MP4s like other reports. Co-authored-by: Cursor * Share one DishCam video player and pass posters as ids, not full file rows. Skip MP4 preload until play, answer HEAD on the local mirror, and treat S3 403 as missing. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .github/workflows/python-test.yml | 3 + developer-docs/getting-started.md | 1 + developer-docs/lambda.md | 10 +- lambda/Dockerfile | 5 + lambda/src/data_hub_lambda/cli.py | 40 + .../src/data_hub_lambda/dishcam/__init__.py | 3 + .../data_hub_lambda/dishcam/encode_video.py | 183 + .../src/data_hub_lambda/dishcam/filenames.py | 24 + .../data_hub_lambda/dishcam/parse_metadata.py | 62 + .../data_hub_lambda/dishcam/process_file.py | 263 ++ lambda/src/data_hub_lambda/processors.py | 6 + lambda/tests/dishcam/test_encode_video.py | 69 + lambda/tests/dishcam/test_parse_metadata.py | 61 + lambda/tests/dishcam/test_process_file.py | 536 +++ lambda/tests/fixtures/dishcam_example.tif | Bin 0 -> 37688 bytes lambda/tests/fixtures/dishcam_run.json | 13 + lambda/tests/integration/conftest.py | 35 + lambda/tests/integration/test_lambda_api.py | 102 + lambda/tests/test_processors.py | 8 + .../shared/src/data_hub_shared/s3_utils.py | 38 +- packages/shared/tests/test_s3_utils.py | 60 + .../api/local-s3/[bucket]/[...key]/route.ts | 76 +- .../instruments/edit-instrument-dialog.tsx | 1 + .../notification-bell-content.tsx | 2 + web/components/runs/report-item-seeker.tsx | 7 + web/components/runs/run-detail.ts | 2 + web/components/runs/run-report-section.tsx | 35 +- web/components/runs/run-video-player.tsx | 47 + .../runs/variants/dishcam-run-detail.tsx | 66 + web/components/runs/variants/index.tsx | 3 + web/components/runs/video-carousel-report.tsx | 59 + web/drizzle/0040_dishcam_type.sql | 1 + web/drizzle/meta/0040_snapshot.json | 3102 +++++++++++++++++ web/drizzle/meta/_journal.json | 7 + web/lib/api/instrument-runs.ts | 1 + web/lib/api/openapi/paths/runs.ts | 2 +- web/lib/api/report-items.ts | 4 + web/lib/db/schema.ts | 1 + web/lib/instruments/processable-types.ts | 1 + web/lib/runs/report-items.ts | 3 +- web/lib/runs/run-file-types.ts | 47 + web/lib/s3-local-mirror.ts | 52 + web/tests/integration/report-items.test.ts | 22 + web/tests/unit/report-items.test.ts | 26 + web/tests/unit/run-file-types.test.ts | 125 + web/tests/unit/s3-local-mirror.test.ts | 43 + 46 files changed, 5236 insertions(+), 21 deletions(-) create mode 100644 lambda/src/data_hub_lambda/dishcam/__init__.py create mode 100644 lambda/src/data_hub_lambda/dishcam/encode_video.py create mode 100644 lambda/src/data_hub_lambda/dishcam/filenames.py create mode 100644 lambda/src/data_hub_lambda/dishcam/parse_metadata.py create mode 100644 lambda/src/data_hub_lambda/dishcam/process_file.py create mode 100644 lambda/tests/dishcam/test_encode_video.py create mode 100644 lambda/tests/dishcam/test_parse_metadata.py create mode 100644 lambda/tests/dishcam/test_process_file.py create mode 100644 lambda/tests/fixtures/dishcam_example.tif create mode 100644 lambda/tests/fixtures/dishcam_run.json create mode 100644 packages/shared/tests/test_s3_utils.py create mode 100644 web/components/runs/run-video-player.tsx create mode 100644 web/components/runs/variants/dishcam-run-detail.tsx create mode 100644 web/components/runs/video-carousel-report.tsx create mode 100644 web/drizzle/0040_dishcam_type.sql create mode 100644 web/drizzle/meta/0040_snapshot.json create mode 100644 web/tests/unit/report-items.test.ts create mode 100644 web/tests/unit/run-file-types.test.ts create mode 100644 web/tests/unit/s3-local-mirror.test.ts diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 7ee8038a..470a4392 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -63,6 +63,9 @@ jobs: run: npm ci working-directory: web + - name: Install ffmpeg + run: sudo apt-get update && sudo apt-get install -y ffmpeg + - name: Install Python packages run: uv sync --all-packages diff --git a/developer-docs/getting-started.md b/developer-docs/getting-started.md index 41651162..e868c9e7 100644 --- a/developer-docs/getting-started.md +++ b/developer-docs/getting-started.md @@ -11,6 +11,7 @@ This guide walks through setting up the full Data Hub development environment. | Node.js | >= 22 | Web application | | PostgreSQL | >= 15 | Database (local development) | | Docker | latest | Lambda container builds | +| ffmpeg | latest | DishCam TIFF → MP4 encode (`brew install ffmpeg` / `apt install ffmpeg`). Encode tests skip if it is missing. | | [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) | latest | Infrastructure deployment (optional — only needed for deploying to AWS) | ## Clone and install diff --git a/developer-docs/lambda.md b/developer-docs/lambda.md index 1afafa1b..79d7c73b 100644 --- a/developer-docs/lambda.md +++ b/developer-docs/lambda.md @@ -50,6 +50,7 @@ Dispatch is by `instrument_type` (Postgres/TS enum), not instrument ID. The regi | `epson_v700_scanner` | `epson_v700_scanner` | `.tif` / `.tiff` | | `hina_microscope` | `hina_microscope` | `.nd2` | | `plate_reader` | `spectramax_plate_reader` | `.xls` | +| `dishcam` | `dishcam` | `.tif` / `.tiff` / `run.json` | | `generic`, `instant_raman` | — | — | **One type = one vendor's output format.** Names like `qpcr` and `fplc` sound generic, but the parsers behind them are vendor-specific (Azure Cielo, ÄKTA, …). Adding a second vendor under an existing type requires splitting the type, not reusing it. @@ -98,9 +99,10 @@ Available commands: | `qpcr` | Parse dye channels from an Azure Cielo qPCR Cq Values CSV | | `spectramax` | Parse metadata and raw well data from a SpectraMax `.xls` export | | `tapestation` | Extract the tape type from a TapeStation CSV filename | +| `dishcam` | Convert a DishCam TIFF stack plus `run.json` into an MP4 preview and JPEG poster | | `handler` | Stage a file into a local S3 mirror and invoke `lambda_handler` against the local dev API. See [Testing the Lambda end-to-end](local-development.md#testing-the-lambda-end-to-end) for the workflow. | -The first six subcommands need no S3 or API access — they call into the same parsing/processing utilities the lambda uses, but stop short of the network. `handler` is different: it expects a running dev API and a `LOCAL_S3_MIRROR` directory, and uses the same dispatch path production uses. +The instrument-specific subcommands need no S3 or API access — they call into the same parsing/processing utilities the lambda uses, but stop short of the network. `handler` is different: it expects a running dev API and a `LOCAL_S3_MIRROR` directory, and uses the same dispatch path production uses. Examples: @@ -120,8 +122,9 @@ make docker-build-lambda The Dockerfile is a multi-stage build: -1. **Builder stage**: Uses `uv` to export and install third-party dependencies into the Lambda task root. -2. **Final stage**: Copies the installed dependencies plus the `data_hub_shared` and `data_hub_lambda` source packages. +1. **ffmpeg stage**: Copies a static linux/amd64 `ffmpeg` (libx264) from the version-tagged `mwader/static-ffmpeg` image. +2. **Builder stage**: Uses `uv` to export and install third-party dependencies into the Lambda task root. +3. **Final stage**: Copies ffmpeg, the installed dependencies, and the `data_hub_shared` and `data_hub_lambda` source packages. The entry point is `data_hub_lambda.handler.lambda_handler`. @@ -134,6 +137,7 @@ The Lambda function depends on a scientific Python stack: - `matplotlib` — plotting - `scikit-image` — image processing - `tifffile` — TIFF file reading +- `ffmpeg` — static linux/amd64 binary from `mwader/static-ffmpeg` for DishCam H.264 encode - `arcadia-microscopy-tools` — ND2 reading, channel handling, and multi-channel compositing for the Hina microscope - `pydantic` — data validation - `requests` — HTTP client for the Data Hub API diff --git a/lambda/Dockerfile b/lambda/Dockerfile index 7e26703f..24cf1221 100644 --- a/lambda/Dockerfile +++ b/lambda/Dockerfile @@ -1,5 +1,9 @@ FROM ghcr.io/astral-sh/uv:0.8.12 AS uv +# Version-tagged static build with libx264. BtbN autobuild URLs are +# deleted; this Hub tag stays pullable. linux/amd64 matches Lambda. +FROM --platform=linux/amd64 mwader/static-ffmpeg:8.1.2 AS ffmpeg + FROM public.ecr.aws/lambda/python:3.12 AS builder ENV UV_COMPILE_BYTECODE=1 @@ -21,6 +25,7 @@ RUN --mount=from=uv,source=/uv,target=/bin/uv \ FROM public.ecr.aws/lambda/python:3.12 +COPY --from=ffmpeg /ffmpeg /usr/local/bin/ffmpeg COPY --from=builder ${LAMBDA_TASK_ROOT} ${LAMBDA_TASK_ROOT} # Workspace packages are copied as source (not pip-installed) so they land # directly on the Python path alongside the third-party deps. diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 1b6c30e5..53273ff8 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -227,6 +227,46 @@ def tapestation(filename: str) -> None: click.echo("No tape type found in filename.") +# --------------------------------------------------------------------------- +# DishCam +# --------------------------------------------------------------------------- + + +@cli.command("dishcam") +@click.argument("tiff", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option( + "--run-json", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, + help="Sidecar run.json (fps / frame count).", +) +@click.option( + "--output-dir", + type=click.Path(file_okay=False, path_type=Path), + default=None, + help="Directory for the MP4 and JPEG poster (default: same directory as TIFF).", +) +def dishcam_cmd(tiff: Path, run_json: Path, output_dir: Path | None) -> None: + """Convert a DishCam TIFF stack to an MP4 preview and JPEG poster.""" + from data_hub_lambda.dishcam.encode_video import encode_tiff_stack + from data_hub_lambda.dishcam.parse_metadata import ( + encode_fps, + parse_run_json, + playback_fps, + ) + + metadata = parse_run_json(run_json) + fps = playback_fps(encode_fps(metadata)) + dest_dir = output_dir or tiff.parent + dest_dir.mkdir(parents=True, exist_ok=True) + mp4_path = dest_dir / f"{tiff.stem}.mp4" + poster_path = dest_dir / f"{tiff.stem}.jpg" + encode_tiff_stack(tiff, mp4_path, poster_path, fps) + click.echo(f"Exported MP4: {mp4_path}") + click.echo(f"Exported JPEG: {poster_path}") + click.echo(json.dumps(metadata, indent=2)) + + # --------------------------------------------------------------------------- # End-to-end handler invocation against a local S3 mirror # --------------------------------------------------------------------------- diff --git a/lambda/src/data_hub_lambda/dishcam/__init__.py b/lambda/src/data_hub_lambda/dishcam/__init__.py new file mode 100644 index 00000000..84b4a274 --- /dev/null +++ b/lambda/src/data_hub_lambda/dishcam/__init__.py @@ -0,0 +1,3 @@ +from data_hub_lambda.dishcam.process_file import process_file + +__all__ = ["process_file"] diff --git a/lambda/src/data_hub_lambda/dishcam/encode_video.py b/lambda/src/data_hub_lambda/dishcam/encode_video.py new file mode 100644 index 00000000..10240491 --- /dev/null +++ b/lambda/src/data_hub_lambda/dishcam/encode_video.py @@ -0,0 +1,183 @@ +"""Stream a DishCam TIFF stack through ffmpeg into an HTML5 MP4 + JPEG poster.""" + +from __future__ import annotations +import logging +import shutil +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import numpy as np +import tifffile +from numpy.typing import NDArray + +logger = logging.getLogger(__name__) + +# Width cap keeps the preview inside H.264 Level 5.2 / browser limits +# (a 3040x4056 stack will not play in Safari). `h=-2` keeps height even +# and aspect; the trunc expression makes width even for yuv420p. +_SCALE_FILTER = "scale=w='trunc(min(1920,iw)/2)*2':h=-2" + +# Progress lines would fill the stderr pipe (~64 KB) and deadlock a +# long encode. Failures still land in the temp-file capture below. +_FFMPEG_QUIET = ("-nostats", "-loglevel", "error") + + +def resolve_ffmpeg() -> str: + path = shutil.which("ffmpeg") + if path is None: + raise RuntimeError("ffmpeg is not on PATH") + return path + + +def _ffmpeg_rate(fps: float) -> str: + """Fixed-point rate. ffmpeg rejects scientific notation like `1e-05`.""" + return f"{fps:.6f}".rstrip("0").rstrip(".") + + +def _as_rgb24(frame: NDArray) -> NDArray: # type: ignore[type-arg] + """Return a C-contiguous uint8 RGB frame for rawvideo stdin. + + DishCam writes uint8. A later camera with a wider type needs an + explicit converter — do not guess 12-bit from a uint16 peak. + """ + if frame.ndim == 2: + frame = np.stack([frame, frame, frame], axis=-1) + elif frame.ndim == 3 and frame.shape[-1] == 4: + frame = frame[..., :3] + elif frame.ndim != 3 or frame.shape[-1] != 3: + raise ValueError(f"Unsupported TIFF page shape: {frame.shape}") + if frame.dtype != np.uint8: + raise ValueError(f"Expected uint8 RGB pages, got {frame.dtype}") + return np.ascontiguousarray(frame) + + +def _pipe_ffmpeg(cmd: list[str], chunks: Iterator[bytes]) -> None: + """Write raw frames to ffmpeg; kill the child if the parent fails. + + stderr goes to a temp file so a chatty encode cannot fill a pipe + and deadlock against stdin. + """ + with tempfile.NamedTemporaryFile() as err_file: + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=err_file, + ) + assert proc.stdin is not None + try: + try: + for chunk in chunks: + proc.stdin.write(chunk) + proc.stdin.close() + except BrokenPipeError: + pass + proc.wait() + except Exception: + proc.kill() + proc.wait() + raise + err_file.seek(0) + stderr = err_file.read().decode(errors="replace") + if proc.returncode != 0: + raise RuntimeError(f"ffmpeg encode failed: {stderr}") + + +def _write_jpeg(ffmpeg: str, frame: NDArray, dest: Path) -> None: # type: ignore[type-arg] + height, width = frame.shape[:2] + dest.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + ffmpeg, + "-y", + *_FFMPEG_QUIET, + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-s:v", + f"{width}x{height}", + "-i", + "pipe:0", + "-vf", + _SCALE_FILTER, + "-frames:v", + "1", + "-q:v", + "3", + str(dest), + ] + _pipe_ffmpeg(cmd, iter((frame.tobytes(),))) + + +def encode_tiff_stack( + tiff_path: Path, + mp4_path: Path, + poster_path: Path, + fps: float, +) -> None: + """Decode pages one at a time and pipe RGB24 into ffmpeg. + + Loading the full stack would be ~35 MB per 12 MP frame; real runs can + be hundreds of frames. + """ + ffmpeg = resolve_ffmpeg() + mp4_path.parent.mkdir(parents=True, exist_ok=True) + + with tifffile.TiffFile(tiff_path) as tif: + if not tif.pages: + raise ValueError(f"{tiff_path.name} has no TIFF pages") + + first = _as_rgb24(tif.pages[0].asarray()) + height, width = first.shape[:2] + _write_jpeg(ffmpeg, first, poster_path) + + cmd = [ + ffmpeg, + "-y", + *_FFMPEG_QUIET, + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-s:v", + f"{width}x{height}", + "-r", + _ffmpeg_rate(fps), + "-i", + "pipe:0", + "-vf", + _SCALE_FILTER, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-crf", + "23", + "-preset", + "veryfast", + "-movflags", + "+faststart", + # Every frame is a keyframe so the in-app player can scrub + # a short timelapse without waiting for the next GOP. + "-g", + "1", + "-an", + str(mp4_path), + ] + logger.info( + "Encoding %s (%dx%d, %s fps) → %s", + tiff_path.name, + width, + height, + fps, + mp4_path.name, + ) + + def _chunks() -> Iterator[bytes]: + yield first.tobytes() + for page in tif.pages[1:]: + yield _as_rgb24(page.asarray()).tobytes() + + _pipe_ffmpeg(cmd, _chunks()) diff --git a/lambda/src/data_hub_lambda/dishcam/filenames.py b/lambda/src/data_hub_lambda/dishcam/filenames.py new file mode 100644 index 00000000..ed55ac8f --- /dev/null +++ b/lambda/src/data_hub_lambda/dishcam/filenames.py @@ -0,0 +1,24 @@ +"""Filename helpers for DishCam S3 gates and sibling lookup.""" + +from __future__ import annotations + +RUN_JSON_NAME = "run.json" +_TIFF_SUFFIXES = (".tif", ".tiff") + + +def is_run_json(filename: str) -> bool: + return filename.lower() == RUN_JSON_NAME + + +def is_tiff(filename: str) -> bool: + return filename.lower().endswith(_TIFF_SUFFIXES) + + +def matches_filename(filename: str) -> bool: + """S3 events for a stack or the sidecar can start encode. + + `run.json` therefore also passes the handler's union gate for every + instrument type. The extra instrument lookup is cheap; the per-type + gate still rejects it on non-DishCam instruments. + """ + return is_tiff(filename) or is_run_json(filename) diff --git a/lambda/src/data_hub_lambda/dishcam/parse_metadata.py b/lambda/src/data_hub_lambda/dishcam/parse_metadata.py new file mode 100644 index 00000000..d316c781 --- /dev/null +++ b/lambda/src/data_hub_lambda/dishcam/parse_metadata.py @@ -0,0 +1,62 @@ +"""Parse DishCam sidecar `run.json` into run metadata and encode fps.""" + +from __future__ import annotations +import json +from pathlib import Path +from typing import Any + +_METADATA_KEYS = ( + "fps", + "measured_fps", + "frames", + "duration_seconds", + "quality", + "format", + "started", + "finished", +) + + +def parse_run_json(path: Path) -> dict[str, Any]: + """Return the sidecar fields stored on the run. + + Raises `ValueError` when the file is not an object or has no usable fps. + """ + try: + payload = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid run.json: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError("run.json must be a JSON object") + + metadata = { + key: payload[key] for key in _METADATA_KEYS if key in payload and payload[key] is not None + } + encode_fps(metadata) + return metadata + + +def encode_fps(metadata: dict[str, Any]) -> float: + """Prefer measured fps when present; otherwise use the planned fps.""" + raw = metadata.get("measured_fps") + if raw is None: + raw = metadata.get("fps") + if raw is None: + raise ValueError("run.json is missing fps and measured_fps") + try: + fps = float(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"run.json fps is not a number: {raw!r}") from exc + if fps <= 0: + raise ValueError(f"run.json fps must be positive, got {fps}") + return fps + + +# Multi-day captures can land well below 1 fps. The in-app player is a +# preview, so floor the encode rate rather than playing in real time. +MIN_PLAYBACK_FPS = 10.0 + + +def playback_fps(capture_fps: float) -> float: + """Return the fps used for the MP4 preview, not the stored metadata.""" + return max(capture_fps, MIN_PLAYBACK_FPS) diff --git a/lambda/src/data_hub_lambda/dishcam/process_file.py b/lambda/src/data_hub_lambda/dishcam/process_file.py new file mode 100644 index 00000000..ac2f6c9f --- /dev/null +++ b/lambda/src/data_hub_lambda/dishcam/process_file.py @@ -0,0 +1,263 @@ +"""Process DishCam TIFF stacks once `run.json` is also in S3.""" + +from __future__ import annotations +import logging +from pathlib import Path + +from data_hub_lambda.api_client import ApiError, DataHubClient, get_client +from data_hub_lambda.dishcam.encode_video import encode_tiff_stack +from data_hub_lambda.dishcam.filenames import RUN_JSON_NAME, is_tiff, matches_filename +from data_hub_lambda.dishcam.parse_metadata import encode_fps, parse_run_json, playback_fps +from data_hub_shared import s3_utils +from data_hub_shared.config import config + +logger = logging.getLogger(__name__) + + +def process_file(instrument_id: str, run_id: str, filename: str) -> None: + """Encode each TIFF after both the stack(s) and sidecar exist. + + S3 can notify on a stack or on `run.json` first. If the sibling is + missing, return without creating a run or flipping status — the later + event encodes. + + A TIFF event encodes that stack only. A `run.json` event encodes every + TIFF currently under the run prefix, so a sidecar that lands last still + produces an MP4 per stack. Later TIFFs encode themselves once the + sidecar is already in S3. + + Reprocess already marks the trigger `processing`, so a missing sibling + fails that file instead of leaving it stuck. + + TIFF and `run.json` are separate S3 events, so two invocations can + encode the same stack. `create_file` is idempotent; completed/failed + updates swallow 409 so the loser does not fail a successful run. + Duplicate compute is accepted — a lock would need run-level state + we do not have. + """ + if not matches_filename(filename): + logger.info("Ignoring DishCam file %s; not a TIFF or run.json.", filename) + return + + raw_bucket = config.AWS_S3_RAW_DATA_BUCKET or "" + json_key = f"{instrument_id}/{run_id}/{RUN_JSON_NAME}" + json_uri = f"s3://{raw_bucket}/{json_key}" + tiff_filenames = _tiff_filenames_to_process(raw_bucket, instrument_id, run_id, filename) + + json_exists = s3_utils.object_exists(json_uri) + if not tiff_filenames or not json_exists: + missing = "run.json" if not json_exists else "TIFF stack" + logger.info("DishCam run %s is missing %s; skipping.", run_id, missing) + _fail_if_processing( + instrument_id, + run_id, + filename, + f"Cannot process: {missing} not found in S3", + ) + return + + logger.info( + "Processing DishCam TIFF%s %s (run: %s)", + "" if len(tiff_filenames) == 1 else "s", + ", ".join(tiff_filenames), + run_id, + ) + client = get_client() + client.ensure_run(instrument_id, run_id) + + raw_dir = config.LOCAL_RAW_DATA_DIRPATH / instrument_id / run_id + local_json = raw_dir / RUN_JSON_NAME + try: + s3_utils.download_file(json_uri, local_json) + metadata = parse_run_json(local_json) + fps = playback_fps(encode_fps(metadata)) + except Exception as exc: + logger.error("Error reading DishCam run.json for %s: %s", run_id, exc) + for tiff_filename in tiff_filenames: + tiff_key = f"{instrument_id}/{run_id}/{tiff_filename}" + record = client.create_file( + instrument_id=instrument_id, + run_id=run_id, + s3_bucket=raw_bucket, + s3_key=tiff_key, + filename=tiff_filename, + ) + _update_file_status(client, record.id, "failed", error_message=str(exc)) + raise + + last_error: Exception | None = None + encoded_any = False + for tiff_filename in tiff_filenames: + try: + _encode_tiff( + client, + instrument_id, + run_id, + raw_bucket, + raw_dir, + tiff_filename, + fps, + ) + encoded_any = True + except Exception as exc: + logger.error("Error processing DishCam file %s: %s", tiff_filename, exc) + last_error = exc + + if encoded_any: + client.update_run(instrument_id, run_id, metadata=metadata) + if last_error is not None: + raise last_error + + +def _tiff_filenames_to_process( + raw_bucket: str, + instrument_id: str, + run_id: str, + filename: str, +) -> list[str]: + if is_tiff(filename): + return [filename] + return _list_tiff_filenames(raw_bucket, instrument_id, run_id) + + +def _list_tiff_filenames(raw_bucket: str, instrument_id: str, run_id: str) -> list[str]: + prefix = f"s3://{raw_bucket}/{instrument_id}/{run_id}/" + names: list[str] = [] + for uri in sorted(s3_utils.list_objects(prefix)): + name = uri.rsplit("/", 1)[-1] + if is_tiff(name): + names.append(name) + return names + + +def _encode_tiff( + client: DataHubClient, + instrument_id: str, + run_id: str, + raw_bucket: str, + raw_dir: Path, + tiff_filename: str, + fps: float, +) -> None: + tiff_key = f"{instrument_id}/{run_id}/{tiff_filename}" + tiff_uri = f"s3://{raw_bucket}/{tiff_key}" + tiff_record = client.create_file( + instrument_id=instrument_id, + run_id=run_id, + s3_bucket=raw_bucket, + s3_key=tiff_key, + filename=tiff_filename, + ) + tiff_id = tiff_record.id + + try: + client.update_file(tiff_id, status="processing") + + local_tiff = raw_dir / tiff_filename + s3_utils.download_file(tiff_uri, local_tiff) + + mp4_path = raw_dir / f"{Path(tiff_filename).stem}.mp4" + poster_path = raw_dir / f"{Path(tiff_filename).stem}.jpg" + encode_tiff_stack(local_tiff, mp4_path, poster_path, fps) + + processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET or "" + _upload_processed( + client, + instrument_id, + run_id, + processed_bucket, + mp4_path, + "video/mp4", + ) + _upload_processed( + client, + instrument_id, + run_id, + processed_bucket, + poster_path, + "image/jpeg", + ) + + if not _update_file_status(client, tiff_id, "completed"): + logger.info( + "DishCam file %s already finished by a sibling invocation.", + tiff_filename, + ) + return + logger.info("DishCam file %s marked as completed.", tiff_filename) + except Exception as exc: + _update_file_status(client, tiff_id, "failed", error_message=str(exc)) + raise + + +def _upload_processed( + client: DataHubClient, + instrument_id: str, + run_id: str, + processed_bucket: str, + local_path: Path, + content_type: str, +) -> None: + s3_key = f"{instrument_id}/{run_id}/{local_path.name}" + s3_utils.upload_file(local_path, f"s3://{processed_bucket}/{s3_key}") + processed = client.create_file( + instrument_id=instrument_id, + run_id=run_id, + s3_bucket=processed_bucket, + s3_key=s3_key, + filename=local_path.name, + category="processed", + ) + client.update_file( + processed.id, + size_bytes=local_path.stat().st_size, + content_type=content_type, + ) + + +def _update_file_status( + client: DataHubClient, + file_id: int, + status: str, + *, + error_message: str | None = None, +) -> bool: + """Set status. Return False if another invocation already moved the file.""" + try: + client.update_file(file_id, status=status, error_message=error_message) + except ApiError as exc: + if exc.status_code == 409: + logger.info( + "File %s status conflict when setting %s (sibling encode likely won).", + file_id, + status, + ) + return False + raise + return True + + +def _fail_if_processing( + instrument_id: str, + run_id: str, + filename: str, + error_message: str, +) -> None: + """Fail a reprocess that is already `processing` when a sibling is missing.""" + raw_bucket = config.AWS_S3_RAW_DATA_BUCKET or "" + s3_key = f"{instrument_id}/{run_id}/{filename}" + try: + client = get_client() + record = client.create_file( + instrument_id=instrument_id, + run_id=run_id, + s3_bucket=raw_bucket, + s3_key=s3_key, + filename=filename, + ) + except ApiError as exc: + if exc.status_code == 404: + return + raise + if record.status == "processing": + client.update_file(record.id, status="failed", error_message=error_message) diff --git a/lambda/src/data_hub_lambda/processors.py b/lambda/src/data_hub_lambda/processors.py index 595f6b4f..29e4148c 100644 --- a/lambda/src/data_hub_lambda/processors.py +++ b/lambda/src/data_hub_lambda/processors.py @@ -15,10 +15,12 @@ akta_fplc, azure_600_gel_doc, azure_cielo_qpcr, + dishcam, epson_v700_scanner, hina_microscope, spectramax_plate_reader, ) +from data_hub_lambda.dishcam.filenames import matches_filename as _is_dishcam_input ProcessFileFn = Callable[[str, str, str], None] @@ -70,6 +72,10 @@ def _match(filename: str) -> bool: process_file=akta_fplc.process_file, matches_filename=_ends_with_any(".pdf"), ), + "dishcam": ProcessorEntry( + process_file=dishcam.process_file, + matches_filename=_is_dishcam_input, + ), } diff --git a/lambda/tests/dishcam/test_encode_video.py b/lambda/tests/dishcam/test_encode_video.py new file mode 100644 index 00000000..3a1a8b2b --- /dev/null +++ b/lambda/tests/dishcam/test_encode_video.py @@ -0,0 +1,69 @@ +"""Unit tests for DishCam ffmpeg encode.""" + +from __future__ import annotations +import shutil +from pathlib import Path + +import numpy as np +import pytest +import tifffile + +from data_hub_lambda.dishcam.encode_video import ( + _ffmpeg_rate, + encode_tiff_stack, + resolve_ffmpeg, +) + +requires_ffmpeg = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg is not on PATH") + + +def _write_stack(path: Path, frames: int = 4, width: int = 64, height: int = 48) -> Path: + pages = [] + for i in range(frames): + img = np.zeros((height, width, 3), dtype=np.uint8) + img[..., 0] = 30 + i * 15 + img[..., 1] = 90 + img[..., 2] = 180 + pages.append(img) + tifffile.imwrite(path, np.stack(pages), photometric="rgb") + return path + + +@requires_ffmpeg +def test_encode_writes_mp4_and_poster(tmp_path: Path) -> None: + tiff = _write_stack(tmp_path / "stack.tif") + mp4 = tmp_path / "stack.mp4" + poster = tmp_path / "stack.jpg" + + encode_tiff_stack(tiff, mp4, poster, fps=1.0) + + assert mp4.is_file() + assert mp4.stat().st_size > 0 + assert poster.is_file() + assert poster.read_bytes()[:2] == b"\xff\xd8" + # ISO BMFF / MP4 brand + assert b"ftyp" in mp4.read_bytes()[:32] + + +@requires_ffmpeg +def test_encode_odd_width_stack(tmp_path: Path) -> None: + tiff = _write_stack(tmp_path / "odd.tif", frames=2, width=47, height=48) + mp4 = tmp_path / "odd.mp4" + poster = tmp_path / "odd.jpg" + + encode_tiff_stack(tiff, mp4, poster, fps=10.0) + + assert mp4.is_file() + assert mp4.stat().st_size > 0 + assert poster.is_file() + + +@requires_ffmpeg +def test_resolve_ffmpeg_finds_binary() -> None: + assert Path(resolve_ffmpeg()).name.startswith("ffmpeg") + + +def test_ffmpeg_rate_avoids_scientific_notation() -> None: + assert "e" not in _ffmpeg_rate(1e-5).lower() + assert _ffmpeg_rate(10.0) == "10" + assert _ffmpeg_rate(0.95) == "0.95" diff --git a/lambda/tests/dishcam/test_parse_metadata.py b/lambda/tests/dishcam/test_parse_metadata.py new file mode 100644 index 00000000..67871e53 --- /dev/null +++ b/lambda/tests/dishcam/test_parse_metadata.py @@ -0,0 +1,61 @@ +"""Unit tests for DishCam `run.json` parsing.""" + +from __future__ import annotations +import json +from pathlib import Path + +import pytest + +from data_hub_lambda.dishcam.parse_metadata import ( + MIN_PLAYBACK_FPS, + encode_fps, + parse_run_json, + playback_fps, +) + + +def _write_json(path: Path, payload: object) -> Path: + path.write_text(json.dumps(payload)) + return path + + +def test_prefers_measured_fps(tmp_path: Path) -> None: + sidecar = _write_json( + tmp_path / "run.json", + {"fps": 1.0, "measured_fps": 0.95, "frames": 10, "quality": "High"}, + ) + metadata = parse_run_json(sidecar) + assert encode_fps(metadata) == 0.95 + assert metadata["fps"] == 1.0 + assert metadata["frames"] == 10 + + +def test_falls_back_to_planned_fps(tmp_path: Path) -> None: + sidecar = _write_json(tmp_path / "run.json", {"fps": 2.5, "frames": 8}) + metadata = parse_run_json(sidecar) + assert encode_fps(metadata) == 2.5 + + +def test_rejects_missing_fps(tmp_path: Path) -> None: + sidecar = _write_json(tmp_path / "run.json", {"frames": 8}) + with pytest.raises(ValueError, match="missing fps"): + parse_run_json(sidecar) + + +def test_rejects_non_object(tmp_path: Path) -> None: + sidecar = _write_json(tmp_path / "run.json", ["not", "an", "object"]) + with pytest.raises(ValueError, match="JSON object"): + parse_run_json(sidecar) + + +def test_omits_missing_and_null_keys(tmp_path: Path) -> None: + sidecar = _write_json(tmp_path / "run.json", {"fps": 1.0, "quality": None}) + metadata = parse_run_json(sidecar) + assert "duration_seconds" not in metadata + assert "quality" not in metadata + assert metadata["fps"] == 1.0 + + +def test_playback_fps_floors_slow_captures() -> None: + assert playback_fps(0.0167) == MIN_PLAYBACK_FPS + assert playback_fps(24.0) == 24.0 diff --git a/lambda/tests/dishcam/test_process_file.py b/lambda/tests/dishcam/test_process_file.py new file mode 100644 index 00000000..50ca1281 --- /dev/null +++ b/lambda/tests/dishcam/test_process_file.py @@ -0,0 +1,536 @@ +"""Unit tests for DishCam `process_file` orchestration.""" + +from __future__ import annotations +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from data_hub_lambda.api_client import ApiError +from data_hub_lambda.dishcam.parse_metadata import MIN_PLAYBACK_FPS +from data_hub_lambda.models import FileResponse, RunResponse + + +@pytest.fixture(autouse=True) +def _reset_api_client() -> Any: + import data_hub_lambda.api_client as api_module + + original = api_module._client + api_module._client = None + try: + yield + finally: + api_module._client = original + + +def _file_response( + file_id: int, + filename: str, + status: str = "uploaded", + category: str = "raw", +) -> FileResponse: + return FileResponse( + id=file_id, + instrument_run_id="run-uuid", + filename=filename, + s3_bucket="raw", + s3_key=f"dishcam/run-xyz/{filename}", + category=category, + status=status, + ) + + +def _run_response() -> RunResponse: + return RunResponse( + id="run-uuid", + instrument_id="dishcam", + run_id="run-xyz", + source="lambda", + metadata={}, + ) + + +def _exists_for(*keys: str): + present = set(keys) + + def _exists(s3_uri: str, **_: Any) -> bool: + key = s3_uri.split("//", 1)[1].split("/", 1)[1] + return key in present + + return _exists + + +class TestProcessFileSkipUntilBothPresent: + def test_tiff_without_json_does_not_ensure_run_or_download(self) -> None: + client = MagicMock() + client.create_file.return_value = _file_response(1, "stack.tif") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + side_effect=_exists_for("dishcam/run-xyz/stack.tif"), + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.download_file") as download, + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "stack.tif") + + client.ensure_run.assert_not_called() + download.assert_not_called() + client.update_file.assert_not_called() + + def test_json_without_tiff_does_not_encode(self) -> None: + client = MagicMock() + client.create_file.return_value = _file_response(2, "run.json") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + side_effect=_exists_for("dishcam/run-xyz/run.json"), + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + return_value=[], + ), + patch("data_hub_lambda.dishcam.process_file.encode_tiff_stack") as encode, + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "run.json") + + encode.assert_not_called() + client.ensure_run.assert_not_called() + + def test_reprocess_fails_when_sibling_missing(self) -> None: + client = MagicMock() + client.create_file.return_value = _file_response(3, "stack.tif", status="processing") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + side_effect=_exists_for("dishcam/run-xyz/stack.tif"), + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "stack.tif") + + client.update_file.assert_called_once_with( + 3, + status="failed", + error_message="Cannot process: run.json not found in S3", + ) + + def test_missing_run_on_skip_is_not_an_error(self) -> None: + client = MagicMock() + client.create_file.side_effect = ApiError("not found", status_code=404) + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=False, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "stack.tif") + + client.update_file.assert_not_called() + + +class TestProcessFileEncodesWhenBothPresent: + def test_run_json_trigger_encodes_sibling_tiff(self, tmp_path: Path) -> None: + client = MagicMock() + client.ensure_run.return_value = _run_response() + client.create_file.side_effect = [ + _file_response(10, "stack.tif"), + _file_response(11, "stack.mp4", category="processed"), + _file_response(12, "stack.jpg", category="processed"), + ] + + tiff = tmp_path / "stack.tif" + sidecar = tmp_path / "run.json" + tiff.write_bytes(b"tiff") + sidecar.write_text("{}") + mp4 = tmp_path / "stack.mp4" + poster = tmp_path / "stack.jpg" + mp4.write_bytes(b"mp4") + poster.write_bytes(b"jpg") + + def _download(s3_uri: str, local_path: Path, **_: Any) -> None: + local_path.parent.mkdir(parents=True, exist_ok=True) + if s3_uri.endswith("run.json"): + local_path.write_text( + '{"fps": 1.0, "measured_fps": 0.9, "frames": 4, "quality": "High"}' + ) + else: + local_path.write_bytes(b"tiff") + + def _encode(tiff_path: Path, mp4_path: Path, poster_path: Path, fps: float) -> None: + assert fps == MIN_PLAYBACK_FPS + mp4_path.write_bytes(b"mp4") + poster_path.write_bytes(b"jpg") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=True, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + return_value=["s3://raw/dishcam/run-xyz/stack.tif"], + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.download_file", + side_effect=_download, + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.upload_file"), + patch( + "data_hub_lambda.dishcam.process_file.encode_tiff_stack", + side_effect=_encode, + ), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "raw", + ), + patch( + "data_hub_shared.config.config.AWS_S3_PROCESSED_DATA_BUCKET", + "processed", + ), + patch( + "data_hub_shared.config.config.LOCAL_RAW_DATA_DIRPATH", + tmp_path, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "run.json") + + client.ensure_run.assert_called_once_with("dishcam", "run-xyz") + client.update_run.assert_called_once() + metadata = client.update_run.call_args.kwargs["metadata"] + assert metadata["measured_fps"] == 0.9 + assert metadata["frames"] == 4 + completed = [ + call + for call in client.update_file.call_args_list + if call.kwargs.get("status") == "completed" + ] + assert completed + assert completed[-1].args[0] == 10 + + def test_completed_conflict_is_not_a_failure(self, tmp_path: Path) -> None: + client = MagicMock() + client.ensure_run.return_value = _run_response() + client.create_file.side_effect = [ + _file_response(10, "stack.tif"), + _file_response(11, "stack.mp4", category="processed"), + _file_response(12, "stack.jpg", category="processed"), + ] + client.update_file.side_effect = [ + _file_response(10, "stack.tif", status="processing"), + _file_response(11, "stack.mp4", category="processed"), + _file_response(12, "stack.jpg", category="processed"), + ApiError("conflict", status_code=409), + ] + + def _download(s3_uri: str, local_path: Path, **_: Any) -> None: + local_path.parent.mkdir(parents=True, exist_ok=True) + if s3_uri.endswith("run.json"): + local_path.write_text('{"fps": 1.0}') + else: + local_path.write_bytes(b"tiff") + + def _encode(tiff_path: Path, mp4_path: Path, poster_path: Path, fps: float) -> None: + mp4_path.write_bytes(b"mp4") + poster_path.write_bytes(b"jpg") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=True, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + return_value=["s3://raw/dishcam/run-xyz/stack.tif"], + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.download_file", + side_effect=_download, + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.upload_file"), + patch( + "data_hub_lambda.dishcam.process_file.encode_tiff_stack", + side_effect=_encode, + ), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "raw", + ), + patch( + "data_hub_shared.config.config.AWS_S3_PROCESSED_DATA_BUCKET", + "processed", + ), + patch( + "data_hub_shared.config.config.LOCAL_RAW_DATA_DIRPATH", + tmp_path, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "stack.tif") + + failed = [ + call + for call in client.update_file.call_args_list + if call.kwargs.get("status") == "failed" + ] + assert failed == [] + + def test_run_json_trigger_encodes_every_tiff(self, tmp_path: Path) -> None: + client = MagicMock() + client.ensure_run.return_value = _run_response() + client.create_file.side_effect = [ + _file_response(10, "empty.tif"), + _file_response(11, "empty.mp4", category="processed"), + _file_response(12, "empty.jpg", category="processed"), + _file_response(20, "ruler.tif"), + _file_response(21, "ruler.mp4", category="processed"), + _file_response(22, "ruler.jpg", category="processed"), + ] + + def _download(s3_uri: str, local_path: Path, **_: Any) -> None: + local_path.parent.mkdir(parents=True, exist_ok=True) + if s3_uri.endswith("run.json"): + local_path.write_text('{"fps": 1.0}') + else: + local_path.write_bytes(b"tiff") + + def _encode(tiff_path: Path, mp4_path: Path, poster_path: Path, fps: float) -> None: + mp4_path.write_bytes(b"mp4") + poster_path.write_bytes(b"jpg") + + encode = MagicMock(side_effect=_encode) + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=True, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + return_value=[ + "s3://raw/dishcam/run-xyz/empty.tif", + "s3://raw/dishcam/run-xyz/ruler.tif", + "s3://raw/dishcam/run-xyz/run.json", + ], + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.download_file", + side_effect=_download, + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.upload_file"), + patch( + "data_hub_lambda.dishcam.process_file.encode_tiff_stack", + encode, + ), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "raw", + ), + patch( + "data_hub_shared.config.config.AWS_S3_PROCESSED_DATA_BUCKET", + "processed", + ), + patch( + "data_hub_shared.config.config.LOCAL_RAW_DATA_DIRPATH", + tmp_path, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "run.json") + + encoded = [call.args[0].name for call in encode.call_args_list] + assert encoded == ["empty.tif", "ruler.tif"] + completed = [ + call.args[0] + for call in client.update_file.call_args_list + if call.kwargs.get("status") == "completed" + ] + assert completed == [10, 20] + + def test_tiff_trigger_encodes_only_that_stack(self, tmp_path: Path) -> None: + client = MagicMock() + client.ensure_run.return_value = _run_response() + client.create_file.side_effect = [ + _file_response(20, "ruler.tif"), + _file_response(21, "ruler.mp4", category="processed"), + _file_response(22, "ruler.jpg", category="processed"), + ] + + def _download(s3_uri: str, local_path: Path, **_: Any) -> None: + local_path.parent.mkdir(parents=True, exist_ok=True) + if s3_uri.endswith("run.json"): + local_path.write_text('{"fps": 1.0}') + else: + local_path.write_bytes(b"tiff") + + def _encode(tiff_path: Path, mp4_path: Path, poster_path: Path, fps: float) -> None: + mp4_path.write_bytes(b"mp4") + poster_path.write_bytes(b"jpg") + + encode = MagicMock(side_effect=_encode) + list_objects = MagicMock( + return_value=[ + "s3://raw/dishcam/run-xyz/empty.tif", + "s3://raw/dishcam/run-xyz/ruler.tif", + ] + ) + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=True, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + list_objects, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.download_file", + side_effect=_download, + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.upload_file"), + patch( + "data_hub_lambda.dishcam.process_file.encode_tiff_stack", + encode, + ), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "raw", + ), + patch( + "data_hub_shared.config.config.AWS_S3_PROCESSED_DATA_BUCKET", + "processed", + ), + patch( + "data_hub_shared.config.config.LOCAL_RAW_DATA_DIRPATH", + tmp_path, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + process_file("dishcam", "run-xyz", "ruler.tif") + + list_objects.assert_not_called() + assert [call.args[0].name for call in encode.call_args_list] == ["ruler.tif"] + + def test_one_failed_stack_does_not_block_the_others(self, tmp_path: Path) -> None: + client = MagicMock() + client.ensure_run.return_value = _run_response() + client.create_file.side_effect = [ + _file_response(10, "empty.tif"), + _file_response(20, "ruler.tif"), + _file_response(21, "ruler.mp4", category="processed"), + _file_response(22, "ruler.jpg", category="processed"), + ] + + def _download(s3_uri: str, local_path: Path, **_: Any) -> None: + local_path.parent.mkdir(parents=True, exist_ok=True) + if s3_uri.endswith("run.json"): + local_path.write_text('{"fps": 1.0}') + else: + local_path.write_bytes(b"tiff") + + def _encode(tiff_path: Path, mp4_path: Path, poster_path: Path, fps: float) -> None: + if tiff_path.name == "empty.tif": + raise RuntimeError("empty stack is corrupt") + mp4_path.write_bytes(b"mp4") + poster_path.write_bytes(b"jpg") + + with ( + patch( + "data_hub_lambda.dishcam.process_file.get_client", + return_value=client, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.object_exists", + return_value=True, + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.list_objects", + return_value=[ + "s3://raw/dishcam/run-xyz/empty.tif", + "s3://raw/dishcam/run-xyz/ruler.tif", + ], + ), + patch( + "data_hub_lambda.dishcam.process_file.s3_utils.download_file", + side_effect=_download, + ), + patch("data_hub_lambda.dishcam.process_file.s3_utils.upload_file"), + patch( + "data_hub_lambda.dishcam.process_file.encode_tiff_stack", + side_effect=_encode, + ), + patch( + "data_hub_shared.config.config.AWS_S3_RAW_DATA_BUCKET", + "raw", + ), + patch( + "data_hub_shared.config.config.AWS_S3_PROCESSED_DATA_BUCKET", + "processed", + ), + patch( + "data_hub_shared.config.config.LOCAL_RAW_DATA_DIRPATH", + tmp_path, + ), + ): + from data_hub_lambda.dishcam.process_file import process_file + + with pytest.raises(RuntimeError, match="corrupt"): + process_file("dishcam", "run-xyz", "run.json") + + statuses = { + call.args[0]: call.kwargs.get("status") + for call in client.update_file.call_args_list + if call.kwargs.get("status") in {"completed", "failed"} + } + assert statuses[10] == "failed" + assert statuses[20] == "completed" + client.update_run.assert_called_once() diff --git a/lambda/tests/fixtures/dishcam_example.tif b/lambda/tests/fixtures/dishcam_example.tif new file mode 100644 index 0000000000000000000000000000000000000000..b1ed3f39ca4cba1daf72e50a22cefb955719830d GIT binary patch literal 37688 zcmeI%Jxjzu5P;#?M8(37vryr*I7HzP5k(OZ3)}0`i-n+vf`Xuph#=NBT3GuVtSrUK z-(zW`lVcAejA_nWH}GC|*Ni<} zCGihNZ61Wb$r3Dye=utEpsM?u4@D9I1Q0*~0R#|0009ILKmY**5I_I{1Q4i%K$hLn z5I_I{1Q0+VDuMFvr%{z^7DN3eORyyV!KlrH@Hbh4CGihNZ5~v0fAgV8B7gt_2q1s} z0tg_000IagfB*srAb6w68~V-=0W(I zEWwia2ctF*s=B}VP$UsR009ILKmY**5I_I{1Q0*~0R#|00D(#fjEcAcaU!y#-T8P$ ziB#wl4_jK$@`Fx6OAYQ{Mv%Zq|fJTrV9H^`&i~h>Ta;J$BXVc Generator[MagicMock, None, None]: yield mock +@pytest.fixture(autouse=True) +def mock_s3_object_exists( + s3_fixture_files: dict[str, Path], +) -> Generator[MagicMock, None, None]: + """HEAD exists when the key was registered on `s3_fixture_files`.""" + + def _exists(s3_uri: str, **_: Any) -> bool: + key = s3_uri.split("//", 1)[1].split("/", 1)[1] + return key in s3_fixture_files + + with patch("data_hub_shared.s3_utils.object_exists", side_effect=_exists) as mock: + yield mock + + +@pytest.fixture(autouse=True) +def mock_s3_list_objects( + s3_fixture_files: dict[str, Path], +) -> Generator[MagicMock, None, None]: + """List registered fixture keys under the requested prefix.""" + + def _list(s3_uri_prefix: str, suffix: str = "", **_: Any) -> list[str]: + rest = s3_uri_prefix.split("//", 1)[1] + bucket, prefix = rest.split("/", 1) + uris: list[str] = [] + for key in s3_fixture_files: + if key.startswith(prefix) and (not suffix or key.endswith(suffix)): + uris.append(f"s3://{bucket}/{key}") + return uris + + with patch("data_hub_shared.s3_utils.list_objects", side_effect=_list) as mock: + yield mock + + # --------------------------------------------------------------------------- # Step 5 — mock Lambda context # --------------------------------------------------------------------------- diff --git a/lambda/tests/integration/test_lambda_api.py b/lambda/tests/integration/test_lambda_api.py index d089db75..b1b6b5e6 100644 --- a/lambda/tests/integration/test_lambda_api.py +++ b/lambda/tests/integration/test_lambda_api.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +import shutil from collections.abc import Callable from pathlib import Path from typing import Any @@ -240,6 +241,107 @@ def test_tif_completes_with_metadata_and_processed_image( assert upload_dest == f"s3://test-processed-bucket/azure-600-gel-doc/{run_id}/{run_id}.png" +# ------------------------------------------------------------------ +# Test 4b'': DishCam — both files present, encode MP4 + poster +# ------------------------------------------------------------------ + + +class TestDishCamHappyPath: + @pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg is not on PATH") + def test_run_json_encodes_sibling_tiff( + self, + integration_env: IntegrationEnv, + make_s3_event: Callable[..., dict[str, Any]], + s3_fixture_files: dict[str, Path], + mock_context: MagicMock, + mock_s3_upload: MagicMock, + ) -> None: + run_id = "dishcam-test_20260819_155803" + tiff_name = "dishcam-test.tif" + s3_fixture_files[f"dishcam/{run_id}/{tiff_name}"] = _FIXTURES_DIR / "dishcam_example.tif" + s3_fixture_files[f"dishcam/{run_id}/run.json"] = _FIXTURES_DIR / "dishcam_run.json" + + event = make_s3_event("dishcam", run_id, "run.json") + lambda_handler(event, mock_context) + + run = _api_get( + integration_env.base_url, + integration_env.api_token, + f"/api/v1/instruments/dishcam/runs/{run_id}", + ) + + assert run["source"] == "lambda" + assert run["metadata"]["frames"] == 4 + assert run["metadata"]["measured_fps"] == 0.95 + assert run["metadata"]["quality"] == "High" + + raw_tiff = next(f for f in run["files"] if f["filename"] == tiff_name) + assert raw_tiff["status"] == "completed" + + processed = [f for f in run["files"] if f["category"] == "processed"] + names = {f["filename"] for f in processed} + assert names == {"dishcam-test.mp4", "dishcam-test.jpg"} + + uploaded = {call.args[1] for call in mock_s3_upload.call_args_list} + assert f"s3://test-processed-bucket/dishcam/{run_id}/dishcam-test.mp4" in uploaded + assert f"s3://test-processed-bucket/dishcam/{run_id}/dishcam-test.jpg" in uploaded + + @pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg is not on PATH") + def test_run_json_encodes_every_tiff( + self, + integration_env: IntegrationEnv, + make_s3_event: Callable[..., dict[str, Any]], + s3_fixture_files: dict[str, Path], + mock_context: MagicMock, + mock_s3_upload: MagicMock, + ) -> None: + run_id = "dishcam-multi-tiff" + s3_fixture_files[f"dishcam/{run_id}/empty.tif"] = _FIXTURES_DIR / "dishcam_example.tif" + s3_fixture_files[f"dishcam/{run_id}/ruler.tif"] = _FIXTURES_DIR / "dishcam_example.tif" + s3_fixture_files[f"dishcam/{run_id}/run.json"] = _FIXTURES_DIR / "dishcam_run.json" + + event = make_s3_event("dishcam", run_id, "run.json") + lambda_handler(event, mock_context) + + run = _api_get( + integration_env.base_url, + integration_env.api_token, + f"/api/v1/instruments/dishcam/runs/{run_id}", + ) + + raw = {f["filename"]: f for f in run["files"] if f["category"] == "raw"} + assert raw["empty.tif"]["status"] == "completed" + assert raw["ruler.tif"]["status"] == "completed" + + processed = {f["filename"] for f in run["files"] if f["category"] == "processed"} + assert processed == { + "empty.mp4", + "empty.jpg", + "ruler.mp4", + "ruler.jpg", + } + + def test_tiff_alone_does_not_create_a_run( + self, + integration_env: IntegrationEnv, + make_s3_event: Callable[..., dict[str, Any]], + s3_fixture_files: dict[str, Path], + mock_context: MagicMock, + ) -> None: + run_id = "dishcam-tiff-only" + s3_fixture_files[f"dishcam/{run_id}/stack.tif"] = _FIXTURES_DIR / "dishcam_example.tif" + + event = make_s3_event("dishcam", run_id, "stack.tif") + lambda_handler(event, mock_context) + + resp = requests.get( + f"{integration_env.base_url}/api/v1/instruments/dishcam/runs/{run_id}", + headers={"Authorization": f"Bearer {integration_env.api_token}"}, + timeout=10, + ) + assert resp.status_code == 404 + + # ------------------------------------------------------------------ # Test 4c: Malformed file — failure path # ------------------------------------------------------------------ diff --git a/lambda/tests/test_processors.py b/lambda/tests/test_processors.py index 485e492f..3d286f67 100644 --- a/lambda/tests/test_processors.py +++ b/lambda/tests/test_processors.py @@ -44,6 +44,13 @@ class TestFilenameGates: ("epson_v700_scanner", "scan.jpg", False), ("fplc", "chromatogram.pdf", True), ("fplc", "notes.txt", False), + ("dishcam", "stack.tif", True), + ("dishcam", "stack.tiff", True), + ("dishcam", "stack.TIF", True), + ("dishcam", "run.json", True), + ("dishcam", "RUN.JSON", True), + ("dishcam", "notes.json", False), + ("dishcam", "stack.png", False), ], ) def test_per_type_gate(self, instrument_type: str, filename: str, expected: bool) -> None: @@ -54,6 +61,7 @@ def test_union_gate_matches_any_processor(self) -> None: assert matches_any_processor_gate("Experiment_Cq Values.csv") assert matches_any_processor_gate("scan.TIFF") assert matches_any_processor_gate("well.nd2") + assert matches_any_processor_gate("run.json") assert not matches_any_processor_gate("readme.txt") assert not matches_any_processor_gate("notes.md") diff --git a/packages/shared/src/data_hub_shared/s3_utils.py b/packages/shared/src/data_hub_shared/s3_utils.py index abf65cc2..c73fe68c 100644 --- a/packages/shared/src/data_hub_shared/s3_utils.py +++ b/packages/shared/src/data_hub_shared/s3_utils.py @@ -13,6 +13,7 @@ from urllib.parse import urlparse import boto3 +from botocore.exceptions import ClientError logger = logging.getLogger(__name__) @@ -42,9 +43,9 @@ def _extra_args(file_path: Path) -> dict[str, str]: content_type, _ = mimetypes.guess_type(str(file_path)) if content_type: args["ContentType"] = content_type - # Images get "inline" disposition so browsers render them directly + # Images and video get "inline" so browsers render / play them # instead of prompting a download when accessed via pre-signed URL. - if content_type.startswith("image/"): + if content_type.startswith("image/") or content_type.startswith("video/"): args["ContentDisposition"] = "inline" return args @@ -63,6 +64,39 @@ def upload_file( client.upload_file(str(local_path), bucket, key, ExtraArgs=extra) +def object_exists( + s3_uri: str, + *, + s3_client: S3Client | None = None, +) -> bool: + """Return True if *s3_uri* exists (HEAD), False when it does not. + + Missing keys normally 404. Without `s3:ListBucket`, S3 returns 403 + instead — treat that as missing too, matching `headS3Object` in + `web/lib/s3.ts`. + """ + client = s3_client or get_s3_client() + bucket, key = parse_s3_uri(s3_uri) + try: + client.head_object(Bucket=bucket, Key=key) + except ClientError as exc: + error = exc.response.get("Error", {}) + code = str(error.get("Code", "")) + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if code in {"404", "NoSuchKey", "NotFound"} or status == 404: + return False + if code in {"403", "AccessDenied", "Forbidden"} or status == 403: + logger.warning( + "HEAD s3://%s/%s returned 403; treating as missing. " + "If this is steady-state, grant s3:ListBucket on the bucket ARN.", + bucket, + key, + ) + return False + raise + return True + + def download_file( s3_uri: str, local_path: Path, diff --git a/packages/shared/tests/test_s3_utils.py b/packages/shared/tests/test_s3_utils.py new file mode 100644 index 00000000..54149409 --- /dev/null +++ b/packages/shared/tests/test_s3_utils.py @@ -0,0 +1,60 @@ +"""Unit tests for shared S3 helpers.""" + +from __future__ import annotations +from unittest.mock import MagicMock + +import pytest +from botocore.exceptions import ClientError + +from data_hub_shared.s3_utils import object_exists + + +def _client_error(code: str, status: int) -> ClientError: + return ClientError( + { + "Error": {"Code": code, "Message": "nope"}, + "ResponseMetadata": {"HTTPStatusCode": status}, + }, + "HeadObject", + ) + + +def test_object_exists_true() -> None: + client = MagicMock() + assert object_exists("s3://bucket/key", s3_client=client) is True + client.head_object.assert_called_once_with(Bucket="bucket", Key="key") + + +@pytest.mark.parametrize( + ("code", "status"), + [ + ("404", 404), + ("NoSuchKey", 404), + ("NotFound", 404), + ], +) +def test_object_exists_missing(code: str, status: int) -> None: + client = MagicMock() + client.head_object.side_effect = _client_error(code, status) + assert object_exists("s3://bucket/key", s3_client=client) is False + + +@pytest.mark.parametrize( + ("code", "status"), + [ + ("403", 403), + ("AccessDenied", 403), + ("Forbidden", 403), + ], +) +def test_object_exists_403_is_missing(code: str, status: int) -> None: + client = MagicMock() + client.head_object.side_effect = _client_error(code, status) + assert object_exists("s3://bucket/key", s3_client=client) is False + + +def test_object_exists_other_errors_raise() -> None: + client = MagicMock() + client.head_object.side_effect = _client_error("500", 500) + with pytest.raises(ClientError): + object_exists("s3://bucket/key", s3_client=client) diff --git a/web/app/api/local-s3/[bucket]/[...key]/route.ts b/web/app/api/local-s3/[bucket]/[...key]/route.ts index 0c84dc45..22935309 100644 --- a/web/app/api/local-s3/[bucket]/[...key]/route.ts +++ b/web/app/api/local-s3/[bucket]/[...key]/route.ts @@ -1,10 +1,10 @@ // Local-only S3 mirror endpoint. Serves bytes from -// `//` on GET and writes bytes there on -// PUT. The matching dispatch lives in `web/lib/s3-local-mirror.ts`, +// `//` on GET, metadata on HEAD, and writes +// bytes there on PUT. The matching dispatch lives in `web/lib/s3-local-mirror.ts`, // which `web/lib/s3.ts` calls into when the env var is set. // // Gating: `getLocalMirrorRoot()` returns `null` whenever -// `NODE_ENV === "production"` OR `LOCAL_S3_MIRROR` is unset. Both +// `NODE_ENV === "production"` OR `LOCAL_S3_MIRROR` is unset. All // handlers short-circuit to a 404 in that case, so a production // build can never expose the filesystem even if the file is somehow // included in the bundle. @@ -23,6 +23,7 @@ import type { NextRequest } from "next/server"; import { getLocalMirrorRoot, mimeFor, + parseByteRange, resolveMirrorPath, } from "@/lib/s3-local-mirror"; @@ -32,10 +33,13 @@ interface RouteContext { const NOT_FOUND_RESPONSE = () => new Response("Not Found", { status: 404 }); -export async function GET(request: NextRequest, { params }: RouteContext) { +async function locateMirrorFile( + params: RouteContext["params"], + method: string +): Promise<{ filePath: string; fileSize: number } | null> { const root = getLocalMirrorRoot(); if (!root) { - return NOT_FOUND_RESPONSE(); + return null; } const { bucket, key } = await params; @@ -45,36 +49,86 @@ export async function GET(request: NextRequest, { params }: RouteContext) { try { filePath = resolveMirrorPath(root, bucket, joinedKey); } catch (err) { - console.warn(`[local-s3] rejected GET ${bucket}/${joinedKey}: ${err}`); - return NOT_FOUND_RESPONSE(); + console.warn( + `[local-s3] rejected ${method} ${bucket}/${joinedKey}: ${err}` + ); + return null; } - let fileSize: number; try { const s = await stat(filePath); if (!s.isFile()) { - return NOT_FOUND_RESPONSE(); + return null; } - fileSize = s.size; + return { filePath, fileSize: s.size }; } catch { + return null; + } +} + +export async function GET(request: NextRequest, { params }: RouteContext) { + const located = await locateMirrorFile(params, "GET"); + if (!located) { return NOT_FOUND_RESPONSE(); } + const { filePath, fileSize } = located; const disposition = new URL(request.url).searchParams.get("disposition"); + const range = parseByteRange(request.headers.get("range"), fileSize); + const start = range.kind === "partial" ? range.start : 0; + const end = range.kind === "partial" ? range.end : Math.max(fileSize - 1, 0); + const length = range.kind === "unsatisfiable" ? 0 : end - start + 1; + + if (range.kind === "unsatisfiable") { + return new Response("Range Not Satisfiable", { + status: 416, + headers: { + "Content-Range": `bytes */${fileSize}`, + "Accept-Ranges": "bytes", + "Cache-Control": "no-store", + }, + }); + } // `Readable.toWeb` returns the `node:stream/web` `ReadableStream` type, // which is structurally compatible with the global `ReadableStream` // that `Response` expects. Cast through `unknown` to bridge the two // declarations without pulling DOM lib types into the Node side. const body = Readable.toWeb( - createReadStream(filePath) + createReadStream(filePath, range.kind === "partial" ? { start, end } : {}) ) as unknown as ReadableStream; return new Response(body, { + status: range.kind === "partial" ? 206 : 200, + headers: { + "Content-Type": mimeFor(filePath), + "Content-Length": String(range.kind === "partial" ? length : fileSize), + "Accept-Ranges": "bytes", + ...(range.kind === "partial" && { + "Content-Range": `bytes ${start}-${end}/${fileSize}`, + }), + ...(disposition && { "Content-Disposition": disposition }), + "Cache-Control": "no-store", + }, + }); +} + +// Safari (and some Chrome probes) send HEAD before Range GET. Mirror S3: +// same type/length/Accept-Ranges as GET, no body. +export async function HEAD(request: NextRequest, { params }: RouteContext) { + const located = await locateMirrorFile(params, "HEAD"); + if (!located) { + return NOT_FOUND_RESPONSE(); + } + const { filePath, fileSize } = located; + const disposition = new URL(request.url).searchParams.get("disposition"); + + return new Response(null, { status: 200, headers: { "Content-Type": mimeFor(filePath), "Content-Length": String(fileSize), + "Accept-Ranges": "bytes", ...(disposition && { "Content-Disposition": disposition }), "Cache-Control": "no-store", }, diff --git a/web/components/instruments/edit-instrument-dialog.tsx b/web/components/instruments/edit-instrument-dialog.tsx index b6c24c88..1322141a 100644 --- a/web/components/instruments/edit-instrument-dialog.tsx +++ b/web/components/instruments/edit-instrument-dialog.tsx @@ -35,6 +35,7 @@ const TYPE_LABELS: Record = { epson_v700_scanner: "Epson V700 Scanner", instant_raman: "InstantRaman", fplc: "FPLC", + dishcam: "DishCam", }; const INSTRUMENT_TYPE_OPTIONS = VALID_INSTRUMENT_TYPES.map((value) => ({ diff --git a/web/components/notifications/notification-bell-content.tsx b/web/components/notifications/notification-bell-content.tsx index 3273773e..8d6f7149 100644 --- a/web/components/notifications/notification-bell-content.tsx +++ b/web/components/notifications/notification-bell-content.tsx @@ -12,6 +12,7 @@ import { ScanLine, Settings, TestTube, + Video, } from "lucide-react"; import Link from "next/link"; import { useMemo, useState } from "react"; @@ -48,6 +49,7 @@ const INSTRUMENT_TYPE_ICON: Record = { epson_v700_scanner: ScanLine, instant_raman: Radar, fplc: FlaskConical, + dishcam: Video, }; // Bucket labels live alongside the buckets themselves so the section diff --git a/web/components/runs/report-item-seeker.tsx b/web/components/runs/report-item-seeker.tsx index 6eddf188..3e62d55b 100644 --- a/web/components/runs/report-item-seeker.tsx +++ b/web/components/runs/report-item-seeker.tsx @@ -50,6 +50,13 @@ const LABELS: Record = { search: "Search spectra...", select: "Select a spectrum\u2026", }, + video: { + empty: "No videos found.", + next: "Next video", + previous: "Previous video", + search: "Search videos...", + select: "Select a video\u2026", + }, }; function LoadSentinel({ diff --git a/web/components/runs/run-detail.ts b/web/components/runs/run-detail.ts index f98d0a4a..68d766ac 100644 --- a/web/components/runs/run-detail.ts +++ b/web/components/runs/run-detail.ts @@ -6,6 +6,7 @@ import { RunFilesSection } from "@/components/runs/run-files-section"; import { RunHeader } from "@/components/runs/run-header"; import { RunMetadata } from "@/components/runs/run-metadata"; import { RunReportSection } from "@/components/runs/run-report-section"; +import { VideoCarouselReport } from "@/components/runs/video-carousel-report"; import type { RawWellRow, RunDetail as RunDetailType, @@ -24,6 +25,7 @@ export const RunDetail = { ImageCarousel: ImageCarouselReport, PdfCarousel: PdfCarouselReport, RamanReport: RamanReportSection, + VideoCarousel: VideoCarouselReport, }; export interface RunDetailProps { diff --git a/web/components/runs/run-report-section.tsx b/web/components/runs/run-report-section.tsx index 4843c5fc..9112fd62 100644 --- a/web/components/runs/run-report-section.tsx +++ b/web/components/runs/run-report-section.tsx @@ -1,10 +1,18 @@ import { ExternalLink } from "lucide-react"; import { ColonyDataTable } from "@/components/runs/colony-data-table"; import { RunSectionHeading } from "@/components/runs/run-section-heading"; +import { RunVideoPlayer } from "@/components/runs/run-video-player"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import type { RunFile } from "@/lib/api/instrument-runs"; -import { isCsvFile, isImageFile, isPdfFile } from "@/lib/runs/run-file-types"; +import { + fileStem, + isCsvFile, + isImageFile, + isPdfFile, + isVideoFile, + posterFileIdsByVideoFilename, +} from "@/lib/runs/run-file-types"; function ProcessedImagePreview({ file }: { file: RunFile }) { const downloadUrl = `/api/v1/files/${file.id}/download`; @@ -70,14 +78,27 @@ export function RunReportSection({ (f) => f.category === "processed" && f.deletedAt === null && isCsvFile(f) ); + const processedVideos = files.filter( + (f) => f.category === "processed" && f.deletedAt === null && isVideoFile(f) + ); + const posterFileIds = posterFileIdsByVideoFilename(files); + const videoStems = new Set(processedVideos.map((f) => fileStem(f.filename))); + const processedImages = files.filter( - (f) => f.category === "processed" && f.deletedAt === null && isImageFile(f) + (f) => + f.category === "processed" && + f.deletedAt === null && + isImageFile(f) && + !videoStems.has(fileStem(f.filename)) ); const pdfFiles = files.filter((f) => f.deletedAt === null && isPdfFile(f)); const totalCount = - processedCsvs.length + processedImages.length + pdfFiles.length; + processedCsvs.length + + processedImages.length + + processedVideos.length + + pdfFiles.length; if (totalCount === 0) { return ( @@ -99,6 +120,14 @@ export function RunReportSection({ + {processedVideos.map((file) => ( + + ))} {processedImages.map((file) => ( ))} diff --git a/web/components/runs/run-video-player.tsx b/web/components/runs/run-video-player.tsx new file mode 100644 index 00000000..28dc6175 --- /dev/null +++ b/web/components/runs/run-video-player.tsx @@ -0,0 +1,47 @@ +import { ExternalLink } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +function fileDownloadUrl(fileId: number): string { + return `/api/v1/files/${fileId}/download`; +} + +export function RunVideoPlayer({ + fileId, + filename, + posterFileId, +}: { + fileId: number; + filename: string; + posterFileId?: number; +}) { + const downloadUrl = fileDownloadUrl(fileId); + const posterUrl = + posterFileId === undefined ? undefined : fileDownloadUrl(posterFileId); + + return ( +
+
+

{filename}

+ +
+ {/* `preload="none"` keeps the MP4 off the wire until play; the JPEG is the still. */} +
+ {/* biome-ignore lint/a11y/useMediaCaption: instrument preview has no captions */} +
+
+ ); +} diff --git a/web/components/runs/variants/dishcam-run-detail.tsx b/web/components/runs/variants/dishcam-run-detail.tsx new file mode 100644 index 00000000..571f3c3d --- /dev/null +++ b/web/components/runs/variants/dishcam-run-detail.tsx @@ -0,0 +1,66 @@ +import { DeleteRunDialog } from "@/components/runs/delete-run-dialog"; +import { RestoreRunButton } from "@/components/runs/restore-run-button"; +import type { RunDetailProps } from "@/components/runs/run-detail"; +import { RunDetail } from "@/components/runs/run-detail"; +import { posterFileIdsByVideoFilename } from "@/lib/runs/run-file-types"; + +export function DishcamRunDetail({ + run, + files, + filesDownloadableCount, + filesPagination, + fileStats, + reportFiles, + reportItems, + instrumentId, + runId, + attributionsSlot, + runNavSlot, +}: RunDetailProps) { + const isDeleted = run.deletedAt !== null; + const activeFileCount = fileStats.active; + const hasProcessedFiles = fileStats.processedActive > 0; + + return ( + <> + + {!isDeleted && ( + + )} + {isDeleted && ( + + )} + + + + + + + + + + ); +} diff --git a/web/components/runs/variants/index.tsx b/web/components/runs/variants/index.tsx index ac18587f..2f77f5e1 100644 --- a/web/components/runs/variants/index.tsx +++ b/web/components/runs/variants/index.tsx @@ -1,6 +1,7 @@ import type { RunDetailProps } from "@/components/runs/run-detail"; import { DefaultRunDetail } from "./default-run-detail"; +import { DishcamRunDetail } from "./dishcam-run-detail"; import { EpsonScannerRunDetail } from "./epson-scanner-run-detail"; import { GelDocRunDetail } from "./gel-doc-run-detail"; import { HinaMicroscopeRunDetail } from "./hina-microscope-run-detail"; @@ -33,6 +34,8 @@ function renderRunDetailVariant(props: RunDetailVariantProps) { return ; case "instant_raman": return ; + case "dishcam": + return ; default: return ; } diff --git a/web/components/runs/video-carousel-report.tsx b/web/components/runs/video-carousel-report.tsx new file mode 100644 index 00000000..1280a4e3 --- /dev/null +++ b/web/components/runs/video-carousel-report.tsx @@ -0,0 +1,59 @@ +"use client"; + +import { ReportDataShell } from "@/components/runs/report-data-shell"; +import { ReportItemSeeker } from "@/components/runs/report-item-seeker"; +import { + ReportItemsProvider, + type ReportViewerProps, + useReportItemsContext, +} from "@/components/runs/report-items-provider"; +import { RunVideoPlayer } from "@/components/runs/run-video-player"; + +function SelectedVideo({ + posterFileIds, +}: { + posterFileIds: Record; +}) { + const { state } = useReportItemsContext(); + + if (!state.selectedItem) { + return ( +
+ {state.error ?? + (state.isLoading ? "Loading\u2026" : "No videos found.")} +
+ ); + } + + return ( + + ); +} + +// DishCam and other video-primary instruments: one MP4 at a time, seeked +// against the run's full processed-video set. `posterFileIds` is a slim +// filename → file-id map so full `RunFile` rows never reach the client. +export function VideoCarouselReport({ + initialPage, + instrumentId, + posterFileIds, + runId, +}: ReportViewerProps & { posterFileIds: Record }) { + return ( + + + + + + + ); +} diff --git a/web/drizzle/0040_dishcam_type.sql b/web/drizzle/0040_dishcam_type.sql new file mode 100644 index 00000000..91f8fe7c --- /dev/null +++ b/web/drizzle/0040_dishcam_type.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."instrument_type" ADD VALUE 'dishcam'; diff --git a/web/drizzle/meta/0040_snapshot.json b/web/drizzle/meta/0040_snapshot.json new file mode 100644 index 00000000..64679a47 --- /dev/null +++ b/web/drizzle/meta/0040_snapshot.json @@ -0,0 +1,3102 @@ +{ + "id": "feee444f-f627-4b82-97c0-7b3cc2e1d370", + "prevId": "c42bd017-b3c1-4322-a423-d55689c6f200", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accountId": { + "name": "accountId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "providerId": { + "name": "providerId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accessToken": { + "name": "accessToken", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refreshToken": { + "name": "refreshToken", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idToken": { + "name": "idToken", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "accessTokenExpiresAt": { + "name": "accessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refreshTokenExpiresAt": { + "name": "refreshTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.archive_jobs": { + "name": "archive_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_bucket": { + "name": "archive_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "archive_job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_archive_jobs_inflight": { + "name": "uq_archive_jobs_inflight", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"archive_jobs\".\"status\" in ('pending', 'building')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_archive_jobs_run_fingerprint_status": { + "name": "idx_archive_jobs_run_fingerprint_status", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archive_jobs_instrument_run_id_instrument_runs_id_fk": { + "name": "archive_jobs_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "archive_jobs_created_by_user_id_fk": { + "name": "archive_jobs_created_by_user_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_key": { + "name": "s3_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "file_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'raw'" + }, + "status": { + "name": "status", + "type": "file_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_at": { + "name": "detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "upload_requested_at": { + "name": "upload_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "file_created_at": { + "name": "file_created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_files_instrument_run_id_relative_path": { + "name": "uq_files_instrument_run_id_relative_path", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "relative_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"relative_path\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_active_instrument_run_id_filename": { + "name": "uq_files_active_instrument_run_id_filename", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_s3_key": { + "name": "uq_files_s3_key", + "columns": [ + { + "expression": "s3_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"s3_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_instrument_run_id": { + "name": "idx_files_instrument_run_id", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_status_instrument_run_id": { + "name": "idx_files_status_instrument_run_id", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_upload_queue": { + "name": "idx_files_upload_queue", + "columns": [ + { + "expression": "upload_requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"upload_requested_at\" is not null and \"files\".\"uploaded_at\" is null and \"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_metadata_gin": { + "name": "idx_files_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_files_filename_trgm": { + "name": "idx_files_filename_trgm", + "columns": [ + { + "expression": "\"filename\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "files_instrument_run_id_instrument_runs_id_fk": { + "name": "files_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "files", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_notification_subscriptions": { + "name": "instrument_notification_subscriptions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_instrument_notification_subscriptions_user_id": { + "name": "idx_instrument_notification_subscriptions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "instrument_notification_subscriptions_user_id_user_id_fk": { + "name": "instrument_notification_subscriptions_user_id_user_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "instrument_notification_subscriptions_instrument_id_instruments_id_fk": { + "name": "instrument_notification_subscriptions_instrument_id_instruments_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "instrument_notification_subscriptions_user_id_instrument_id_pk": { + "name": "instrument_notification_subscriptions_user_id_instrument_id_pk", + "columns": ["user_id", "instrument_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_runs": { + "name": "instrument_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "instrument_run_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'lambda'" + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_instrument_runs_instrument_id_created_at": { + "name": "idx_instrument_runs_instrument_id_created_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active": { + "name": "idx_instrument_runs_active", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active_acquired_at": { + "name": "idx_instrument_runs_active_acquired_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"acquired_at\", \"created_at\") desc", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_metadata_gin": { + "name": "idx_instrument_runs_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_instrument_runs_run_id_trgm": { + "name": "idx_instrument_runs_run_id_trgm", + "columns": [ + { + "expression": "\"run_id\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "instrument_runs_instrument_id_instruments_id_fk": { + "name": "instrument_runs_instrument_id_instruments_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_watcher_id_watchers_id_fk": { + "name": "instrument_runs_watcher_id_watchers_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_deleted_by_user_id_fk": { + "name": "instrument_runs_deleted_by_user_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "user", + "columnsFrom": ["deleted_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_instrument_runs_instrument_id_run_id": { + "name": "uq_instrument_runs_instrument_id_run_id", + "nullsNotDistinct": false, + "columns": ["instrument_id", "run_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instruments": { + "name": "instruments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "instrument_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "instrument_type": { + "name": "instrument_type", + "type": "instrument_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'generic'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "retired_by": { + "name": "retired_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_instruments_display_name_trgm": { + "name": "idx_instruments_display_name_trgm", + "columns": [ + { + "expression": "\"display_name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "instruments_retired_by_user_id_fk": { + "name": "instruments_retired_by_user_id_fk", + "tableFrom": "instruments", + "tableTo": "user", + "columnsFrom": ["retired_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "publicKey": { + "name": "publicKey", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "privateKey": { + "name": "privateKey", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_preferences": { + "name": "notification_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "runs_all_muted": { + "name": "runs_all_muted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "comments_attributed_enabled": { + "name": "comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "comments_participated_enabled": { + "name": "comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "slack_runs_enabled": { + "name": "slack_runs_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_attributed_enabled": { + "name": "slack_comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_participated_enabled": { + "name": "slack_comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "notification_preferences_user_id_user_id_fk": { + "name": "notification_preferences_user_id_user_id_fk", + "tableFrom": "notification_preferences", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notifications": { + "name": "notifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "comment_id": { + "name": "comment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "read_at": { + "name": "read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_notifications_user_id_created_at": { + "name": "idx_notifications_user_id_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notifications_user_id_unread": { + "name": "idx_notifications_user_id_unread", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"notifications\".\"read_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notifications_user_id_user_id_fk": { + "name": "notifications_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_run_id_instrument_runs_id_fk": { + "name": "notifications_run_id_instrument_runs_id_fk", + "tableFrom": "notifications", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_comment_id_run_comments_id_fk": { + "name": "notifications_comment_id_run_comments_id_fk", + "tableFrom": "notifications", + "tableTo": "run_comments", + "columnsFrom": ["comment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_actor_user_id_user_id_fk": { + "name": "notifications_actor_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["actor_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "clientId": { + "name": "clientId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sessionId": { + "name": "sessionId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refreshId": { + "name": "refreshId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "clientId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "sessionId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refreshId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_clientId_oauth_client_clientId_fk": { + "name": "oauth_access_token_clientId_oauth_client_clientId_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": ["clientId"], + "columnsTo": ["clientId"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_sessionId_session_id_fk": { + "name": "oauth_access_token_sessionId_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": ["sessionId"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_userId_user_id_fk": { + "name": "oauth_access_token_userId_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refreshId_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refreshId_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": ["refreshId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "clientId": { + "name": "clientId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "clientSecret": { + "name": "clientSecret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skipConsent": { + "name": "skipConsent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enableEndSession": { + "name": "enableEndSession", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subjectType": { + "name": "subjectType", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "softwareId": { + "name": "softwareId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "softwareVersion": { + "name": "softwareVersion", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "softwareStatement": { + "name": "softwareStatement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirectUris": { + "name": "redirectUris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "postLogoutRedirectUris": { + "name": "postLogoutRedirectUris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tokenEndpointAuthMethod": { + "name": "tokenEndpointAuthMethod", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grantTypes": { + "name": "grantTypes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "responseTypes": { + "name": "responseTypes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requirePKCE": { + "name": "requirePKCE", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_userId_user_id_fk": { + "name": "oauth_client_userId_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_clientId_unique": { + "name": "oauth_client_clientId_unique", + "nullsNotDistinct": false, + "columns": ["clientId"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "clientId": { + "name": "clientId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "clientId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_clientId_oauth_client_clientId_fk": { + "name": "oauth_consent_clientId_oauth_client_clientId_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": ["clientId"], + "columnsTo": ["clientId"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_userId_user_id_fk": { + "name": "oauth_consent_userId_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "clientId": { + "name": "clientId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sessionId": { + "name": "sessionId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "authTime": { + "name": "authTime", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "clientId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "sessionId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_clientId_oauth_client_clientId_fk": { + "name": "oauth_refresh_token_clientId_oauth_client_clientId_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": ["clientId"], + "columnsTo": ["clientId"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_sessionId_session_id_fk": { + "name": "oauth_refresh_token_sessionId_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": ["sessionId"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_userId_user_id_fk": { + "name": "oauth_refresh_token_userId_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_access_tokens": { + "name": "personal_access_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "ARRAY['*']::text[]" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_personal_access_tokens_user_id": { + "name": "idx_personal_access_tokens_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_access_tokens_user_id_user_id_fk": { + "name": "personal_access_tokens_user_id_user_id_fk", + "tableFrom": "personal_access_tokens", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personal_access_tokens_token_hash_unique": { + "name": "personal_access_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": ["token_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_attributions": { + "name": "run_attributions", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_run_attributions_run_id": { + "name": "idx_run_attributions_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_attributions_user_id": { + "name": "idx_run_attributions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_attributions_run_id_instrument_runs_id_fk": { + "name": "run_attributions_run_id_instrument_runs_id_fk", + "tableFrom": "run_attributions", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_attributions_user_id_user_id_fk": { + "name": "run_attributions_user_id_user_id_fk", + "tableFrom": "run_attributions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "run_attributions_run_id_user_id_pk": { + "name": "run_attributions_run_id_user_id_pk", + "columns": ["run_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_comments": { + "name": "run_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_run_comments_run_id_created_at": { + "name": "idx_run_comments_run_id_created_at", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_comments_user_id": { + "name": "idx_run_comments_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_comments_body_trgm": { + "name": "idx_run_comments_body_trgm", + "columns": [ + { + "expression": "\"body\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"run_comments\".\"deleted_at\" is null", + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "run_comments_run_id_instrument_runs_id_fk": { + "name": "run_comments_run_id_instrument_runs_id_fk", + "tableFrom": "run_comments", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_comments_user_id_user_id_fk": { + "name": "run_comments_user_id_user_id_fk", + "tableFrom": "run_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ipAddress": { + "name": "ipAddress", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "userAgent": { + "name": "userAgent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_channel_config": { + "name": "slack_channel_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_channel_config_updated_by_user_id_fk": { + "name": "slack_channel_config_updated_by_user_id_fk", + "tableFrom": "slack_channel_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "slack_channel_config_singleton": { + "name": "slack_channel_config_singleton", + "value": "\"slack_channel_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.slack_connections": { + "name": "slack_connections", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slack_user_id": { + "name": "slack_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_id": { + "name": "slack_team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_name": { + "name": "slack_team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_connections_user_id_user_id_fk": { + "name": "slack_connections_user_id_user_id_fk", + "tableFrom": "slack_connections", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "emailVerified": { + "name": "emailVerified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_verification_identifier": { + "name": "idx_verification_identifier", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_events": { + "name": "watcher_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "watcher_event_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_events_watcher_id_timestamp": { + "name": "idx_watcher_events_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_watcher_events_watcher_id_event_type": { + "name": "idx_watcher_events_watcher_id_event_type", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_events_watcher_id_watchers_id_fk": { + "name": "watcher_events_watcher_id_watchers_id_fk", + "tableFrom": "watcher_events", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_heartbeats": { + "name": "watcher_heartbeats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upload_mode": { + "name": "upload_mode", + "type": "upload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "files_uploaded_since_last": { + "name": "files_uploaded_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "runs_reported_since_last": { + "name": "runs_reported_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "errors_since_last": { + "name": "errors_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "uptime_seconds": { + "name": "uptime_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_heartbeats_watcher_id_timestamp": { + "name": "idx_watcher_heartbeats_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_heartbeats_watcher_id_watchers_id_fk": { + "name": "watcher_heartbeats_watcher_id_watchers_id_fk", + "tableFrom": "watcher_heartbeats", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_release_config": { + "name": "watcher_release_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "latest_version": { + "name": "latest_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "min_supported_version": { + "name": "min_supported_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mandatory": { + "name": "mandatory", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "watcher_release_config_updated_by_user_id_fk": { + "name": "watcher_release_config_updated_by_user_id_fk", + "tableFrom": "watcher_release_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "watcher_release_config_singleton": { + "name": "watcher_release_config_singleton", + "value": "\"watcher_release_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.watchers": { + "name": "watchers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "os_info": { + "name": "os_info", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "watcher_version": { + "name": "watcher_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_checksum": { + "name": "config_checksum", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "watcher_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'registered'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deregistered_by": { + "name": "deregistered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registered_by_token": { + "name": "registered_by_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_watchers_active_instrument_id": { + "name": "uq_watchers_active_instrument_id", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"watchers\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watchers_instrument_id_instruments_id_fk": { + "name": "watchers_instrument_id_instruments_id_fk", + "tableFrom": "watchers", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "watchers_deregistered_by_user_id_fk": { + "name": "watchers_deregistered_by_user_id_fk", + "tableFrom": "watchers", + "tableTo": "user", + "columnsFrom": ["deregistered_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "watchers_registered_by_token_personal_access_tokens_id_fk": { + "name": "watchers_registered_by_token_personal_access_tokens_id_fk", + "tableFrom": "watchers", + "tableTo": "personal_access_tokens", + "columnsFrom": ["registered_by_token"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.archive_job_status": { + "name": "archive_job_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.file_category": { + "name": "file_category", + "schema": "public", + "values": ["raw", "processed"] + }, + "public.file_status": { + "name": "file_status", + "schema": "public", + "values": [ + "detected", + "upload_requested", + "uploaded", + "processing", + "completed", + "failed" + ] + }, + "public.instrument_run_source": { + "name": "instrument_run_source", + "schema": "public", + "values": ["lambda", "watcher"] + }, + "public.instrument_status": { + "name": "instrument_status", + "schema": "public", + "values": ["pending", "active", "inactive"] + }, + "public.instrument_type": { + "name": "instrument_type", + "schema": "public", + "values": [ + "generic", + "plate_reader", + "gel_doc", + "qpcr", + "tape_station", + "hina_microscope", + "epson_v700_scanner", + "instant_raman", + "fplc", + "dishcam" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": ["run_created", "comment_attributed", "comment_participated"] + }, + "public.upload_mode": { + "name": "upload_mode", + "schema": "public", + "values": ["auto", "manual"] + }, + "public.watcher_event_type": { + "name": "watcher_event_type", + "schema": "public", + "values": [ + "watcher_started", + "watcher_stopped", + "file_uploaded", + "upload_failed", + "run_reported", + "config_synced", + "error", + "update_started", + "update_succeeded", + "update_failed" + ] + }, + "public.watcher_status": { + "name": "watcher_status", + "schema": "public", + "values": ["registered", "watching", "stopped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index 4a755c96..cc98a1db 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -281,6 +281,13 @@ "when": 1786128237567, "tag": "0039_add_natural_filename_collation", "breakpoints": true + }, + { + "idx": 40, + "version": "7", + "when": 1787187000000, + "tag": "0040_dishcam_type", + "breakpoints": true } ] } diff --git a/web/lib/api/instrument-runs.ts b/web/lib/api/instrument-runs.ts index 89c624bf..7c9e4d50 100644 --- a/web/lib/api/instrument-runs.ts +++ b/web/lib/api/instrument-runs.ts @@ -1413,6 +1413,7 @@ export async function getInstrumentFilterOptions( case "generic": case "tape_station": case "instant_raman": + case "dishcam": return { kind: "default" }; default: return { kind: "default" }; diff --git a/web/lib/api/openapi/paths/runs.ts b/web/lib/api/openapi/paths/runs.ts index 0a5fdcf9..cbeaef0e 100644 --- a/web/lib/api/openapi/paths/runs.ts +++ b/web/lib/api/openapi/paths/runs.ts @@ -152,7 +152,7 @@ registry.registerPath({ path: "/instruments/{instrumentId}/runs/{runId}/report-items", operationId: "listRunReportItems", summary: "List a run's report items", - description: `${scoped("files:read")} Returns \`{id, filename}\` for the run's renderable images, PDFs, or spectra, ordered by filename. Paged with \`offset\`/\`limit\` so viewers can seek by item index.`, + description: `${scoped("files:read")} Returns \`{id, filename}\` for the run's renderable images, PDFs, spectra, or videos, ordered by filename. Paged with \`offset\`/\`limit\` so viewers can seek by item index.`, tags: tag, security: bearerSecurity, request: { params: runParams, query: reportItemsQuery }, diff --git a/web/lib/api/report-items.ts b/web/lib/api/report-items.ts index bee67173..aec6d623 100644 --- a/web/lib/api/report-items.ts +++ b/web/lib/api/report-items.ts @@ -37,6 +37,10 @@ const KIND_PREDICATES: Record = { ${files.contentType} = 'text/csv' or lower(${files.filename}) like '%.csv' )`, + video: sql`( + ${files.contentType} like 'video/%' + or lower(${files.filename}) like '%.mp4' + )`, }; function reportItemsWhere( diff --git a/web/lib/db/schema.ts b/web/lib/db/schema.ts index 8e9fc95b..1b742071 100644 --- a/web/lib/db/schema.ts +++ b/web/lib/db/schema.ts @@ -61,6 +61,7 @@ export const instrumentTypeEnum = pgEnum("instrument_type", [ "epson_v700_scanner", "instant_raman", "fplc", + "dishcam", ]); export const VALID_INSTRUMENT_TYPES = instrumentTypeEnum.enumValues; diff --git a/web/lib/instruments/processable-types.ts b/web/lib/instruments/processable-types.ts index 20e542ea..2bab49c1 100644 --- a/web/lib/instruments/processable-types.ts +++ b/web/lib/instruments/processable-types.ts @@ -11,6 +11,7 @@ export const PROCESSABLE_INSTRUMENT_TYPES = [ "hina_microscope", "epson_v700_scanner", "fplc", + "dishcam", ] as const satisfies readonly InstrumentType[]; const PROCESSABLE_SET = new Set(PROCESSABLE_INSTRUMENT_TYPES); diff --git a/web/lib/runs/report-items.ts b/web/lib/runs/report-items.ts index 8655f920..c3d73208 100644 --- a/web/lib/runs/report-items.ts +++ b/web/lib/runs/report-items.ts @@ -1,6 +1,6 @@ import type { InstrumentType } from "@/lib/db/schema"; -export const REPORT_ITEM_KINDS = ["image", "pdf", "spectrum"] as const; +export const REPORT_ITEM_KINDS = ["image", "pdf", "spectrum", "video"] as const; export type ReportItemKind = (typeof REPORT_ITEM_KINDS)[number]; @@ -33,6 +33,7 @@ const KIND_BY_INSTRUMENT: Partial> = { hina_microscope: "image", tape_station: "pdf", instant_raman: "spectrum", + dishcam: "video", }; export function reportItemKindForInstrument( diff --git a/web/lib/runs/run-file-types.ts b/web/lib/runs/run-file-types.ts index b950d970..61f6e899 100644 --- a/web/lib/runs/run-file-types.ts +++ b/web/lib/runs/run-file-types.ts @@ -5,6 +5,12 @@ type RunFileIdentity = Pick; const IMAGE_EXTENSIONS = /\.(png|jpe?g|gif|webp|svg|tiff?)$/i; const PDF_EXTENSION = /\.pdf$/i; const CSV_EXTENSION = /\.csv$/i; +const VIDEO_EXTENSION = /\.mp4$/i; + +export function fileStem(filename: string): string { + const dot = filename.lastIndexOf("."); + return dot > 0 ? filename.slice(0, dot) : filename; +} export function isImageFile(file: RunFileIdentity): boolean { return ( @@ -22,3 +28,44 @@ export function isPdfFile(file: RunFileIdentity): boolean { export function isCsvFile(file: RunFileIdentity): boolean { return file.contentType === "text/csv" || CSV_EXTENSION.test(file.filename); } + +export function isVideoFile(file: RunFileIdentity): boolean { + return ( + file.contentType?.startsWith("video/") === true || + VIDEO_EXTENSION.test(file.filename) + ); +} + +type PosterSourceFile = Pick< + RunFile, + "category" | "contentType" | "deletedAt" | "filename" | "id" +>; + +function isActiveProcessed(file: PosterSourceFile): boolean { + return file.category === "processed" && file.deletedAt === null; +} + +// DishCam writes `{stem}.mp4` and `{stem}.jpg`. Match on the last +// extension only so `foo.bar.mp4` still pairs with `foo.bar.jpg`. +export function posterFileIdsByVideoFilename( + files: readonly PosterSourceFile[] +): Record { + const postersByStem = new Map(); + for (const file of files) { + if (isActiveProcessed(file) && isImageFile(file)) { + postersByStem.set(fileStem(file.filename), file.id); + } + } + + const posterFileIds: Record = {}; + for (const file of files) { + if (!(isActiveProcessed(file) && isVideoFile(file))) { + continue; + } + const posterId = postersByStem.get(fileStem(file.filename)); + if (posterId !== undefined) { + posterFileIds[file.filename] = posterId; + } + } + return posterFileIds; +} diff --git a/web/lib/s3-local-mirror.ts b/web/lib/s3-local-mirror.ts index 695df79e..c90a3679 100644 --- a/web/lib/s3-local-mirror.ts +++ b/web/lib/s3-local-mirror.ts @@ -34,6 +34,7 @@ const MIME_MAP: Record = { ".pdf": "application/pdf", ".zip": "application/zip", ".nd2": "application/octet-stream", + ".mp4": "video/mp4", }; export function getLocalMirrorRoot(): string | null { @@ -67,6 +68,57 @@ export function mimeFor(filePath: string): string { return MIME_MAP[ext] ?? "application/octet-stream"; } +export type ByteRange = + | { kind: "full" } + | { kind: "partial"; start: number; end: number } + | { kind: "unsatisfiable" }; + +// Safari (and Chrome when scrubbing) send `Range` against the local +// mirror the same way they do against S3. A 200 of the whole file +// without `Accept-Ranges` leaves seeking broken in local dev. +export function parseByteRange( + header: string | null, + fileSize: number +): ByteRange { + if (!header) { + return { kind: "full" }; + } + const match = /^bytes=(\d*)-(\d*)$/i.exec(header.trim()); + if (!match) { + return { kind: "full" }; + } + const startToken = match[1]; + const endToken = match[2]; + if (startToken === "" && endToken === "") { + return { kind: "full" }; + } + if (fileSize === 0) { + return { kind: "unsatisfiable" }; + } + if (startToken === "") { + const suffix = Number(endToken); + if (!Number.isFinite(suffix) || suffix <= 0) { + return { kind: "unsatisfiable" }; + } + return { + kind: "partial", + start: Math.max(0, fileSize - suffix), + end: fileSize - 1, + }; + } + const start = Number(startToken); + const end = + endToken === "" ? fileSize - 1 : Math.min(Number(endToken), fileSize - 1); + if ( + !(Number.isFinite(start) && Number.isFinite(end)) || + start >= fileSize || + start > end + ) { + return { kind: "unsatisfiable" }; + } + return { kind: "partial", start, end }; +} + // Sanitize a filename for use inside a `Content-Disposition` header. // Mirrors the same logic in `web/lib/s3.ts` so the local route's // response headers match what the AWS path would have produced via diff --git a/web/tests/integration/report-items.test.ts b/web/tests/integration/report-items.test.ts index 65f63d98..29e509b9 100644 --- a/web/tests/integration/report-items.test.ts +++ b/web/tests/integration/report-items.test.ts @@ -67,9 +67,20 @@ describe("Report Items API", () => { s3Key: `${instrumentId}/${runId}/Site_${n}.csv`, })); + const videoRows = ["empty", "gk134_high", "ruler"].map((stem) => ({ + instrumentRunId: runInternalId, + relativePath: `${stem}.mp4`, + filename: `${stem}.mp4`, + contentType: "video/mp4", + status: "completed" as const, + s3Bucket: "test-bucket", + s3Key: `${instrumentId}/${runId}/${stem}.mp4`, + })); + await db.insert(files).values([ ...imageRows, ...spectrumRows, + ...videoRows, { instrumentRunId: runInternalId, relativePath: "peaks.csv", @@ -174,6 +185,17 @@ describe("Report Items API", () => { ).toBe(true); }); + it("lists videos without mixing in other kinds", async () => { + const res = await api(path("kind=video"), { token }); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.pagination.total).toBe(3); + expect( + body.data.map((item: { filename: string }) => item.filename) + ).toEqual(["empty.mp4", "gk134_high.mp4", "ruler.mp4"]); + }); + it("orders unpadded numeric filenames naturally", async () => { const res = await api(path("kind=spectrum&search=Site_"), { token }); expect(res.status).toBe(200); diff --git a/web/tests/unit/report-items.test.ts b/web/tests/unit/report-items.test.ts new file mode 100644 index 00000000..43eabff2 --- /dev/null +++ b/web/tests/unit/report-items.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { + isReportItemKind, + REPORT_ITEM_KINDS, + reportItemKindForInstrument, +} from "@/lib/runs/report-items"; + +describe("reportItemKindForInstrument", () => { + it("maps dishcam runs to the video seeker", () => { + expect(reportItemKindForInstrument("dishcam")).toBe("video"); + }); + + it("leaves unmapped types without a seeker", () => { + expect(reportItemKindForInstrument("generic")).toBeNull(); + expect(reportItemKindForInstrument("plate_reader")).toBeNull(); + }); +}); + +describe("isReportItemKind", () => { + it("accepts every documented kind", () => { + expect(REPORT_ITEM_KINDS).toContain("video"); + for (const kind of REPORT_ITEM_KINDS) { + expect(isReportItemKind(kind)).toBe(true); + } + }); +}); diff --git a/web/tests/unit/run-file-types.test.ts b/web/tests/unit/run-file-types.test.ts new file mode 100644 index 00000000..e4b61c30 --- /dev/null +++ b/web/tests/unit/run-file-types.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { + fileStem, + isCsvFile, + isImageFile, + isPdfFile, + isVideoFile, + posterFileIdsByVideoFilename, +} from "@/lib/runs/run-file-types"; + +describe("isVideoFile", () => { + it("matches video content types", () => { + expect( + isVideoFile({ filename: "clip.bin", contentType: "video/mp4" }) + ).toBe(true); + }); + + it("matches .mp4 filenames", () => { + expect(isVideoFile({ filename: "stack.MP4", contentType: null })).toBe( + true + ); + }); + + it("does not match images or csv", () => { + expect( + isVideoFile({ filename: "stack.jpg", contentType: "image/jpeg" }) + ).toBe(false); + expect(isCsvFile({ filename: "data.csv", contentType: "text/csv" })).toBe( + true + ); + expect( + isImageFile({ filename: "stack.jpg", contentType: "image/jpeg" }) + ).toBe(true); + expect( + isPdfFile({ filename: "report.pdf", contentType: "application/pdf" }) + ).toBe(true); + }); +}); + +describe("fileStem", () => { + it("strips the last extension", () => { + expect(fileStem("stack.mp4")).toBe("stack"); + expect(fileStem("stack")).toBe("stack"); + expect(fileStem("foo.bar.mp4")).toBe("foo.bar"); + expect(fileStem("foo.bar.jpg")).toBe("foo.bar"); + }); +}); + +describe("posterFileIdsByVideoFilename", () => { + const processed = { + category: "processed" as const, + deletedAt: null, + }; + + it("maps a video to a same-stem processed jpeg", () => { + expect( + posterFileIdsByVideoFilename([ + { + ...processed, + contentType: "video/mp4", + filename: "stack.mp4", + id: 1, + }, + { + ...processed, + contentType: "image/jpeg", + filename: "stack.jpg", + id: 2, + }, + ]) + ).toEqual({ "stack.mp4": 2 }); + }); + + it("pairs dotted stems by last extension only", () => { + expect( + posterFileIdsByVideoFilename([ + { + ...processed, + contentType: "video/mp4", + filename: "foo.bar.mp4", + id: 1, + }, + { + ...processed, + contentType: "image/jpeg", + filename: "foo.bar.jpg", + id: 2, + }, + ]) + ).toEqual({ "foo.bar.mp4": 2 }); + }); + + it("omits videos with no poster and ignores raw or deleted files", () => { + expect( + posterFileIdsByVideoFilename([ + { + ...processed, + contentType: "video/mp4", + filename: "lonely.mp4", + id: 1, + }, + { + category: "raw", + contentType: "image/jpeg", + deletedAt: null, + filename: "lonely.jpg", + id: 2, + }, + { + category: "processed", + contentType: "image/jpeg", + deletedAt: new Date("2026-01-01"), + filename: "gone.jpg", + id: 3, + }, + { + ...processed, + contentType: "video/mp4", + filename: "gone.mp4", + id: 4, + }, + ]) + ).toEqual({}); + }); +}); diff --git a/web/tests/unit/s3-local-mirror.test.ts b/web/tests/unit/s3-local-mirror.test.ts new file mode 100644 index 00000000..246777f1 --- /dev/null +++ b/web/tests/unit/s3-local-mirror.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { mimeFor, parseByteRange } from "@/lib/s3-local-mirror"; + +describe("parseByteRange", () => { + it("serves the full file when Range is missing or malformed", () => { + expect(parseByteRange(null, 100)).toEqual({ kind: "full" }); + expect(parseByteRange("bytes=", 100)).toEqual({ kind: "full" }); + expect(parseByteRange("wibble", 100)).toEqual({ kind: "full" }); + }); + + it("parses open-ended and closed ranges", () => { + expect(parseByteRange("bytes=0-1", 100)).toEqual({ + kind: "partial", + start: 0, + end: 1, + }); + expect(parseByteRange("bytes=50-", 100)).toEqual({ + kind: "partial", + start: 50, + end: 99, + }); + expect(parseByteRange("bytes=-10", 100)).toEqual({ + kind: "partial", + start: 90, + end: 99, + }); + }); + + it("rejects ranges past the end of the file", () => { + expect(parseByteRange("bytes=100-101", 100)).toEqual({ + kind: "unsatisfiable", + }); + expect(parseByteRange("bytes=0-1", 0)).toEqual({ + kind: "unsatisfiable", + }); + }); +}); + +describe("mimeFor", () => { + it("maps mp4 to video/mp4 so the local player can seek", () => { + expect(mimeFor("/tmp/stack.mp4")).toBe("video/mp4"); + }); +});