From 0b7fabe1076be29f08266ec2f00084184ffbafd9 Mon Sep 17 00:00:00 2001 From: Wasim Sandhu Date: Thu, 21 May 2026 12:39:35 -0700 Subject: [PATCH 1/6] Add `data-hub-process handler` CLI for end-to-end local Lambda testing Adds a new `handler` subcommand to the lambda CLI that drives `lambda_handler` against a gitignored local directory mirroring the S3 layout, so devs can iterate on `process_file` modules against the zero-credential local web app without S3, AWS credentials, or LocalStack. The CLI stages a `--source` file into `lambda/.local-s3////`, monkey-patches `s3_utils.download_file` / `upload_file` to copy from/to the mirror, and invokes `lambda_handler` with a synthesized S3 event. The reusable patch surface lives in `local_s3_mirror.py` and mirrors what `tests/integration/conftest.py` already does for CI. Documents the workflow in `docs/local-development.md` and links from the "Adding an instrument" guide. Co-authored-by: Cursor --- .gitignore | 5 +- docs/guides/adding-an-instrument.md | 2 + docs/local-development.md | 43 ++++- lambda/src/data_hub_lambda/cli.py | 150 ++++++++++++++++++ lambda/src/data_hub_lambda/local_s3_mirror.py | 95 +++++++++++ 5 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 lambda/src/data_hub_lambda/local_s3_mirror.py diff --git a/.gitignore b/.gitignore index 18d48560..d07f03ac 100644 --- a/.gitignore +++ b/.gitignore @@ -218,4 +218,7 @@ __marimo__/ .streamlit/secrets.toml # macOS -.DS_Store \ No newline at end of file +.DS_Store + +# Local S3 mirror used by `data-hub-process handler` (see docs/local-development.md). +lambda/.local-s3/ \ No newline at end of file diff --git a/docs/guides/adding-an-instrument.md b/docs/guides/adding-an-instrument.md index 336bb545..d1e33a9b 100644 --- a/docs/guides/adding-an-instrument.md +++ b/docs/guides/adding-an-instrument.md @@ -111,6 +111,8 @@ Don't forget to add the import at the top of `handler.py`. Add unit tests in `lambda/tests/` for the new processor. Integration tests will automatically cover the new instrument if it's registered in the shared library. +For a quick end-to-end smoke against your local web app — without S3, AWS credentials, or LocalStack — use `data-hub-process handler` to drive `lambda_handler` against a gitignored local mirror. See [Testing the Lambda end-to-end](../local-development.md#testing-the-lambda-end-to-end). + ### 4.5 Configure the S3 trigger Add a `LambdaConfiguration` entry to the `RawDataBucket` resource's `NotificationConfiguration` in `infra/template.yaml`. Each entry specifies a prefix (the instrument ID) and a suffix (the file extension): diff --git a/docs/local-development.md b/docs/local-development.md index 5bc49502..7e9f0125 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -125,9 +125,50 @@ Some features depend on services that aren't running in this workflow. Each one | Run archive ("Download all") | 503 "Archive builder is not configured" | Set `LAMBDA_FUNCTION_URL` + `S3_ARCHIVES_BUCKET` and grant `lambda:InvokeFunctionUrl` | | File reprocessing | The reprocess endpoint returns null and no Lambda is invoked | Same | | Slack notifications on new runs | `console.warn` only, no HTTP call | Set `SLACK_WEBHOOK_URL` | -| Watcher uploads → Lambda → API loop | Not exercised; the seed inserts the resulting rows directly | Run the watcher (`docs/watcher.md`) and the Lambda (`docs/lambda.md`) end-to-end | +| Watcher uploads → Lambda → API loop | Not exercised; the seed inserts the resulting rows directly. For Lambda-only smoke testing, see [Testing the Lambda end-to-end](#testing-the-lambda-end-to-end) below | Run the watcher (`docs/watcher.md`) and the Lambda (`docs/lambda.md`) end-to-end | | Sign in with Google | The button still renders but OAuth callback will 4xx without `AUTH_GOOGLE_ID` / `AUTH_GOOGLE_SECRET` | `vercel env pull` per `docs/getting-started.md` | +## Testing the Lambda end-to-end + +Working on a `process_file()` module (or wiring up a brand new one — see [Adding an instrument](guides/adding-an-instrument.md)) and want to run it against the local web app without standing up real S3? The lambda CLI ships a `handler` subcommand that drives `lambda_handler` end-to-end against a gitignored directory mirroring the S3 layout. + +```sh +cd lambda + +# Point the inner DataHubClient at the local dev API. The PAT is printed +# by `npm run db:seed` / `make db-reseed`. +export DATA_HUB_API_URL=http://localhost:3000/api/v1 +export DATA_HUB_API_KEY=dhub_ + +# instrument_id / run_id / filename match the kebab-case S3 key layout the +# real Lambda expects; --source is the local file you want "uploaded". +uv run data-hub-process handler \ + agilent-4150-tapestation \ + run-1 \ + sample.csv \ + --source ~/Downloads/sample.csv +``` + +What happens under the hood: + +1. The CLI copies `--source` into `lambda/.local-s3/test-raw-data-bucket///`. That mirror directory is gitignored. +2. `AWS_S3_RAW_DATA_BUCKET` / `AWS_S3_PROCESSED_DATA_BUCKET` are set so `process_file()` modules see consistent bucket names. +3. `data_hub_shared.s3_utils.download_file` and `upload_file` are monkey-patched for the duration of the call to `shutil.copy2` from/to the mirror — no boto3, no AWS credentials, no LocalStack. +4. A synthetic S3 event is built and `lambda_handler(event, ctx)` runs the same dispatch path production uses, calling the dev API at `localhost:3000` for the run/file upserts. + +After it returns, navigate to `http://localhost:3000/instruments//runs/` to inspect what landed; processed artifacts show up under `lambda/.local-s3/test-processed-data-bucket/...`. + +Useful flags (`uv run data-hub-process handler --help` for the full list): + +| Flag | Default | Purpose | +| --- | --- | --- | +| `--source FILE` | required | Local file to stage as the "uploaded" raw object. | +| `--mirror-root DIR` | `/lambda/.local-s3` (or `$LOCAL_S3_MIRROR`) | Where the mirror lives on disk. | +| `--raw-bucket NAME` | `test-raw-data-bucket` | First path segment under the mirror for the raw file. | +| `--processed-bucket NAME` | `test-processed-data-bucket` | First path segment for `upload_file` calls from the processor. | + +The wiring lives in [lambda/src/data_hub_lambda/cli.py](../lambda/src/data_hub_lambda/cli.py) (`handler` subcommand) and [lambda/src/data_hub_lambda/local_s3_mirror.py](../lambda/src/data_hub_lambda/local_s3_mirror.py) (`patched_s3` context manager). The same patch surface backs the integration suite at [lambda/tests/integration/conftest.py](../lambda/tests/integration/conftest.py), so anything that works under the CLI is exercised in CI too. + ## Where the seed lives - [web/lib/db/seed.ts](../web/lib/db/seed.ts) — shared builder functions (`seedDevUser`, `seedInstruments`, `seedRuns`, etc.) plus a schema-driven `clearAll()`. diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 3b8cf920..6d7405f8 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -2,11 +2,19 @@ from __future__ import annotations import json +import os import shutil from pathlib import Path import click +# Repo root: ``cli.py`` lives at ``/lambda/src/data_hub_lambda/cli.py``, +# so four ``parents`` levels up land on the repo root. Used as the default +# anchor for ``--mirror-root`` so devs don't have to type a path on every +# ``handler`` invocation. +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DEFAULT_MIRROR_ROOT = _REPO_ROOT / "lambda" / ".local-s3" + @click.group() def cli() -> None: @@ -217,3 +225,145 @@ def tapestation(filename: str) -> None: click.echo(f"Tape type: {tape_type}") else: click.echo("No tape type found in filename.") + + +# --------------------------------------------------------------------------- +# End-to-end handler invocation against a local S3 mirror +# --------------------------------------------------------------------------- + + +def _reset_config_singletons() -> None: + """Re-initialize shared/lambda config and drop the cached API client. + + Mirrors ``lambda/tests/integration/conftest.py``: callers mutate + process env vars (bucket names, API URL) and then need the long-lived + ``config`` / ``lambda_config`` singletons to re-read them. Replacing + the objects would break ``from … import config`` consumers, so we + re-run ``__init__`` in place instead. + """ + import data_hub_lambda.api_client as _api_mod + import data_hub_lambda.config as _lcfg_mod + import data_hub_shared.config as _scfg_mod + + _api_mod._client = None + _scfg_mod.config.__init__() # type: ignore[misc] + _lcfg_mod.lambda_config.__init__() # type: ignore[misc] + + +@cli.command("handler") +@click.argument("instrument_id") +@click.argument("run_id") +@click.argument("filename") +@click.option( + "--source", + "source", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, + help="Local file to stage into the mirror as the 'uploaded' raw file.", +) +@click.option( + "--mirror-root", + type=click.Path(file_okay=False, path_type=Path), + default=None, + help=( + "Directory that mirrors S3 (defaults to /lambda/.local-s3 " + "or $LOCAL_S3_MIRROR if set)." + ), +) +@click.option( + "--raw-bucket", + default="test-raw-data-bucket", + show_default=True, + help="Bucket name used for the staged raw file and the synthesized S3 event.", +) +@click.option( + "--processed-bucket", + default="test-processed-data-bucket", + show_default=True, + help="Bucket name used by `upload_file` for processed artifacts.", +) +def handler( + instrument_id: str, + run_id: str, + filename: str, + source: Path, + mirror_root: Path | None, + raw_bucket: str, + processed_bucket: str, +) -> None: + """Run `lambda_handler` end-to-end against a local S3 mirror. + + Stages SOURCE at ////, + monkey-patches `s3_utils.download_file` / `upload_file` to copy from/to + the mirror, and invokes `lambda_handler` with a synthesized S3 event so + the per-instrument `process_file` runs against the local web app. + + Requires `DATA_HUB_API_URL` and `DATA_HUB_API_KEY` to be set so the + inner `DataHubClient` can hit the dev API (typically + `http://localhost:3000/api/v1` plus a PAT printed by `npm run db:seed`). + """ + from urllib.parse import quote_plus + + from data_hub_lambda.handler import lambda_handler + from data_hub_lambda.local_s3_mirror import make_mock_context, patched_s3 + from data_hub_shared.enums import Instrument + + valid_ids = sorted(member.value for member in Instrument) + if instrument_id not in valid_ids: + raise click.BadParameter( + f"Unknown instrument_id '{instrument_id}'. Valid values: {', '.join(valid_ids)}", + param_hint="INSTRUMENT_ID", + ) + + for var in ("DATA_HUB_API_URL", "DATA_HUB_API_KEY"): + if not os.environ.get(var): + raise click.UsageError( + f"{var} is not set. Point it at the local dev API " + "(e.g. http://localhost:3000/api/v1) and the seeded PAT " + "printed by `npm run db:seed`." + ) + + if mirror_root is None: + env_root = os.environ.get("LOCAL_S3_MIRROR") + mirror_root = Path(env_root) if env_root else _DEFAULT_MIRROR_ROOT + mirror_root = mirror_root.resolve() + + staged = mirror_root / raw_bucket / instrument_id / run_id / filename + staged.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, staged) + click.echo(f"Staged {source} -> {staged}") + + os.environ["AWS_S3_RAW_DATA_BUCKET"] = raw_bucket + os.environ["AWS_S3_PROCESSED_DATA_BUCKET"] = processed_bucket + _reset_config_singletons() + + # Real S3 events form-encode the object key (spaces -> '+', '+' -> '%2B'), + # which the handler decodes with `unquote_plus`. Match that here so a + # `filename` containing spaces or '+' round-trips the same as production. + s3_key = f"{instrument_id}/{run_id}/{filename}" + event = { + "Records": [ + { + "eventVersion": "2.1", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventName": "ObjectCreated:Put", + "s3": { + "bucket": {"name": raw_bucket}, + "object": { + "key": quote_plus(s3_key, safe="/"), + "size": staged.stat().st_size, + }, + }, + } + ] + } + + click.echo(f"Invoking lambda_handler for s3://{raw_bucket}/{s3_key}") + with patched_s3(mirror_root): + lambda_handler(event, make_mock_context()) # type: ignore[arg-type] + + click.echo("") + click.echo("Done. Inspect the result in the dev UI:") + click.echo(f" http://localhost:3000/instruments/{instrument_id}/runs/{run_id}") + click.echo(f"Mirror root: {mirror_root}") diff --git a/lambda/src/data_hub_lambda/local_s3_mirror.py b/lambda/src/data_hub_lambda/local_s3_mirror.py new file mode 100644 index 00000000..d3ed4c71 --- /dev/null +++ b/lambda/src/data_hub_lambda/local_s3_mirror.py @@ -0,0 +1,95 @@ +"""Local-disk S3 mirror used by the `data-hub-process handler` CLI. + +A "mirror" is a directory whose layout matches an S3 bucket layout — +``//``. Pairing this with monkey-patches of +``data_hub_shared.s3_utils.download_file`` / ``upload_file`` lets a +developer drive ``lambda_handler`` end-to-end against the local web app +without LocalStack, MinIO, or real AWS credentials. See +``docs/local-development.md`` for the full workflow. + +Kept intentionally small: a path mapper, a context manager that swaps +the two ``s3_utils`` entry points for ``shutil.copy2`` calls against the +mirror, and a ``MagicMock`` ``Context`` factory shared with the CLI. +The integration test conftest already mocks the same surface (see +``lambda/tests/integration/conftest.py``); this module is the +non-pytest equivalent. +""" + +from __future__ import annotations +import shutil +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from aws_lambda_typing.context import Context + +from data_hub_shared.s3_utils import parse_s3_uri + + +def mirror_path(root: Path, s3_uri: str) -> Path: + """Map an ``s3:///`` URI to ``//``. + + The bucket name becomes the first path segment so a single mirror + root can host both raw and processed buckets side-by-side, matching + how a real account isolates them. + """ + bucket, key = parse_s3_uri(s3_uri) + return root / bucket / key + + +@contextmanager +def patched_s3(root: Path) -> Generator[None, None, None]: + """Patch S3 download/upload to copy from/to a local mirror directory. + + ``download_file(s3_uri, local_path)`` copies ``//`` + into ``local_path``. ``upload_file(local_path, s3_uri)`` copies + ``local_path`` into ``//``. Both create parent + directories on the destination side so the caller never has to + pre-create them. + + A missing source on download raises ``FileNotFoundError`` with the + expected mirror path so the developer sees exactly where to drop + their fixture if they invoked the handler without staging first. + """ + + def _fake_download(s3_uri: str, local_path: Path, **_: Any) -> None: + src = mirror_path(root, s3_uri) + if not src.exists(): + raise FileNotFoundError( + f"No file staged at {src} for {s3_uri}. " + f"Stage one with `--source` or copy it into the mirror." + ) + local_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, local_path) + + def _fake_upload(local_path: Path, s3_uri: str, **_: Any) -> None: + dest = mirror_path(root, s3_uri) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(local_path, dest) + + with ( + patch("data_hub_shared.s3_utils.download_file", side_effect=_fake_download), + patch("data_hub_shared.s3_utils.upload_file", side_effect=_fake_upload), + ): + yield + + +def make_mock_context() -> MagicMock: + """Return a ``MagicMock`` shaped like ``aws_lambda_typing.context.Context``. + + The ``lambda_handler`` only reads a handful of attributes off the + context (logging metadata), so a thin mock with the typical AWS + identifiers is enough — and keeping it here means the CLI doesn't + need to import ``unittest.mock`` directly. + """ + ctx = MagicMock(spec=Context) + ctx.invoked_function_arn = "arn:aws:lambda:us-east-1:123456789012:function:data-hub-lambda" + ctx.log_group_name = "/aws/lambda/data-hub-lambda" + ctx.log_stream_name = "local-cli/handler" + ctx.function_name = "data-hub-lambda" + ctx.function_version = "$LATEST" + ctx.memory_limit_in_mb = "256" + ctx.aws_request_id = "local-cli-request" + return ctx From faee416dbc1e9d1cd8d4483f6e7d26558fefe3e0 Mon Sep 17 00:00:00 2001 From: Wasim Sandhu Date: Thu, 21 May 2026 16:24:29 -0700 Subject: [PATCH 2/6] Add filesystem-backed S3 mirror for local web app development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the Next.js app into the same `lambda/.local-s3` mirror that the lambda CLI's `data-hub-process handler` writes to, so file downloads, server-side reads, archive HEAD checks, and watcher-style PUT uploads work locally without AWS or LocalStack. When `LOCAL_S3_MIRROR` is set (and `NODE_ENV != production`), the four helpers in `web/lib/s3.ts` short-circuit the AWS SDK and resolve to `//` via a dev-only catch-all route at `/api/_local-s3/{bucket}/{...key}`. The route handles GET (streams the file with optional `Content-Disposition`) and PUT (writes the request body to disk via `pipeline()`). The dev seed copies fixture bytes from `lambda/tests/fixtures/` into the mirror for qPCR / gel doc / plate reader runs so seeded runs render real bytes immediately after `make db-reseed`. Other instrument types still 404 — devs stage real bytes via the lambda CLI. Co-authored-by: Cursor --- docs/local-development.md | 50 +++++-- web/.env.example | 9 ++ .../api/_local-s3/[bucket]/[...key]/route.ts | 114 +++++++++++++++ web/lib/db/seed.ts | 109 +++++++++++++-- web/lib/s3-local-mirror.ts | 131 ++++++++++++++++++ web/lib/s3.ts | 46 ++++++ web/scripts/seed-database.ts | 4 +- 7 files changed, 444 insertions(+), 19 deletions(-) create mode 100644 web/app/api/_local-s3/[bucket]/[...key]/route.ts create mode 100644 web/lib/s3-local-mirror.ts diff --git a/docs/local-development.md b/docs/local-development.md index 7e9f0125..3c689d56 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -39,15 +39,22 @@ DATABASE_URL=postgres://localhost:5432/data-hub-local # Any 32+ character string. NextAuth uses it to sign session JWTs. AUTH_SECRET=local-dev-secret-at-least-32-characters!! -# Dummy AWS credentials. The AWS SDK signs presigned URLs locally -# (HMAC-only — no network call), so any non-empty values work. Actual -# uploads/downloads against `test-raw-data-bucket` will fail at the -# browser, which is fine for local dev (see "What's deliberately -# missing" below). +# Dummy AWS credentials. The local-mirror branch in `web/lib/s3.ts` +# bypasses the AWS SDK entirely when LOCAL_S3_MIRROR is set, but a +# few server-side modules instantiate the SDK at import time so any +# non-empty values keep them happy. AWS_ACCESS_KEY_ID=test-key AWS_SECRET_ACCESS_KEY=test-secret AWS_REGION=us-east-1 S3_RAW_DATA_BUCKET=test-raw-data-bucket + +# Filesystem-backed S3 mirror. When set (and NODE_ENV != production), +# `web/lib/s3.ts` swaps presigned-URL generation, HEAD checks, and +# server-side stream reads for `//` lookups, and +# the seed copies fixture bytes into the mirror so seeded runs +# render real bytes. Path is resolved relative to web/. See +# "Working with file bytes locally" below. +LOCAL_S3_MIRROR=../lambda/.local-s3 ``` Explicitly **do not** set the following — leaving them unset is what makes the relevant features short-circuit cleanly: @@ -104,7 +111,7 @@ You can also run `npm run db:seed` on its own — it calls the schema-driven `cl | `watcher_heartbeats` | ~10 per watching watcher | Spread over the last hour | | `watcher_events` | 3 per watching watcher | `watcher_started`, `config_synced`, `file_uploaded` | | `instrument_runs` | 5 per active instrument | Spread across the last ~2 weeks (3, 6, 9, 12, 15 days back), alternating `lambda` / `watcher` source | -| `files` | 3 per run | Mix of `uploaded` / `completed` / `failed`, `raw` / `processed` | +| `files` | 3 per run | Mix of `uploaded` / `completed` / `failed`, `raw` / `processed`. For qPCR / gel doc / plate reader the first slot uses a real fixture filename and bytes are copied into `LOCAL_S3_MIRROR` (see [Working with file bytes locally](#working-with-file-bytes-locally)) | | `run_comments` | 1 per run | Authored by the dev user | | `run_attributions` | 1 per run | Dev user attributed | | `archive_jobs` | 3 | One each of `ready` / `building` / `failed` | @@ -120,12 +127,12 @@ Some features depend on services that aren't running in this workflow. Each one | Feature | Behavior locally | How to enable | | --- | --- | --- | -| File download | Presigned URL renders but GET fails — no real S3 object | Point S3 env vars at a real bucket or LocalStack/MinIO | -| File upload (from watcher) | `request-upload-url` returns a usable signed URL, but actually PUTting fails | Same | +| File download | Served from `LOCAL_S3_MIRROR` if set (real bytes for seeded qPCR / gel doc / plate reader runs out of the box; other instruments staged via `data-hub-process handler`). 404s when unset. See [Working with file bytes locally](#working-with-file-bytes-locally) | Point S3 env vars at a real bucket or LocalStack/MinIO | +| File upload (from watcher) | `request-upload-url` returns a same-origin URL routed to `/api/_local-s3/...`; `PUT` writes bytes into the mirror | Same | | Run archive ("Download all") | 503 "Archive builder is not configured" | Set `LAMBDA_FUNCTION_URL` + `S3_ARCHIVES_BUCKET` and grant `lambda:InvokeFunctionUrl` | | File reprocessing | The reprocess endpoint returns null and no Lambda is invoked | Same | | Slack notifications on new runs | `console.warn` only, no HTTP call | Set `SLACK_WEBHOOK_URL` | -| Watcher uploads → Lambda → API loop | Not exercised; the seed inserts the resulting rows directly. For Lambda-only smoke testing, see [Testing the Lambda end-to-end](#testing-the-lambda-end-to-end) below | Run the watcher (`docs/watcher.md`) and the Lambda (`docs/lambda.md`) end-to-end | +| Watcher uploads → Lambda → API loop | Not exercised end-to-end; the seed inserts the resulting rows directly. For Lambda-only smoke testing, see [Testing the Lambda end-to-end](#testing-the-lambda-end-to-end) below | Run the watcher (`docs/watcher.md`) and the Lambda (`docs/lambda.md`) end-to-end | | Sign in with Google | The button still renders but OAuth callback will 4xx without `AUTH_GOOGLE_ID` / `AUTH_GOOGLE_SECRET` | `vercel env pull` per `docs/getting-started.md` | ## Testing the Lambda end-to-end @@ -169,6 +176,31 @@ Useful flags (`uv run data-hub-process handler --help` for the full list): The wiring lives in [lambda/src/data_hub_lambda/cli.py](../lambda/src/data_hub_lambda/cli.py) (`handler` subcommand) and [lambda/src/data_hub_lambda/local_s3_mirror.py](../lambda/src/data_hub_lambda/local_s3_mirror.py) (`patched_s3` context manager). The same patch surface backs the integration suite at [lambda/tests/integration/conftest.py](../lambda/tests/integration/conftest.py), so anything that works under the CLI is exercised in CI too. +## Working with file bytes locally + +`LOCAL_S3_MIRROR` makes the Next.js app share the same on-disk layout the lambda CLI writes to. When it's set (and `NODE_ENV != production`), the four helpers in [web/lib/s3.ts](../web/lib/s3.ts) — `getPresignedDownloadUrl`, `getPresignedUploadUrl`, `headS3Object`, `getS3ObjectStream` — short-circuit AWS and serve from disk. The HTTP face of the mirror is a single dev-only catch-all at [web/app/api/_local-s3/[bucket]/[...key]/route.ts](../web/app/api/_local-s3/%5Bbucket%5D/%5B...key%5D/route.ts) that handles GET (download with optional `Content-Disposition`) and PUT (writes bytes from the request body). + +What this gets you out of the box after `make db-reseed`: + +| Instrument type | Seeded fixture | Where it comes from | +| --- | --- | --- | +| qPCR | `azure_cielo_qpcr_example.csv` | `lambda/tests/fixtures/` | +| Gel doc | `azure_600_gel_doc_example.tif` | `lambda/tests/fixtures/` | +| Plate reader | `spectramax_plate_reader_endpoint.xls` | `lambda/tests/fixtures/` | +| Other instruments | none — files 404 in the mirror | Stage real bytes via `data-hub-process handler` | + +The seed copies the fixture into `/test-raw-data-bucket///` for every seeded run on those instruments, so navigating to `/instruments/seed-qpcr/runs/seed-run-1` shows a real CSV in the file browser, the colony / plate-reader viewers fetch real bytes via `/api/v1/files//download`, and PNG / TIFF / PDF previews on `RunReportSection` render without 404s. + +For instrument types without a fixture (or new file types you're adding components for), the existing CLI flow stays the same: run `data-hub-process handler --source ` and the dashboard picks up the file the moment the API row lands. + +Components don't need to change — every existing run viewer already fetches `/api/v1/files//download`, which 302s to whatever `getPresignedDownloadUrl` returns. New custom components for a specific instrument should follow the same pattern (`fetch("/api/v1/files//download")` for raw bytes, `` for images) and inherit local-mirror support automatically. + +A few details worth knowing: + +- Adding fixtures for more instruments is one entry in `INSTRUMENT_FIXTURES` in [web/lib/db/seed.ts](../web/lib/db/seed.ts), pointing at any file under `lambda/tests/fixtures/`. +- The route is gated on `NODE_ENV !== "production"` AND `LOCAL_S3_MIRROR` set; either condition unmet returns 404 unconditionally, so a production build can never expose the filesystem. +- The MCP tool at `/api/v1/mcp` returns a relative `/api/_local-s3/...` URL when the mirror is active — browsers resolve it against the current origin, but non-browser MCP clients on localhost may need to prefix with `http://localhost:3000`. + ## Where the seed lives - [web/lib/db/seed.ts](../web/lib/db/seed.ts) — shared builder functions (`seedDevUser`, `seedInstruments`, `seedRuns`, etc.) plus a schema-driven `clearAll()`. diff --git a/web/.env.example b/web/.env.example index 244ff65c..f079cb2e 100644 --- a/web/.env.example +++ b/web/.env.example @@ -33,3 +33,12 @@ SLACK_WEBHOOK_URL= # saves a row, GET /api/v1/watchers/:id/update-check returns # latest_version: null and watchers skip the self-update — the same # behavior as leaving WATCHER_LATEST_VERSION unset used to be. + +# Optional: local S3 mirror for dev. When set (and NODE_ENV != production), +# the four helpers in `lib/s3.ts` short-circuit AWS and serve bytes from +# this directory via `/api/_local-s3/{bucket}/{key}` instead. The seed +# also copies fixture files from `lambda/tests/fixtures/` into the +# mirror for qPCR / gel doc / plate reader so seeded runs render real +# bytes immediately. Path is resolved relative to web/. See +# docs/local-development.md. +# LOCAL_S3_MIRROR=../lambda/.local-s3 diff --git a/web/app/api/_local-s3/[bucket]/[...key]/route.ts b/web/app/api/_local-s3/[bucket]/[...key]/route.ts new file mode 100644 index 00000000..db39add0 --- /dev/null +++ b/web/app/api/_local-s3/[bucket]/[...key]/route.ts @@ -0,0 +1,114 @@ +// Local-only S3 mirror endpoint. Serves bytes from +// `//` on GET and writes bytes there on +// PUT. The matching dispatch lives in `web/lib/s3-local-mirror.ts`, +// which `web/lib/s3.ts` calls into when the env var is set. +// +// Gating: `getLocalMirrorRoot()` returns `null` whenever +// `NODE_ENV === "production"` OR `LOCAL_S3_MIRROR` is unset. Both +// handlers short-circuit to a 404 in that case, so a production +// build can never expose the filesystem even if the file is somehow +// included in the bundle. +// +// Runtime: stays on the default Node.js runtime — `fs` is unavailable +// on the Edge runtime and there's no production deployment story for +// this route anyway. + +import { + getLocalMirrorRoot, + mimeFor, + resolveMirrorPath, +} from "@/lib/s3-local-mirror"; +import type { NextRequest } from "next/server"; +import { createReadStream, createWriteStream } from "node:fs"; +import { mkdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as NodeWebReadableStream } from "node:stream/web"; + +type RouteContext = { + params: Promise<{ bucket: string; key: string[] }>; +}; + +const NOT_FOUND_RESPONSE = () => new Response("Not Found", { status: 404 }); + +export async function GET(request: NextRequest, { params }: RouteContext) { + const root = getLocalMirrorRoot(); + if (!root) return NOT_FOUND_RESPONSE(); + + const { bucket, key } = await params; + const joinedKey = key.join("/"); + + let filePath: string; + try { + filePath = resolveMirrorPath(root, bucket, joinedKey); + } catch (err) { + console.warn(`[local-s3] rejected GET ${bucket}/${joinedKey}: ${err}`); + return NOT_FOUND_RESPONSE(); + } + + let fileSize: number; + try { + const s = await stat(filePath); + if (!s.isFile()) return NOT_FOUND_RESPONSE(); + fileSize = s.size; + } catch { + return NOT_FOUND_RESPONSE(); + } + + const disposition = new URL(request.url).searchParams.get("disposition"); + + // `Readable.toWeb` returns the `node:stream/web` `ReadableStream` type, + // which is structurally compatible with the global `ReadableStream` + // that `Response` expects. Cast through `unknown` to bridge the two + // declarations without pulling DOM lib types into the Node side. + const body = Readable.toWeb( + createReadStream(filePath) + ) as unknown as ReadableStream; + + return new Response(body, { + status: 200, + headers: { + "Content-Type": mimeFor(filePath), + "Content-Length": String(fileSize), + ...(disposition && { "Content-Disposition": disposition }), + "Cache-Control": "no-store", + }, + }); +} + +export async function PUT(request: NextRequest, { params }: RouteContext) { + const root = getLocalMirrorRoot(); + if (!root) return NOT_FOUND_RESPONSE(); + + const { bucket, key } = await params; + const joinedKey = key.join("/"); + + let filePath: string; + try { + filePath = resolveMirrorPath(root, bucket, joinedKey); + } catch (err) { + console.warn(`[local-s3] rejected PUT ${bucket}/${joinedKey}: ${err}`); + return new Response("Bad Request", { status: 400 }); + } + + if (!request.body) { + return new Response("Empty body", { status: 400 }); + } + + await mkdir(path.dirname(filePath), { recursive: true }); + + // `pipeline` ensures the write stream is closed even if the upload + // is aborted mid-stream, and surfaces backpressure errors as a + // rejected promise. Casting the request body to the Node web + // ReadableStream type lets `Readable.fromWeb` accept it without TS + // complaining about the DOM/Node `ReadableStream` mismatch. + await pipeline( + Readable.fromWeb( + request.body as unknown as NodeWebReadableStream + ), + createWriteStream(filePath) + ); + + return new Response(null, { status: 200 }); +} diff --git a/web/lib/db/seed.ts b/web/lib/db/seed.ts index 9c1f7879..128810ab 100644 --- a/web/lib/db/seed.ts +++ b/web/lib/db/seed.ts @@ -12,10 +12,20 @@ import { generateToken, getTokenPrefix, hashToken } from "@/lib/tokens"; import { getTableName, isTable, sql } from "drizzle-orm"; import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; +import { copyFile, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import * as schema from "./schema"; export type Db = PostgresJsDatabase; +// Bucket name shared with the lambda CLI's `data-hub-process handler` +// (`--raw-bucket` default). Exporting the constant means the seed and +// the CLI never drift — both the row's `s3_bucket` field and the +// directory under `LOCAL_S3_MIRROR` use the same string. +export const RAW_BUCKET = "test-raw-data-bucket"; +export const ARCHIVES_BUCKET = "test-archives-bucket"; + // --------------------------------------------------------------------------- // clearAll — schema-driven TRUNCATE of every `pgTable` declared in // `lib/db/schema.ts`. Uses CASCADE so FK ordering doesn't matter; the @@ -250,10 +260,13 @@ export async function seedWatchers( } // --------------------------------------------------------------------------- -// Runs + files — fake S3 keys under a `test-raw-data-bucket` namespace so -// signed-URL generation works locally (signing is HMAC-only). Actually -// fetching the files won't work without real S3, which is documented in -// docs/local-development.md. +// Runs + files — keys live under `RAW_BUCKET` so the lambda CLI's +// `--raw-bucket` default and any `LOCAL_S3_MIRROR` directory layout +// match. When `LOCAL_S3_MIRROR` is set in dev, `seedRuns` also copies +// the fixture from `lambda/tests/fixtures/` for instruments listed in +// `INSTRUMENT_FIXTURES` so seeded runs render real bytes in the +// dashboard out of the box. Other instruments still 404 — devs use +// `data-hub-process handler` to stage real files for those. // --------------------------------------------------------------------------- export type SeededRun = { @@ -264,10 +277,49 @@ export type SeededRun = { const FILE_STATUSES = ["uploaded", "completed", "failed"] as const; +// Maps each instrument-type that has a fixture file checked into the +// repo to its fixture filename and content-type. Adding a new entry +// here is enough to make every seeded run for that instrument type +// render real bytes (provided `LOCAL_S3_MIRROR` is set). Instruments +// without an entry keep their synthetic `raw_1.csv` filenames and +// 404 against the local mirror. +const INSTRUMENT_FIXTURES: Partial< + Record +> = { + qpcr: { + filename: "azure_cielo_qpcr_example.csv", + contentType: "text/csv", + }, + gel_doc: { + filename: "azure_600_gel_doc_example.tif", + contentType: "image/tiff", + }, + plate_reader: { + filename: "spectramax_plate_reader_endpoint.xls", + contentType: "application/vnd.ms-excel", + }, +}; + +// Resolve `lambda/tests/fixtures/` relative to this file so the path +// doesn't depend on `process.cwd()`. The seed entry-point and the +// integration test harness run from different working directories +// but both end up importing this module from the same on-disk path. +const SEED_DIR = path.dirname(fileURLToPath(import.meta.url)); +const FIXTURES_DIR = path.resolve( + SEED_DIR, + "..", + "..", + "..", + "lambda", + "tests", + "fixtures" +); + export async function seedRuns( db: Db, instrumentId: string, - count: number = 5 + count: number = 5, + instrumentType?: schema.InstrumentType ): Promise { if (count <= 0) return []; @@ -296,18 +348,31 @@ export async function seedRuns( runId: schema.instrumentRuns.runId, }); + const fixture = instrumentType + ? INSTRUMENT_FIXTURES[instrumentType] + : undefined; + const fileRows = runs.flatMap((run, runIdx) => Array.from({ length: 3 }, (_, fi) => { const status = FILE_STATUSES[(runIdx + fi) % FILE_STATUSES.length]; const category = fi === 2 ? ("processed" as const) : ("raw" as const); - const filename = `${category}_${fi + 1}.csv`; + // Slot 0 is the "real" raw file when a fixture exists. Slots 1 + // and 2 keep their synthetic CSV names for status/category mix + // and 404 in the local mirror — the dev still has at least one + // viewable file per run, which is what custom-component + // development needs. + const useFixture = fi === 0 && fixture !== undefined; + const filename = useFixture + ? fixture.filename + : `${category}_${fi + 1}.csv`; + const contentType = useFixture ? fixture.contentType : "text/csv"; return { instrumentRunId: run.id, relativePath: filename, - s3Bucket: "test-raw-data-bucket", + s3Bucket: RAW_BUCKET, s3Key: `${run.instrumentId}/${run.runId}/${filename}`, filename, - contentType: "text/csv", + contentType, sizeBytes: 1024 * (fi + 1), category, status, @@ -321,6 +386,32 @@ export async function seedRuns( ); await db.insert(schema.files).values(fileRows); + // If the local-mirror env var is set and this instrument type has + // a fixture, copy the fixture bytes into + // `////` + // for every run. The web app's local-mirror route then serves them + // when the dashboard requests `/api/v1/files//download`. + // Production-safety: `LOCAL_S3_MIRROR` is ignored in production by + // `getLocalMirrorRoot` anyway, but seeding is also strictly a dev + // workflow so this branch only fires locally regardless. + const mirrorRoot = process.env.LOCAL_S3_MIRROR; + if (mirrorRoot && fixture) { + const src = path.resolve(FIXTURES_DIR, fixture.filename); + await Promise.all( + runs.map(async (run) => { + const dest = path.resolve( + mirrorRoot, + RAW_BUCKET, + run.instrumentId, + run.runId, + fixture.filename + ); + await mkdir(path.dirname(dest), { recursive: true }); + await copyFile(src, dest); + }) + ); + } + return runs; } @@ -602,7 +693,7 @@ export async function seedArchiveJobs( return { instrumentRunId: run.id, fingerprint: `seed-fingerprint-${run.runId}`, - archiveBucket: status === "ready" ? "test-archives-bucket" : null, + archiveBucket: status === "ready" ? ARCHIVES_BUCKET : null, archiveKey: status === "ready" ? `runs/${run.instrumentId}/${run.runId}/seed.zip` diff --git a/web/lib/s3-local-mirror.ts b/web/lib/s3-local-mirror.ts new file mode 100644 index 00000000..bbb577e6 --- /dev/null +++ b/web/lib/s3-local-mirror.ts @@ -0,0 +1,131 @@ +// Local-disk S3 mirror dispatch for the dev workflow. When +// `LOCAL_S3_MIRROR` is set (and `NODE_ENV !== "production"`), +// `web/lib/s3.ts` short-circuits its AWS SDK calls and serves bytes +// from this mirror instead. The mirror layout matches an S3 bucket +// layout: `//`. +// +// Kept in its own module so the AWS code path in `s3.ts` stays +// untouched and so callers never need to know which implementation +// they're hitting. The companion route at +// `app/api/_local-s3/[bucket]/[...key]/route.ts` is the HTTP face of +// the same mirror; the lambda CLI's `data-hub-process handler` +// command is the python-side writer. +// +// Production-safety: `getLocalMirrorRoot()` returns `null` whenever +// `NODE_ENV === "production"`, so the local code path can never be +// activated in a Vercel production build even if the env var is +// somehow leaked into that environment. + +import { stat } from "node:fs/promises"; +import path from "node:path"; + +const MIME_MAP: Record = { + ".csv": "text/csv", + ".tsv": "text/tab-separated-values", + ".json": "application/json", + ".txt": "text/plain", + ".xls": "application/vnd.ms-excel", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".tif": "image/tiff", + ".tiff": "image/tiff", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".pdf": "application/pdf", + ".zip": "application/zip", + ".nd2": "application/octet-stream", +}; + +export function getLocalMirrorRoot(): string | null { + if (process.env.NODE_ENV === "production") return null; + const v = process.env.LOCAL_S3_MIRROR; + return v ? path.resolve(v) : null; +} + +// Resolve `//` and refuse anything that escapes +// the mirror root. Done with a string-prefix check on the absolute +// resolved path because `path.resolve` collapses `..` segments +// before we can sanity-check them — checking the input string for +// `..` would miss URL-encoded variants and Windows-style separators. +export function resolveMirrorPath( + root: string, + bucket: string, + key: string +): string { + const resolved = path.resolve(root, bucket, key); + const rootWithSep = root.endsWith(path.sep) ? root : root + path.sep; + if (!resolved.startsWith(rootWithSep)) { + throw new Error(`Path traversal blocked: bucket=${bucket} key=${key}`); + } + return resolved; +} + +export function mimeFor(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + return MIME_MAP[ext] ?? "application/octet-stream"; +} + +// Sanitize a filename for use inside a `Content-Disposition` header. +// Mirrors the same logic in `web/lib/s3.ts` so the local route's +// response headers match what the AWS path would have produced via +// `ResponseContentDisposition` on a presigned URL. +function sanitizeContentDispositionFilename(name: string): string { + const cleaned = name + .replaceAll(/[\r\n"\\]/g, "") + .replaceAll(/[\x00-\x1f\x7f]/g, "") + .trim(); + return cleaned.slice(0, 200) || "download"; +} + +// Build the same-origin URL that points at the local-mirror route. +// Each path segment is `encodeURIComponent`'d so a key with spaces or +// `+` round-trips correctly through the `[...key]` catch-all in the +// route handler. The bucket is encoded as a single segment. +// +// Returning a same-origin (relative) URL is intentional: every +// browser-driven consumer (302 redirects, ``, ``, +// `