Add first-class support for the Hina microscope

docs/lambda.md

@@ -33,6 +33,7 @@ When a file fails processing (or needs to be re-run), users can trigger reproces
 | Akta FPLC | `akta_fplc` | `akta-fplc` |
 | Azure 600 Gel Doc | `azure_600_gel_doc` | `azure-600-gel-doc` |
 | Azure Cielo qPCR | `azure_cielo_qpcr` | `azure-cielo-qpcr` |
+| Hina Microscope | `hina_microscope` | `hina-microscope` |
 | SpectraMax iD3 Plate Reader | `spectramax_plate_reader` | `spectramax-id3-plate-reader` |
 | SpectraMax iD5 Plate Reader | `spectramax_plate_reader` | `spectramax-id5-plate-reader` |
 
@@ -67,6 +68,7 @@ Available commands:
 | Command | Description |
 | --- | --- |
 | `gel-doc` | Process an Azure 600 Gel Doc TIFF (contrast-enhanced PNG + metadata) |
+| `hina` | Convert a Hina microscope ND2 file to a JPG overlay + metadata |
 | `qpcr` | Parse dye channels from an Azure Cielo qPCR Cq Values CSV |
 | `spectramax` | Parse metadata and raw well data from a SpectraMax `.xls` export |
 | `tapestation` | Extract the tape type from a TapeStation CSV filename |
@@ -102,6 +104,7 @@ The Lambda function depends on a scientific Python stack:
 - `matplotlib` — plotting
 - `scikit-image` — image processing
 - `tifffile` — TIFF file reading
+- `arcadia-microscopy-tools` — ND2 reading, channel handling, and multi-channel compositing for the Hina microscope
 - `pydantic` — data validation
 - `requests` — HTTP client for the Data Hub API
 - `aws-lambda-typing` — type stubs for Lambda events/context

infra/template.yaml

@@ -52,8 +52,12 @@ Parameters:
 
 Globals:
   Function:
-    Timeout: 300
-    MemorySize: 1024
+    # Large files (e.g. 3-4 GB Hina microscope ND2s) force us near Lambda's
+    # hard limits: 10 GB RAM to hold the decoded image array, 900 s to
+    # download + process, and 10 GB of /tmp (set per-function below) to
+    # land the raw file plus the generated JPG.
+    Timeout: 900
+    MemorySize: 10240
 
 Resources:
   # ---- S3 buckets ----
@@ -107,6 +111,15 @@ Resources:
                     # with `+` — this matches keys ending in `_Cq Values.csv`.
                     Value: _Cq+Values.csv
             Function: !GetAtt DataHubFunction.Arn
+          - Event: s3:ObjectCreated:*
+            Filter:
+              S3Key:
+                Rules:
+                  - Name: prefix
+                    Value: hina-microscope/
+                  - Name: suffix
+                    Value: .nd2
+            Function: !GetAtt DataHubFunction.Arn
           - Event: s3:ObjectCreated:*
             Filter:
               S3Key:
@@ -150,7 +163,9 @@ Resources:
       PackageType: Image
       ImageUri: !Ref EcrImageUri
       EphemeralStorage:
-        Size: 512
+        # 10 GB (the Lambda max) — large Hina ND2 uploads need room for the
+        # raw download, the decoded working copy, and the generated JPG.
+        Size: 10240
       Role: !GetAtt LambdaExecutionRole.Arn
       FunctionUrlConfig:
         AuthType: NONE

lambda/pyproject.toml

@@ -4,6 +4,7 @@ version = "0.2.0"
 requires-python = ">=3.12"
 dependencies = [
     "data-hub-shared",
+    "arcadia-microscopy-tools>=0.3.2",
     "aws-lambda-typing>=2.20.0",
     "click>=8.1",
     "matplotlib>=3.8",

lambda/src/data_hub_lambda/cli.py

@@ -51,6 +51,46 @@ def gel_doc(file: Path, output_dir: Path | None) -> None:
     click.echo(json.dumps(metadata, indent=2))
 
 
+# ---------------------------------------------------------------------------
+# Hina microscope (Nikon ND2)
+# ---------------------------------------------------------------------------
+
+
+@cli.command("hina")
+@click.argument("file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option(
+    "--output-dir",
+    type=click.Path(file_okay=False, path_type=Path),
+    default=None,
+    help="Directory for the exported JPG (default: same directory as FILE).",
+)
+def hina(file: Path, output_dir: Path | None) -> None:
+    """Convert a Hina microscope ND2 file to a JPG overlay.
+
+    Loads the ND2 via `arcadia-microscopy-tools`, produces a composite
+    JPG overlay (per-channel percentile-stretched intensities blended onto
+    a brightfield/zero background using each channel's native color), and
+    prints the parsed run-level metadata.
+    """
+    from data_hub_lambda.hina_microscope.image_processing import ND2Processor
+    from data_hub_lambda.hina_microscope.parse_metadata import parse_metadata
+
+    processor = ND2Processor(file)
+    processor.load()
+    jpg_path = processor.export_jpg()
+
+    if output_dir is not None:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        dest = output_dir / jpg_path.name
+        shutil.move(str(jpg_path), str(dest))
+        jpg_path = dest
+
+    click.echo(f"Exported JPG: {jpg_path}")
+
+    metadata = parse_metadata(processor.image)
+    click.echo(json.dumps(metadata, indent=2))
+
+
 # ---------------------------------------------------------------------------
 # Azure Cielo qPCR
 # ---------------------------------------------------------------------------

lambda/src/data_hub_lambda/handler.py

@@ -17,6 +17,7 @@
     akta_fplc,
     azure_600_gel_doc,
     azure_cielo_qpcr,
+    hina_microscope,
     spectramax_plate_reader,
 )
 from data_hub_shared import slack
@@ -188,6 +189,11 @@ def lambda_handler(event: dict[str, Any], context: Context) -> dict[str, Any] |
     logger.info("Run ID: '%s'", run_id)
     instrument_name = INSTRUMENT_ID_TO_NAME_MAP[instrument_id]
 
+    # Pre-cleanup: if the previous invocation on this warm container was
+    # SIGKILL'd (e.g. OOM), the `finally` block below didn't run and stale
+    # downloads may still be sitting in /tmp. Wipe them before we start.
+    _cleanup_tmp()
+
     try:
         logger.info("Processing file %s...", event_info.filename)
 
@@ -215,6 +221,12 @@ def lambda_handler(event: dict[str, Any], context: Context) -> dict[str, Any] |
                 filename=event_info.filename,
             )
 
+        elif instrument_id == Instrument.HINA_MICROSCOPE.value:
+            result_url = hina_microscope.process_file(
+                run_id=event_info.run_id,
+                filename=event_info.filename,
+            )
+
         elif instrument_id in (
             Instrument.SPECTRAMAX_ID3_PLATE_READER.value,
             Instrument.SPECTRAMAX_ID5_PLATE_READER.value,

lambda/src/data_hub_lambda/hina_microscope/__init__.py

@@ -0,0 +1,3 @@
+from data_hub_lambda.hina_microscope.process_file import (
+    process_file,  # noqa: F401
+)

lambda/src/data_hub_lambda/hina_microscope/image_processing.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+import logging
+from pathlib import Path
+
+import numpy as np
+import skimage as ski
+from arcadia_microscopy_tools import MicroscopyImage
+from arcadia_microscopy_tools.blending import overlay_channels
+from arcadia_microscopy_tools.channels import BRIGHTFIELD, Channel
+from numpy.typing import NDArray
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+ND2_SUFFIXES = (".nd2",)
+
+# Percentile range for per-channel contrast stretching before overlay.
+# 1st-99th percentile clips hot pixels / rare noise peaks while keeping the
+# bulk of the dynamic range visible.
+CONTRAST_PERCENTILES: tuple[float, float] = (1.0, 99.0)
+
+# JPEG quality for the exported composite.
+JPEG_QUALITY = 90
+
+
+class ND2Processor:
+    """Convert a Nikon ND2 file into a per-run JPG preview.
+
+    The pipeline uses `arcadia_microscopy_tools.MicroscopyImage` to load the
+    ND2, reduces each channel down to a single 2D frame (max-projection over
+    Z, first index over T / P), percentile-stretches intensities, and then
+    composites the channels into an RGB overlay using each channel's native
+    fluorophore color via `overlay_channels`.
+    """
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._image: MicroscopyImage | None = None
+
+    def load(self) -> None:
+        if not self.path.exists():
+            raise FileNotFoundError(f"ND2 file not found: {self.path}")
+        if self.path.suffix.lower() not in ND2_SUFFIXES:
+            raise ValueError(f"Expected ND2 file (.nd2), got: {self.path.suffix}")
+
+        self._image = MicroscopyImage.from_nd2_path(self.path)
+
+    @property
+    def image(self) -> MicroscopyImage:
+        if self._image is None:
+            raise RuntimeError("Call load() first.")
+        return self._image
+
+    def export_jpg(self) -> Path:
+        """Render the composite overlay and write it as a JPG next to the source."""
+        rgb = self._render_rgb()
+        rgb_uint8 = (np.clip(rgb, 0.0, 1.0) * 255).astype(np.uint8)
+
+        jpg_path = self.path.parent / f"{self.path.stem}.jpg"
+        Image.fromarray(rgb_uint8, mode="RGB").save(jpg_path, format="JPEG", quality=JPEG_QUALITY)
+        return jpg_path
+
+    def _render_rgb(self) -> NDArray[np.float64]:
+        """Produce the RGB overlay for the loaded image."""
+        per_channel_2d: dict[Channel, NDArray[np.float64]] = {}
+        for channel in self.image.channels:
+            intensities = self.image.get_intensities_from_channel(channel)
+            reduced = self._reduce_to_2d(intensities, self._non_channel_axes())
+            per_channel_2d[channel] = _rescale_percentile(reduced, CONTRAST_PERCENTILES)
+
+        background = self._pick_background(per_channel_2d)
+
+        # Fluorescence channels are overlaid on top of the grayscale background.
+        # Skip the background channel (if it was picked from the image) so it
+        # isn't blended onto itself.
+        overlay_inputs = {
+            ch: arr for ch, arr in per_channel_2d.items() if ch.name != BRIGHTFIELD.name
+        }
+        if not overlay_inputs:
+            # Single-channel brightfield (or equivalent): return the background
+            # as an RGB image so the caller still gets a valid overlay.
+            return ski.color.gray2rgb(background)
+
+        return overlay_channels(background, overlay_inputs)
+
+    def _non_channel_axes(self) -> list[str]:
+        """Ordered axis labels for the per-channel array (C dropped)."""
+        return [axis for axis in self.image.sizes.keys() if axis != "C"]
+
+    @staticmethod
+    def _reduce_to_2d(
+        intensities: NDArray,  # type: ignore[type-arg]
+        axis_labels: list[str],
+    ) -> NDArray[np.float64]:
+        """Collapse leading axes down to a (Y, X) frame.
+
+        Z axes are max-projected; T and P axes fall back to the first index.
+        Any unknown leading axis is also reduced by taking the first index,
+        with a warning logged.
+        """
+        arr = intensities
+        labels = list(axis_labels)
+        while len(labels) > 2:
+            label = labels[0]
+            if label == "Z":
+                arr = arr.max(axis=0)
+            elif label in ("T", "P"):
+                logger.info(
+                    "Reducing axis %s (size %d) by taking first index only.",
+                    label,
+                    arr.shape[0],
+                )
+                arr = arr[0]
+            else:
+                logger.warning(
+                    "Unknown leading axis %s (size %d); taking first index.",
+                    label,
+                    arr.shape[0],
+                )
+                arr = arr[0]
+            labels = labels[1:]
+        return arr
+
+    @staticmethod
+    def _pick_background(
+        per_channel: dict[Channel, NDArray[np.float64]],
+    ) -> NDArray[np.float64]:
+        """Pick a grayscale [0, 1] background for the overlay.
+
+        Prefers an existing BRIGHTFIELD channel (gives a natural context
+        image); falls back to zeros with the same 2D shape as the first
+        channel so fluorescence alone still renders correctly.
+        """
+        for channel, arr in per_channel.items():
+            if channel.name == BRIGHTFIELD.name:
+                return arr
+
+        first = next(iter(per_channel.values()))
+        return np.zeros_like(first, dtype=np.float64)
+
+
+def _rescale_percentile(
+    intensities: NDArray,  # type: ignore[type-arg]
+    percentiles: tuple[float, float],
+) -> NDArray[np.float64]:
+    """Percentile-based contrast stretching into [0, 1]."""
+    if intensities.size == 0:
+        return np.zeros_like(intensities, dtype=np.float64)
+
+    lo, hi = np.percentile(intensities, percentiles)
+    if lo == hi:
+        return np.zeros_like(intensities, dtype=np.float64)
+
+    rescaled = ski.exposure.rescale_intensity(
+        intensities,
+        in_range=(lo, hi),  # type: ignore[arg-type]
+        out_range=(0.0, 1.0),  # type: ignore[arg-type]
+    )
+    return rescaled.astype(np.float64)

lambda/src/data_hub_lambda/hina_microscope/parse_metadata.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+from typing import Any
+
+from arcadia_microscopy_tools import MicroscopyImage
+from arcadia_microscopy_tools.channels import Channel
+from arcadia_microscopy_tools.metadata_structures import DimensionFlags
+
+
+def parse_metadata(image: MicroscopyImage) -> dict[str, Any]:
+    """Extract run-level metadata from a loaded Nikon ND2 image.
+
+    Returns a JSON-serializable dict with three keys:
+
+    - `sizes`: the full dimension map, e.g. `{"C": 4, "Y": 256, "X": 256}`.
+    - `channels`: a list of `{name, excitation_nm, emission_nm, color}` dicts.
+    - `dimensions`: a list of `DimensionFlags` member names set on the image.
+
+    The function operates on the already-loaded `MicroscopyImage` so the
+    caller does not need to re-open the ND2 file for the metadata step.
+    """
+    return {
+        "sizes": dict(image.sizes),
+        "channels": [_channel_to_dict(channel) for channel in image.channels],
+        "dimensions": _dimension_names(image.dimensions),
+    }
+
+
+def _channel_to_dict(channel: Channel) -> dict[str, Any]:
+    color = channel.color.hex_code if channel.color is not None else None
+    return {
+        "name": channel.name,
+        "excitation_nm": channel.excitation_nm,
+        "emission_nm": channel.emission_nm,
+        "color": color,
+    }
+
+
+def _dimension_names(dimensions: DimensionFlags) -> list[str]:
+    """Serialize a `DimensionFlags` IntFlag as a list of member names."""
+    return [flag.name for flag in DimensionFlags if flag in dimensions and flag.name]