From 82cac5e2ae950de864611404443cfcb8db49cccc Mon Sep 17 00:00:00 2001 From: lanery Date: Fri, 8 May 2026 15:34:06 -0700 Subject: [PATCH 01/27] Epson V700 Scanner: Replace deprecated skimage.io with imageio.v3 skimage.io plugin infrastructure is deprecated since 0.25 and will be removed in 0.27. Use imageio.v3 directly for JPEG read/write. Co-authored-by: Cursor --- .../epson_v700_scanner/image_processing.py | 3 ++- .../epson_v700_scanner/test_image_processing.py | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index bffa601c..23805b4d 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any +import imageio.v3 as iio import numpy as np import skimage as ski import tifffile @@ -90,7 +91,7 @@ def export_jpg(self) -> Path: img = self._resize(img) jpg_path = self.path.parent / f"{self.path.stem}.jpg" - ski.io.imsave(str(jpg_path), img, quality=JPEG_QUALITY) + iio.imwrite(jpg_path, img, quality=JPEG_QUALITY) return jpg_path def parse_metadata(self) -> dict[str, Any]: diff --git a/lambda/tests/epson_v700_scanner/test_image_processing.py b/lambda/tests/epson_v700_scanner/test_image_processing.py index b76c8c76..e44fa372 100644 --- a/lambda/tests/epson_v700_scanner/test_image_processing.py +++ b/lambda/tests/epson_v700_scanner/test_image_processing.py @@ -3,9 +3,9 @@ from __future__ import annotations from pathlib import Path +import imageio.v3 as iio import numpy as np import pytest -import skimage as ski import tifffile from data_hub_lambda.epson_v700_scanner.image_processing import ( @@ -33,7 +33,7 @@ def test_produces_valid_jpeg(self, tmp_path: Path) -> None: assert jpg_path.exists() assert jpg_path.suffix == ".jpg" - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert loaded.ndim == 3 assert loaded.shape[2] == 3 @@ -45,7 +45,7 @@ def test_resizes_large_image(self, tmp_path: Path) -> None: converter.load() jpg_path = converter.export_jpg() - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert max(loaded.shape[:2]) <= MAX_DIMENSION def test_does_not_upscale_small_image(self, tmp_path: Path) -> None: @@ -56,7 +56,7 @@ def test_does_not_upscale_small_image(self, tmp_path: Path) -> None: converter.load() jpg_path = converter.export_jpg() - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert loaded.shape[0] == 100 assert loaded.shape[1] == 150 @@ -68,7 +68,7 @@ def test_converts_grayscale_to_rgb(self, tmp_path: Path) -> None: converter.load() jpg_path = converter.export_jpg() - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert loaded.ndim == 3 assert loaded.shape[2] == 3 @@ -80,7 +80,7 @@ def test_converts_rgba_to_rgb(self, tmp_path: Path) -> None: converter.load() jpg_path = converter.export_jpg() - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert loaded.ndim == 3 assert loaded.shape[2] == 3 @@ -92,7 +92,7 @@ def test_handles_16bit_input(self, tmp_path: Path) -> None: converter.load() jpg_path = converter.export_jpg() - loaded = ski.io.imread(str(jpg_path)) + loaded = iio.imread(jpg_path) assert loaded.dtype == np.uint8 From 2d16f8c0b95789d6e8ec65f41a7c0df40cb77f0c Mon Sep 17 00:00:00 2001 From: lanery Date: Fri, 8 May 2026 18:36:45 -0700 Subject: [PATCH 02/27] Epson V700 Scanner: Add plate detection and rename TIFFToJPEGConverter to TiffProcessor Detect gold 3D-printed frames around agar plates using HSV thresholding, emit plate_count/plate_boxes in metadata, and draw green overlay rectangles on the JPEG preview. Remove integration tests that relied on example JPG files not tracked in the repo. Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 4 +- .../epson_v700_scanner/image_processing.py | 128 ++++++++++++++++-- .../epson_v700_scanner/process_file.py | 4 +- .../test_image_processing.py | 93 +++++++++++-- .../epson_v700_scanner/test_process_file.py | 14 +- 5 files changed, 210 insertions(+), 33 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 2177a2ea..94ccd9e5 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -110,9 +110,9 @@ def epson_scanner(file: Path, output_dir: Path | None) -> None: Resizes the high-resolution scan to a web-friendly JPEG preview and extracts TIFF metadata. """ - from data_hub_lambda.epson_v700_scanner.image_processing import TIFFToJPEGConverter + from data_hub_lambda.epson_v700_scanner.image_processing import TiffProcessor - converter = TIFFToJPEGConverter(file) + converter = TiffProcessor(file) converter.load() jpg_path = converter.export_jpg() diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 23805b4d..2c4610e8 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -17,6 +17,22 @@ _TIFF_SUFFIXES = {".tif", ".tiff"} +PlateBox = tuple[int, int, int, int] + +_GOLD_HUE_LOW = 0.06 +_GOLD_HUE_HIGH = 0.18 +_GOLD_SAT_MIN = 0.25 +_GOLD_VAL_MIN = 0.35 + +_MIN_AREA_FRACTION = 0.05 +_MIN_EXTENT = 0.85 + +_DETECTION_DOWNSAMPLE = 4 +_CLOSING_RADIUS = 5 + +_OVERLAY_COLOR: tuple[int, int, int] = (0, 255, 0) +_OVERLAY_THICKNESS = 6 + # PhotometricInterpretation values: 2 = RGB, others (0, 1, 3) are grayscale or # palette and are treated as B&W for our display purposes. _PHOTOMETRIC_RGB = 2 @@ -64,12 +80,13 @@ def _derive_color_mode(samples_per_pixel: Any, photometric_interpretation: Any) } -class TIFFToJPEGConverter: - """Converts high-resolution TIFF scans to resized JPEG images.""" +class TiffProcessor: + """Processes high-resolution TIFF scans from the Epson V700 flatbed scanner.""" def __init__(self, path: Path) -> None: self.path = path self._intensities: NDArray[Any] | None = None + self.plate_boxes: list[PlateBox] = [] def load(self) -> None: if not self.path.exists(): @@ -86,8 +103,17 @@ def intensities(self) -> NDArray[Any]: return self._intensities def export_jpg(self) -> Path: - """Resize the loaded TIFF and write a JPEG next to the source file.""" + """Detect plates, draw overlays, resize, and write a JPEG. + + If no gold frames are detected the full image is exported as-is + (the pre-detection fallback behaviour). + """ img = self._to_rgb_uint8(self.intensities) + self.plate_boxes = self.detect_plates(img) + + if self.plate_boxes: + img = self._draw_plate_overlays(img, self.plate_boxes) + img = self._resize(img) jpg_path = self.path.parent / f"{self.path.stem}.jpg" @@ -97,13 +123,13 @@ def export_jpg(self) -> Path: def parse_metadata(self) -> dict[str, Any]: """Extract TIFF tags as a flat string-keyed dict. - In addition to the raw TIFF tags, this also emits two derived - scalar fields used by the web UI for filtering and display: + Derived fields: - - ``dpi``: integer DPI computed from ``XResolution`` (a (numerator, - denominator) rational). For Epson V700 scans this is 300 or 600. - - ``color_mode``: ``"rgb"`` or ``"bw"``, inferred from - ``SamplesPerPixel`` (preferred) or ``PhotometricInterpretation``. + - ``dpi``: integer DPI from ``XResolution``. + - ``color_mode``: ``"rgb"`` or ``"bw"``. + - ``plate_count``: number of plates detected by :meth:`export_jpg`. + - ``plate_boxes``: list of ``[min_row, min_col, max_row, max_col]`` + bounding boxes in original-image coordinates. """ metadata: dict[str, Any] = {} with tifffile.TiffFile(self.path) as tif: @@ -130,8 +156,92 @@ def parse_metadata(self) -> dict[str, Any]: if color_mode is not None: metadata["color_mode"] = color_mode + metadata["plate_count"] = len(self.plate_boxes) + metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] + return metadata + # ------------------------------------------------------------------ + # Plate detection + # ------------------------------------------------------------------ + + @staticmethod + def detect_plates(img: NDArray[np.uint8]) -> list[PlateBox]: + """Detect agar plates inside gold 3D-printed frames. + + Runs detection on a downsampled copy for speed, then scales + bounding boxes back to original coordinates. Returns + ``(min_row, min_col, max_row, max_col)`` sorted left-to-right. + """ + h_orig, w_orig = img.shape[:2] + s = _DETECTION_DOWNSAMPLE + small: NDArray[np.uint8] = img[::s, ::s] + + hsv = ski.color.rgb2hsv(small) + + gold_mask: NDArray[np.bool_] = ( + (hsv[:, :, 0] >= _GOLD_HUE_LOW) + & (hsv[:, :, 0] <= _GOLD_HUE_HIGH) + & (hsv[:, :, 1] >= _GOLD_SAT_MIN) + & (hsv[:, :, 2] >= _GOLD_VAL_MIN) + ) + + selem = ski.morphology.disk(_CLOSING_RADIUS) + gold_mask = ski.morphology.closing(gold_mask, selem) + + inverted = ~gold_mask + labels = ski.measure.label(inverted) + regions = ski.measure.regionprops(labels) + + h_small, w_small = small.shape[:2] + min_area = h_small * w_small * _MIN_AREA_FRACTION + + boxes: list[PlateBox] = [] + for region in regions: + if region.area < min_area: + continue + if region.extent < _MIN_EXTENT: + continue + + min_row, min_col, max_row, max_col = region.bbox + touches_border = ( + min_row == 0 or min_col == 0 or max_row == h_small or max_col == w_small + ) + if touches_border: + continue + + boxes.append( + ( + min(min_row * s, h_orig), + min(min_col * s, w_orig), + min(max_row * s, h_orig), + min(max_col * s, w_orig), + ) + ) + + boxes.sort(key=lambda b: b[1]) + return boxes + + @staticmethod + def _draw_plate_overlays( + img: NDArray[np.uint8], + boxes: list[PlateBox], + ) -> NDArray[np.uint8]: + """Draw coloured rectangle outlines on a copy of the image.""" + out = img.copy() + h, w = out.shape[:2] + for min_row, min_col, max_row, max_col in boxes: + for offset in range(_OVERLAY_THICKNESS): + r0 = max(min_row - offset, 0) + c0 = max(min_col - offset, 0) + r1 = min(max_row + offset, h - 1) + c1 = min(max_col + offset, w - 1) + rr, cc = ski.draw.rectangle_perimeter( + start=(r0, c0), end=(r1, c1), shape=out.shape[:2] + ) + out[rr, cc] = _OVERLAY_COLOR + return out + # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index ba157e67..505b6337 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -2,7 +2,7 @@ import logging from data_hub_lambda.api_client import get_client -from data_hub_lambda.epson_v700_scanner.image_processing import TIFFToJPEGConverter +from data_hub_lambda.epson_v700_scanner.image_processing import TiffProcessor from data_hub_shared import s3_utils from data_hub_shared.config import config from data_hub_shared.enums import Instrument @@ -48,7 +48,7 @@ def process_file(run_id: str, filename: str) -> None: s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) - converter = TIFFToJPEGConverter(local_file_path) + converter = TiffProcessor(local_file_path) converter.load() jpg_file_path = converter.export_jpg() diff --git a/lambda/tests/epson_v700_scanner/test_image_processing.py b/lambda/tests/epson_v700_scanner/test_image_processing.py index e44fa372..67d58d74 100644 --- a/lambda/tests/epson_v700_scanner/test_image_processing.py +++ b/lambda/tests/epson_v700_scanner/test_image_processing.py @@ -10,11 +10,13 @@ from data_hub_lambda.epson_v700_scanner.image_processing import ( MAX_DIMENSION, - TIFFToJPEGConverter, + TiffProcessor, _derive_color_mode, _derive_dpi, ) +_GOLD_RGB = np.array([200, 170, 40], dtype=np.uint8) + def _write_tiff(path: Path, img: np.ndarray) -> Path: # type: ignore[type-arg] tifffile.imwrite(str(path), img) @@ -26,7 +28,7 @@ def test_produces_valid_jpeg(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (200, 300, 3), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "scan.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -41,7 +43,7 @@ def test_resizes_large_image(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (4000, 6000, 3), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "big.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -52,7 +54,7 @@ def test_does_not_upscale_small_image(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (100, 150, 3), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "small.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -64,7 +66,7 @@ def test_converts_grayscale_to_rgb(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (200, 300), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "gray.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -76,7 +78,7 @@ def test_converts_rgba_to_rgb(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (200, 300, 4), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "rgba.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -88,7 +90,7 @@ def test_handles_16bit_input(self, tmp_path: Path) -> None: img = np.random.randint(0, 65535, (200, 300, 3), dtype=np.uint16) tif_path = _write_tiff(tmp_path / "16bit.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() jpg_path = converter.export_jpg() @@ -101,7 +103,7 @@ def test_returns_original_dimensions(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "scan.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() metadata = converter.parse_metadata() @@ -112,7 +114,7 @@ def test_returns_standard_tiff_tags(self, tmp_path: Path) -> None: img = np.random.randint(0, 255, (100, 200, 3), dtype=np.uint8) tif_path = _write_tiff(tmp_path / "scan.tif", img) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() metadata = converter.parse_metadata() @@ -166,7 +168,7 @@ def test_emits_dpi_and_rgb_color_mode(self, tmp_path: Path) -> None: tif_path = tmp_path / "scan.tif" tifffile.imwrite(str(tif_path), img, resolution=(300, 300)) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() metadata = converter.parse_metadata() @@ -178,7 +180,7 @@ def test_emits_bw_color_mode_for_grayscale(self, tmp_path: Path) -> None: tif_path = tmp_path / "gray.tif" tifffile.imwrite(str(tif_path), img, resolution=(600, 600)) - converter = TIFFToJPEGConverter(tif_path) + converter = TiffProcessor(tif_path) converter.load() metadata = converter.parse_metadata() @@ -188,13 +190,78 @@ def test_emits_bw_color_mode_for_grayscale(self, tmp_path: Path) -> None: class TestValidation: def test_rejects_missing_file(self, tmp_path: Path) -> None: - converter = TIFFToJPEGConverter(tmp_path / "nonexistent.tif") + converter = TiffProcessor(tmp_path / "nonexistent.tif") with pytest.raises(FileNotFoundError): converter.load() def test_rejects_non_tiff_extension(self, tmp_path: Path) -> None: path = tmp_path / "scan.png" path.write_bytes(b"fake") - converter = TIFFToJPEGConverter(path) + converter = TiffProcessor(path) with pytest.raises(ValueError, match="Expected TIFF file"): converter.load() + + +# ------------------------------------------------------------------ +# Plate detection +# ------------------------------------------------------------------ + + +def _make_gold_frame( + canvas_h: int, + canvas_w: int, + top: int, + left: int, + bottom: int, + right: int, + thickness: int = 20, +) -> np.ndarray: # type: ignore[type-arg] + """Paint a gold rectangular frame on a black canvas and return it.""" + img = np.zeros((canvas_h, canvas_w, 3), dtype=np.uint8) + img[top : top + thickness, left:right] = _GOLD_RGB + img[bottom - thickness : bottom, left:right] = _GOLD_RGB + img[top:bottom, left : left + thickness] = _GOLD_RGB + img[top:bottom, right - thickness : right] = _GOLD_RGB + return img + + +class TestDetectPlates: + def test_single_gold_frame(self) -> None: + img = _make_gold_frame(500, 400, top=50, left=50, bottom=350, right=350) + boxes = TiffProcessor.detect_plates(img) + + assert len(boxes) == 1 + min_row, min_col, max_row, max_col = boxes[0] + assert 60 < min_row < 80 + assert 60 < min_col < 80 + assert 320 < max_row < 340 + assert 320 < max_col < 340 + + def test_two_gold_frames_sorted_left_to_right(self) -> None: + img = np.zeros((500, 800, 3), dtype=np.uint8) + frame1 = _make_gold_frame(500, 800, top=50, left=50, bottom=350, right=300) + frame2 = _make_gold_frame(500, 800, top=50, left=400, bottom=350, right=700) + img = np.maximum(img, np.maximum(frame1, frame2)) + + boxes = TiffProcessor.detect_plates(img) + + assert len(boxes) == 2 + assert boxes[0][1] < boxes[1][1] + + def test_no_gold_returns_empty(self) -> None: + img = np.random.randint(0, 50, (400, 400, 3), dtype=np.uint8) + boxes = TiffProcessor.detect_plates(img) + assert boxes == [] + + def test_metadata_includes_plate_count(self, tmp_path: Path) -> None: + img = _make_gold_frame(500, 400, top=50, left=50, bottom=350, right=350) + tif_path = tmp_path / "framed.tif" + tifffile.imwrite(str(tif_path), img) + + proc = TiffProcessor(tif_path) + proc.load() + proc.export_jpg() + metadata = proc.parse_metadata() + + assert metadata["plate_count"] == 1 + assert len(metadata["plate_boxes"]) == 1 diff --git a/lambda/tests/epson_v700_scanner/test_process_file.py b/lambda/tests/epson_v700_scanner/test_process_file.py index cd08522d..ae07835e 100644 --- a/lambda/tests/epson_v700_scanner/test_process_file.py +++ b/lambda/tests/epson_v700_scanner/test_process_file.py @@ -1,7 +1,7 @@ """Unit tests for `epson_v700_scanner.process_file`. These verify the orchestration logic by mocking S3 I/O, the API client, -and the TIFFToJPEGConverter. +and the TiffProcessor. """ from __future__ import annotations @@ -69,7 +69,7 @@ def patched_jpg_path(tmp_path: Path) -> Path: @pytest.fixture def patched_converter(patched_jpg_path: Path) -> MagicMock: - """A stand-in TIFFToJPEGConverter whose export_jpg returns a real tmp file.""" + """A stand-in TiffProcessor whose export_jpg returns a real tmp file.""" converter = MagicMock() converter.load.return_value = None converter.export_jpg.return_value = patched_jpg_path @@ -97,7 +97,7 @@ def test_creates_run_and_files( patch(f"{_PATCH_PREFIX}.get_client", return_value=client), patch(f"{_PATCH_PREFIX}.s3_utils") as s3_mock, patch( - f"{_PATCH_PREFIX}.TIFFToJPEGConverter", + f"{_PATCH_PREFIX}.TiffProcessor", return_value=patched_converter, ), ): @@ -120,7 +120,7 @@ def test_uploads_jpg_and_registers_processed_file( patch(f"{_PATCH_PREFIX}.get_client", return_value=client), patch(f"{_PATCH_PREFIX}.s3_utils"), patch( - f"{_PATCH_PREFIX}.TIFFToJPEGConverter", + f"{_PATCH_PREFIX}.TiffProcessor", return_value=patched_converter, ), ): @@ -144,7 +144,7 @@ def test_stores_metadata_on_run( patch(f"{_PATCH_PREFIX}.get_client", return_value=client), patch(f"{_PATCH_PREFIX}.s3_utils"), patch( - f"{_PATCH_PREFIX}.TIFFToJPEGConverter", + f"{_PATCH_PREFIX}.TiffProcessor", return_value=patched_converter, ), ): @@ -166,7 +166,7 @@ def test_marks_raw_file_completed( patch(f"{_PATCH_PREFIX}.get_client", return_value=client), patch(f"{_PATCH_PREFIX}.s3_utils"), patch( - f"{_PATCH_PREFIX}.TIFFToJPEGConverter", + f"{_PATCH_PREFIX}.TiffProcessor", return_value=patched_converter, ), ): @@ -193,7 +193,7 @@ def test_marks_raw_file_failed_on_exception(self) -> None: patch(f"{_PATCH_PREFIX}.get_client", return_value=client), patch(f"{_PATCH_PREFIX}.s3_utils"), patch( - f"{_PATCH_PREFIX}.TIFFToJPEGConverter", + f"{_PATCH_PREFIX}.TiffProcessor", return_value=failing_converter, ), ): From 0015ec56cd4468330b349567139265ccac426694 Mon Sep 17 00:00:00 2001 From: lanery Date: Fri, 8 May 2026 18:51:21 -0700 Subject: [PATCH 03/27] Epson V700 Scanner: Review fixes for plate detection pipeline Make plate_boxes None-able so parse_metadata() omits plate fields when detection hasn't run (instead of silently returning zeros), simplify overlay drawing to fill-and-restore (drops ski.draw loop), rename converter -> processor, and mark PlateBox as private. Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 12 +++--- .../epson_v700_scanner/image_processing.py | 37 ++++++++++--------- .../epson_v700_scanner/process_file.py | 8 ++-- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 94ccd9e5..67b941ee 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -107,14 +107,14 @@ def hina(file: Path, output_dir: Path | None) -> None: def epson_scanner(file: Path, output_dir: Path | None) -> None: """Process an Epson V700 Scanner TIFF file. - Resizes the high-resolution scan to a web-friendly JPEG preview and - extracts TIFF metadata. + Detects agar plates inside gold frames, draws bounding-box overlays, + resizes to a web-friendly JPEG preview, and extracts TIFF metadata. """ from data_hub_lambda.epson_v700_scanner.image_processing import TiffProcessor - converter = TiffProcessor(file) - converter.load() - jpg_path = converter.export_jpg() + processor = TiffProcessor(file) + processor.load() + jpg_path = processor.export_jpg() if output_dir is not None: output_dir.mkdir(parents=True, exist_ok=True) @@ -124,7 +124,7 @@ def epson_scanner(file: Path, output_dir: Path | None) -> None: click.echo(f"Exported JPG: {jpg_path}") - metadata = converter.parse_metadata() + metadata = processor.parse_metadata() click.echo(json.dumps(metadata, indent=2)) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 2c4610e8..216af903 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -17,7 +17,7 @@ _TIFF_SUFFIXES = {".tif", ".tiff"} -PlateBox = tuple[int, int, int, int] +_PlateBox = tuple[int, int, int, int] _GOLD_HUE_LOW = 0.06 _GOLD_HUE_HIGH = 0.18 @@ -86,7 +86,7 @@ class TiffProcessor: def __init__(self, path: Path) -> None: self.path = path self._intensities: NDArray[Any] | None = None - self.plate_boxes: list[PlateBox] = [] + self.plate_boxes: list[_PlateBox] | None = None def load(self) -> None: if not self.path.exists(): @@ -127,9 +127,11 @@ def parse_metadata(self) -> dict[str, Any]: - ``dpi``: integer DPI from ``XResolution``. - ``color_mode``: ``"rgb"`` or ``"bw"``. - - ``plate_count``: number of plates detected by :meth:`export_jpg`. + - ``plate_count``: number of detected plates (only present after + :meth:`export_jpg` has been called). - ``plate_boxes``: list of ``[min_row, min_col, max_row, max_col]`` - bounding boxes in original-image coordinates. + bounding boxes in original-image coordinates (only present after + :meth:`export_jpg` has been called). """ metadata: dict[str, Any] = {} with tifffile.TiffFile(self.path) as tif: @@ -156,8 +158,9 @@ def parse_metadata(self) -> dict[str, Any]: if color_mode is not None: metadata["color_mode"] = color_mode - metadata["plate_count"] = len(self.plate_boxes) - metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] + if self.plate_boxes is not None: + metadata["plate_count"] = len(self.plate_boxes) + metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] return metadata @@ -166,7 +169,7 @@ def parse_metadata(self) -> dict[str, Any]: # ------------------------------------------------------------------ @staticmethod - def detect_plates(img: NDArray[np.uint8]) -> list[PlateBox]: + def detect_plates(img: NDArray[np.uint8]) -> list[_PlateBox]: """Detect agar plates inside gold 3D-printed frames. Runs detection on a downsampled copy for speed, then scales @@ -196,7 +199,7 @@ def detect_plates(img: NDArray[np.uint8]) -> list[PlateBox]: h_small, w_small = small.shape[:2] min_area = h_small * w_small * _MIN_AREA_FRACTION - boxes: list[PlateBox] = [] + boxes: list[_PlateBox] = [] for region in regions: if region.area < min_area: continue @@ -225,21 +228,19 @@ def detect_plates(img: NDArray[np.uint8]) -> list[PlateBox]: @staticmethod def _draw_plate_overlays( img: NDArray[np.uint8], - boxes: list[PlateBox], + boxes: list[_PlateBox], ) -> NDArray[np.uint8]: """Draw coloured rectangle outlines on a copy of the image.""" out = img.copy() h, w = out.shape[:2] + t = _OVERLAY_THICKNESS for min_row, min_col, max_row, max_col in boxes: - for offset in range(_OVERLAY_THICKNESS): - r0 = max(min_row - offset, 0) - c0 = max(min_col - offset, 0) - r1 = min(max_row + offset, h - 1) - c1 = min(max_col + offset, w - 1) - rr, cc = ski.draw.rectangle_perimeter( - start=(r0, c0), end=(r1, c1), shape=out.shape[:2] - ) - out[rr, cc] = _OVERLAY_COLOR + r0 = max(min_row - t, 0) + c0 = max(min_col - t, 0) + r1 = min(max_row + t, h) + c1 = min(max_col + t, w) + out[r0:r1, c0:c1] = _OVERLAY_COLOR + out[min_row:max_row, min_col:max_col] = img[min_row:max_row, min_col:max_col] return out # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 505b6337..18998818 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -48,9 +48,9 @@ def process_file(run_id: str, filename: str) -> None: s3_utils.download_file(f"s3://{s3_bucket}/{s3_key}", local_file_path) logger.info("Downloaded %s to %s", filename, local_file_path) - converter = TiffProcessor(local_file_path) - converter.load() - jpg_file_path = converter.export_jpg() + processor = TiffProcessor(local_file_path) + processor.load() + jpg_file_path = processor.export_jpg() processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET jpg_s3_key = f"{INSTRUMENT_ID}/{run_id}/{jpg_file_path.name}" @@ -71,7 +71,7 @@ def process_file(run_id: str, filename: str) -> None: content_type="image/jpeg", ) - metadata = converter.parse_metadata() + metadata = processor.parse_metadata() logger.info("Parsed metadata: %s", metadata) client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) From 88e4a74771f68cfd02eda7ece172a24403d81f73 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 10:28:56 -0700 Subject: [PATCH 04/27] Make detect_plates an instance method --- .../epson_v700_scanner/image_processing.py | 6 +++--- lambda/tests/epson_v700_scanner/test_image_processing.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 216af903..aa20c0fb 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -109,7 +109,7 @@ def export_jpg(self) -> Path: (the pre-detection fallback behaviour). """ img = self._to_rgb_uint8(self.intensities) - self.plate_boxes = self.detect_plates(img) + self.detect_plates(img) if self.plate_boxes: img = self._draw_plate_overlays(img, self.plate_boxes) @@ -168,8 +168,7 @@ def parse_metadata(self) -> dict[str, Any]: # Plate detection # ------------------------------------------------------------------ - @staticmethod - def detect_plates(img: NDArray[np.uint8]) -> list[_PlateBox]: + def detect_plates(self, img: NDArray[np.uint8]) -> list[_PlateBox]: """Detect agar plates inside gold 3D-printed frames. Runs detection on a downsampled copy for speed, then scales @@ -223,6 +222,7 @@ def detect_plates(img: NDArray[np.uint8]) -> list[_PlateBox]: ) boxes.sort(key=lambda b: b[1]) + self.plate_boxes = boxes return boxes @staticmethod diff --git a/lambda/tests/epson_v700_scanner/test_image_processing.py b/lambda/tests/epson_v700_scanner/test_image_processing.py index 67d58d74..37f278f8 100644 --- a/lambda/tests/epson_v700_scanner/test_image_processing.py +++ b/lambda/tests/epson_v700_scanner/test_image_processing.py @@ -228,7 +228,8 @@ def _make_gold_frame( class TestDetectPlates: def test_single_gold_frame(self) -> None: img = _make_gold_frame(500, 400, top=50, left=50, bottom=350, right=350) - boxes = TiffProcessor.detect_plates(img) + proc = TiffProcessor(Path("dummy.tif")) + boxes = proc.detect_plates(img) assert len(boxes) == 1 min_row, min_col, max_row, max_col = boxes[0] @@ -243,14 +244,16 @@ def test_two_gold_frames_sorted_left_to_right(self) -> None: frame2 = _make_gold_frame(500, 800, top=50, left=400, bottom=350, right=700) img = np.maximum(img, np.maximum(frame1, frame2)) - boxes = TiffProcessor.detect_plates(img) + proc = TiffProcessor(Path("dummy.tif")) + boxes = proc.detect_plates(img) assert len(boxes) == 2 assert boxes[0][1] < boxes[1][1] def test_no_gold_returns_empty(self) -> None: img = np.random.randint(0, 50, (400, 400, 3), dtype=np.uint8) - boxes = TiffProcessor.detect_plates(img) + proc = TiffProcessor(Path("dummy.tif")) + boxes = proc.detect_plates(img) assert boxes == [] def test_metadata_includes_plate_count(self, tmp_path: Path) -> None: From 9221dd321063a3c3161ab3ba3a3b307bfff5a4e7 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 10:39:45 -0700 Subject: [PATCH 05/27] Epson V700 Scanner: Clean up TiffProcessor API and edge cases - Make detect_plates side-effect-only (results in self.plate_boxes) - Guard against redundant detection in export_jpg - Fix _to_rgb_uint8 redundant float round-trip for RGBA inputs - Use round() instead of int() in _derive_dpi for robustness - Add logging in detect_plates and warning in parse_metadata - Document non-overlapping assumption in _draw_plate_overlays Co-authored-by: Cursor --- .../epson_v700_scanner/image_processing.py | 31 +++++++++++++------ .../test_image_processing.py | 18 ++++++----- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index aa20c0fb..fa085f6b 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -47,7 +47,7 @@ def _derive_dpi(x_resolution: Any) -> int | None: return None if denominator == 0: return None - return int(numerator / denominator) + return round(numerator / denominator) def _derive_color_mode(samples_per_pixel: Any, photometric_interpretation: Any) -> str | None: @@ -109,7 +109,8 @@ def export_jpg(self) -> Path: (the pre-detection fallback behaviour). """ img = self._to_rgb_uint8(self.intensities) - self.detect_plates(img) + if self.plate_boxes is None: + self.detect_plates(img) if self.plate_boxes: img = self._draw_plate_overlays(img, self.plate_boxes) @@ -161,6 +162,11 @@ def parse_metadata(self) -> dict[str, Any]: if self.plate_boxes is not None: metadata["plate_count"] = len(self.plate_boxes) metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] + else: + logger.warning( + "plate_boxes is None; call export_jpg() before" + " parse_metadata() to include plate data" + ) return metadata @@ -168,12 +174,13 @@ def parse_metadata(self) -> dict[str, Any]: # Plate detection # ------------------------------------------------------------------ - def detect_plates(self, img: NDArray[np.uint8]) -> list[_PlateBox]: + def detect_plates(self, img: NDArray[np.uint8]) -> None: """Detect agar plates inside gold 3D-printed frames. Runs detection on a downsampled copy for speed, then scales - bounding boxes back to original coordinates. Returns - ``(min_row, min_col, max_row, max_col)`` sorted left-to-right. + bounding boxes back to original coordinates. Results are stored + in ``self.plate_boxes`` as ``(min_row, min_col, max_row, max_col)`` + tuples sorted left-to-right. """ h_orig, w_orig = img.shape[:2] s = _DETECTION_DOWNSAMPLE @@ -223,14 +230,18 @@ def detect_plates(self, img: NDArray[np.uint8]) -> list[_PlateBox]: boxes.sort(key=lambda b: b[1]) self.plate_boxes = boxes - return boxes + logger.debug("Detected %d plate(s): %s", len(boxes), boxes) @staticmethod def _draw_plate_overlays( img: NDArray[np.uint8], boxes: list[_PlateBox], ) -> NDArray[np.uint8]: - """Draw coloured rectangle outlines on a copy of the image.""" + """Draw coloured rectangle outlines on a copy of the image. + + Assumes non-overlapping boxes; overlapping boxes would cause + later interior restores to overwrite earlier overlay lines. + """ out = img.copy() h, w = out.shape[:2] t = _OVERLAY_THICKNESS @@ -250,14 +261,14 @@ def _draw_plate_overlays( @staticmethod def _to_rgb_uint8(img: NDArray[Any]) -> NDArray[np.uint8]: """Normalize to 8-bit RGB regardless of input dtype/channels.""" + if img.ndim == 3 and img.shape[2] == 4: + img = ski.color.rgba2rgb(img) + if img.dtype != np.uint8: img = ski.util.img_as_ubyte(ski.exposure.rescale_intensity(img)) if img.ndim == 2: img = ski.color.gray2rgb(img) - elif img.ndim == 3 and img.shape[2] == 4: - img = ski.color.rgba2rgb(img) - img = ski.util.img_as_ubyte(img) return img # type: ignore[return-value] diff --git a/lambda/tests/epson_v700_scanner/test_image_processing.py b/lambda/tests/epson_v700_scanner/test_image_processing.py index 37f278f8..834959e9 100644 --- a/lambda/tests/epson_v700_scanner/test_image_processing.py +++ b/lambda/tests/epson_v700_scanner/test_image_processing.py @@ -229,10 +229,11 @@ class TestDetectPlates: def test_single_gold_frame(self) -> None: img = _make_gold_frame(500, 400, top=50, left=50, bottom=350, right=350) proc = TiffProcessor(Path("dummy.tif")) - boxes = proc.detect_plates(img) + proc.detect_plates(img) - assert len(boxes) == 1 - min_row, min_col, max_row, max_col = boxes[0] + assert proc.plate_boxes is not None + assert len(proc.plate_boxes) == 1 + min_row, min_col, max_row, max_col = proc.plate_boxes[0] assert 60 < min_row < 80 assert 60 < min_col < 80 assert 320 < max_row < 340 @@ -245,16 +246,17 @@ def test_two_gold_frames_sorted_left_to_right(self) -> None: img = np.maximum(img, np.maximum(frame1, frame2)) proc = TiffProcessor(Path("dummy.tif")) - boxes = proc.detect_plates(img) + proc.detect_plates(img) - assert len(boxes) == 2 - assert boxes[0][1] < boxes[1][1] + assert proc.plate_boxes is not None + assert len(proc.plate_boxes) == 2 + assert proc.plate_boxes[0][1] < proc.plate_boxes[1][1] def test_no_gold_returns_empty(self) -> None: img = np.random.randint(0, 50, (400, 400, 3), dtype=np.uint8) proc = TiffProcessor(Path("dummy.tif")) - boxes = proc.detect_plates(img) - assert boxes == [] + proc.detect_plates(img) + assert proc.plate_boxes == [] def test_metadata_includes_plate_count(self, tmp_path: Path) -> None: img = _make_gold_frame(500, 400, top=50, left=50, bottom=350, right=350) From b320b284d8c33c1f9a39059c5aace01a52d650da Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 11:45:31 -0700 Subject: [PATCH 06/27] Epson V700 Scanner: Add colony detection and phenotyping pipeline New colony_detection module that operates on individual plate crops: crop margin, optimize contrast, detect colony presence, Gaussian smooth, and Otsu threshold. Measures per-colony area, centroid, eccentricity, and equivalent diameter. Wired into the Lambda pipeline (auto-runs on each detected plate, results stored in run metadata) and the CLI (--detect-colonies flag on epson-scanner command). Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 23 +- .../epson_v700_scanner/colony_detection.py | 219 +++++++++++++++ .../epson_v700_scanner/image_processing.py | 14 + .../epson_v700_scanner/process_file.py | 13 + .../test_colony_detection.py | 260 ++++++++++++++++++ 5 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py create mode 100644 lambda/tests/epson_v700_scanner/test_colony_detection.py diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 67b941ee..e3a7dca6 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -104,7 +104,13 @@ def hina(file: Path, output_dir: Path | None) -> None: default=None, help="Directory for the exported JPG (default: same directory as FILE).", ) -def epson_scanner(file: Path, output_dir: Path | None) -> None: +@click.option( + "--detect-colonies", + is_flag=True, + default=False, + help="Run colony detection and phenotyping on each detected plate.", +) +def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> None: """Process an Epson V700 Scanner TIFF file. Detects agar plates inside gold frames, draws bounding-box overlays, @@ -127,6 +133,21 @@ def epson_scanner(file: Path, output_dir: Path | None) -> None: metadata = processor.parse_metadata() click.echo(json.dumps(metadata, indent=2)) + if detect_colonies: + from data_hub_lambda.epson_v700_scanner.colony_detection import ( + detect_colonies as run_colony_detection, + ) + + plate_crops = processor.crop_plates() + if not plate_crops: + click.echo("No plates detected — skipping colony detection.") + return + + for i, crop in enumerate(plate_crops): + result = run_colony_detection(crop) + click.echo(f"\nPlate {i + 1}:") + click.echo(json.dumps(result.summary(), indent=2)) + # --------------------------------------------------------------------------- # Azure Cielo qPCR diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py new file mode 100644 index 00000000..b663e3e7 --- /dev/null +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -0,0 +1,219 @@ +"""Colony detection and phenotyping from cropped agar-plate images. + +Operates on individual plate crops produced by +:class:`~data_hub_lambda.epson_v700_scanner.image_processing.TiffProcessor`. + +Pipeline +-------- +1. Crop ~10 % margin to remove plate edges / frame artefacts. +2. Optimise contrast (Euclidean distance from estimated background colour). +3. Decide whether colonies are present (contrast above noise floor). +4. Gaussian smooth. +5. Otsu threshold to produce a binary colony mask. +""" + +from __future__ import annotations +import logging +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import skimage as ski +from numpy.typing import NDArray + +logger = logging.getLogger(__name__) + +MARGIN_FRACTION = 0.10 + +_GAUSSIAN_SIGMA = 2.0 + +_CONTRAST_PRESENCE_THRESHOLD = 5.0 +"""Minimum 95th-percentile contrast value to declare colonies present.""" + + +@dataclass +class ColonyProperties: + """Measured properties for a single colony.""" + + label: int + area_px: int + centroid_row: float + centroid_col: float + bbox: tuple[int, int, int, int] + eccentricity: float + equivalent_diameter: float + + +@dataclass +class ColonyDetectionResult: + """Full result of the colony-detection pipeline.""" + + cropped: NDArray[Any] + contrast: NDArray[np.floating[Any]] + has_colonies: bool + mask: NDArray[np.bool_] + colonies: list[ColonyProperties] = field(default_factory=list) + + def summary(self) -> dict[str, Any]: + """Return a JSON-serialisable summary dict.""" + return { + "has_colonies": self.has_colonies, + "colony_count": len(self.colonies), + "colonies": [ + { + "label": c.label, + "area_px": c.area_px, + "centroid": [round(c.centroid_row, 1), round(c.centroid_col, 1)], + "bbox": list(c.bbox), + "eccentricity": round(c.eccentricity, 4), + "equivalent_diameter": round(c.equivalent_diameter, 2), + } + for c in self.colonies + ], + } + + +# ------------------------------------------------------------------ +# Pipeline steps +# ------------------------------------------------------------------ + + +def crop_margin(image: NDArray[Any], margin: float = MARGIN_FRACTION) -> NDArray[Any]: + """Remove a fractional margin from each edge of *image*. + + Args: + image: (H, W) or (H, W, C) array. + margin: Fraction of each dimension to remove per side. + + Returns: + Cropped view of the original array. + """ + h, w = image.shape[:2] + row_margin = int(h * margin) + col_margin = int(w * margin) + return image[row_margin : h - row_margin, col_margin : w - col_margin] + + +def optimize_colony_contrast(image: NDArray[Any]) -> NDArray[np.floating[Any]]: + """Convert to a single channel that maximises colony-background contrast. + + Per-pixel Euclidean distance from the estimated background colour + (median of non-zero pixels). Colonies of any colour appear bright + against a near-zero background. + + Args: + image: (H, W) or (H, W, C) array. + + Returns: + (H, W) float64 distance-from-background image. + """ + if image.ndim == 2: + return image.astype(np.float64) + + pixels = image.reshape(-1, image.shape[-1]) + nonzero = np.any(pixels > 0, axis=1) + background: NDArray[np.floating[Any]] + if nonzero.any(): + background = np.median(pixels[nonzero], axis=0) + else: + background = np.median(pixels, axis=0) + + diff = image.astype(np.float64) - background + distance: NDArray[np.floating[Any]] = np.sqrt(np.sum(diff**2, axis=-1)) + distance[~np.any(image > 0, axis=-1)] = 0.0 + return distance + + +def detect_colony_presence( + contrast: NDArray[np.floating[Any]], + threshold: float = _CONTRAST_PRESENCE_THRESHOLD, +) -> bool: + """Return ``True`` if there is enough contrast to indicate colonies. + + Uses the 95th percentile of the contrast image; plates without + colonies have near-uniform background. + """ + p95 = float(np.percentile(contrast, 95)) + logger.debug("Colony-presence p95 contrast = %.2f (threshold %.2f)", p95, threshold) + return p95 > threshold + + +def smooth( + contrast: NDArray[np.floating[Any]], + sigma: float = _GAUSSIAN_SIGMA, +) -> NDArray[np.floating[Any]]: + """Apply Gaussian smoothing.""" + return ski.filters.gaussian(contrast, sigma=sigma, preserve_range=True) # type: ignore[no-any-return] + + +def threshold_colonies(smoothed: NDArray[np.floating[Any]]) -> NDArray[np.bool_]: + """Otsu threshold to produce a binary colony mask.""" + thresh = ski.filters.threshold_otsu(smoothed) + mask: NDArray[np.bool_] = smoothed > thresh + return mask + + +def measure_colonies(mask: NDArray[np.bool_]) -> list[ColonyProperties]: + """Label connected components and extract per-colony measurements.""" + labels = ski.measure.label(mask) + regions = ski.measure.regionprops(labels) + + colonies: list[ColonyProperties] = [] + for region in regions: + colonies.append( + ColonyProperties( + label=int(region.label), + area_px=int(region.area), + centroid_row=float(region.centroid[0]), + centroid_col=float(region.centroid[1]), + bbox=tuple(region.bbox), # type: ignore[arg-type] + eccentricity=float(region.eccentricity), + equivalent_diameter=float(region.equivalent_diameter_area), + ) + ) + + colonies.sort(key=lambda c: c.area_px, reverse=True) + return colonies + + +# ------------------------------------------------------------------ +# Orchestrator +# ------------------------------------------------------------------ + + +def detect_colonies(plate_image: NDArray[Any]) -> ColonyDetectionResult: + """Run the full colony-detection pipeline on a single plate crop. + + Args: + plate_image: RGB uint8 plate image produced by + :class:`~data_hub_lambda.epson_v700_scanner.image_processing.TiffProcessor`. + + Returns: + A :class:`ColonyDetectionResult` with all intermediate arrays + and measured colony properties. + """ + cropped = crop_margin(plate_image) + contrast = optimize_colony_contrast(cropped) + has_colonies = detect_colony_presence(contrast) + + if not has_colonies: + logger.info("No colonies detected (low contrast).") + return ColonyDetectionResult( + cropped=cropped, + contrast=contrast, + has_colonies=False, + mask=np.zeros(contrast.shape, dtype=bool), + ) + + smoothed = smooth(contrast) + mask = threshold_colonies(smoothed) + colonies = measure_colonies(mask) + + logger.info("Detected %d colony/ies.", len(colonies)) + return ColonyDetectionResult( + cropped=cropped, + contrast=contrast, + has_colonies=True, + mask=mask, + colonies=colonies, + ) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index fa085f6b..71022497 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -254,6 +254,20 @@ def _draw_plate_overlays( out[min_row:max_row, min_col:max_col] = img[min_row:max_row, min_col:max_col] return out + def crop_plates(self) -> list[NDArray[np.uint8]]: + """Return RGB uint8 crops for each detected plate. + + Must be called after :meth:`export_jpg` (or :meth:`detect_plates`) + so that ``plate_boxes`` is populated. + """ + if self.plate_boxes is None: + raise RuntimeError("Call export_jpg() or detect_plates() first.") + img = self._to_rgb_uint8(self.intensities) + crops: list[NDArray[np.uint8]] = [] + for min_row, min_col, max_row, max_col in self.plate_boxes: + crops.append(img[min_row:max_row, min_col:max_col].copy()) + return crops + # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 18998818..7bfb3b2b 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -72,6 +72,19 @@ def process_file(run_id: str, filename: str) -> None: ) metadata = processor.parse_metadata() + + plate_crops = processor.crop_plates() + if plate_crops: + from data_hub_lambda.epson_v700_scanner.colony_detection import detect_colonies + + colony_results = [detect_colonies(crop).summary() for crop in plate_crops] + metadata["colony_detection"] = colony_results + logger.info( + "Colony detection complete for %d plate(s): %s", + len(plate_crops), + [r["colony_count"] for r in colony_results], + ) + logger.info("Parsed metadata: %s", metadata) client.update_run(INSTRUMENT_ID, run_id, metadata=metadata) diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py new file mode 100644 index 00000000..e5a4d707 --- /dev/null +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -0,0 +1,260 @@ +"""Unit tests for `epson_v700_scanner.colony_detection`.""" + +from __future__ import annotations + +import numpy as np + +from data_hub_lambda.epson_v700_scanner.colony_detection import ( + MARGIN_FRACTION, + ColonyDetectionResult, + ColonyProperties, + crop_margin, + detect_colonies, + detect_colony_presence, + measure_colonies, + optimize_colony_contrast, + smooth, + threshold_colonies, +) + + +def _uniform_plate(h: int = 400, w: int = 400, value: int = 120) -> np.ndarray: # type: ignore[type-arg] + """Create a uniform-colour plate image (no colonies).""" + return np.full((h, w, 3), value, dtype=np.uint8) + + +def _plate_with_colonies( + h: int = 400, + w: int = 400, + bg: int = 120, + colony_color: tuple[int, int, int] = (255, 255, 255), + n_colonies: int = 3, + colony_radius: int = 15, +) -> np.ndarray: # type: ignore[type-arg] + """Create a synthetic plate with circular colonies inside the margin.""" + img = np.full((h, w, 3), bg, dtype=np.uint8) + rng = np.random.RandomState(42) + margin = int(max(h, w) * MARGIN_FRACTION) + colony_radius + 5 + for _ in range(n_colonies): + cy = rng.randint(margin, h - margin) + cx = rng.randint(margin, w - margin) + rr, cc = _disk(cy, cx, colony_radius, h, w) + img[rr, cc] = colony_color + return img + + +def _disk(cy: int, cx: int, radius: int, h: int, w: int) -> tuple[np.ndarray, np.ndarray]: # type: ignore[type-arg] + """Return row, col arrays for a filled circle clipped to (h, w).""" + Y, X = np.ogrid[:h, :w] + mask = (Y - cy) ** 2 + (X - cx) ** 2 <= radius**2 + rows, cols = np.where(mask) + return rows, cols + + +# ------------------------------------------------------------------ +# crop_margin +# ------------------------------------------------------------------ + + +class TestCropMargin: + def test_default_removes_10pct(self) -> None: + img = np.zeros((200, 300, 3), dtype=np.uint8) + cropped = crop_margin(img) + expected_h = 200 - 2 * int(200 * MARGIN_FRACTION) + expected_w = 300 - 2 * int(300 * MARGIN_FRACTION) + assert cropped.shape == (expected_h, expected_w, 3) + + def test_grayscale(self) -> None: + img = np.zeros((100, 100), dtype=np.uint8) + cropped = crop_margin(img) + margin = int(100 * MARGIN_FRACTION) + assert cropped.shape == (100 - 2 * margin, 100 - 2 * margin) + + def test_custom_margin(self) -> None: + img = np.zeros((200, 200, 3), dtype=np.uint8) + cropped = crop_margin(img, margin=0.25) + assert cropped.shape == (100, 100, 3) + + def test_preserves_content(self) -> None: + img = np.arange(100 * 100, dtype=np.uint8).reshape(100, 100) + cropped = crop_margin(img, margin=0.10) + m = int(100 * 0.10) + np.testing.assert_array_equal(cropped, img[m : 100 - m, m : 100 - m]) + + +# ------------------------------------------------------------------ +# optimize_colony_contrast +# ------------------------------------------------------------------ + + +class TestOptimizeColonyContrast: + def test_grayscale_passthrough(self) -> None: + img = np.random.randint(0, 255, (50, 50), dtype=np.uint8) + result = optimize_colony_contrast(img) + assert result.shape == (50, 50) + assert result.dtype == np.float64 + + def test_uniform_rgb_gives_near_zero(self) -> None: + img = np.full((50, 50, 3), 128, dtype=np.uint8) + result = optimize_colony_contrast(img) + assert result.max() < 1.0 + + def test_colony_pixels_are_bright(self) -> None: + img = np.full((100, 100, 3), 80, dtype=np.uint8) + img[40:60, 40:60] = [220, 220, 220] + result = optimize_colony_contrast(img) + colony_region = result[40:60, 40:60] + bg_region = result[0:20, 0:20] + assert colony_region.mean() > bg_region.mean() + + def test_zero_pixels_stay_zero(self) -> None: + img = np.full((50, 50, 3), 100, dtype=np.uint8) + img[0:10, :] = 0 + result = optimize_colony_contrast(img) + np.testing.assert_array_equal(result[0:10, :], 0) + + +# ------------------------------------------------------------------ +# detect_colony_presence +# ------------------------------------------------------------------ + + +class TestDetectColonyPresence: + def test_uniform_returns_false(self) -> None: + contrast = np.full((100, 100), 1.0) + assert detect_colony_presence(contrast) is False + + def test_high_contrast_returns_true(self) -> None: + contrast = np.zeros((100, 100)) + contrast[30:70, 30:70] = 50.0 + assert detect_colony_presence(contrast) is True + + def test_custom_threshold(self) -> None: + contrast = np.full((100, 100), 3.0) + assert detect_colony_presence(contrast, threshold=2.0) is True + assert detect_colony_presence(contrast, threshold=4.0) is False + + +# ------------------------------------------------------------------ +# smooth +# ------------------------------------------------------------------ + + +class TestSmooth: + def test_output_shape_preserved(self) -> None: + img = np.random.rand(100, 100) + result = smooth(img) + assert result.shape == img.shape + + def test_smoothing_reduces_noise(self) -> None: + rng = np.random.RandomState(0) + img = rng.rand(200, 200) * 100 + result = smooth(img, sigma=3.0) + assert result.std() < img.std() + + +# ------------------------------------------------------------------ +# threshold_colonies +# ------------------------------------------------------------------ + + +class TestThresholdColonies: + def test_returns_bool_mask(self) -> None: + img = np.random.rand(100, 100) + mask = threshold_colonies(img) + assert mask.dtype == np.bool_ + assert mask.shape == img.shape + + def test_bimodal_separation(self) -> None: + img = np.zeros((100, 100), dtype=np.float64) + img[30:70, 30:70] = 100.0 + mask = threshold_colonies(img) + assert mask[50, 50] + assert not mask[0, 0] + + +# ------------------------------------------------------------------ +# measure_colonies +# ------------------------------------------------------------------ + + +class TestMeasureColonies: + def test_empty_mask_returns_empty(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + assert measure_colonies(mask) == [] + + def test_single_blob(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + mask[40:60, 40:60] = True + colonies = measure_colonies(mask) + assert len(colonies) == 1 + assert colonies[0].area_px == 20 * 20 + + def test_two_blobs_sorted_by_area(self) -> None: + mask = np.zeros((200, 200), dtype=bool) + mask[10:20, 10:20] = True # 100 px + mask[50:80, 50:80] = True # 900 px + colonies = measure_colonies(mask) + assert len(colonies) == 2 + assert colonies[0].area_px > colonies[1].area_px + + def test_colony_properties_populated(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + mask[40:60, 40:60] = True + colony = measure_colonies(mask)[0] + assert isinstance(colony, ColonyProperties) + assert colony.label >= 1 + assert colony.eccentricity >= 0.0 + assert colony.equivalent_diameter > 0.0 + assert len(colony.bbox) == 4 + + +# ------------------------------------------------------------------ +# detect_colonies (full pipeline) +# ------------------------------------------------------------------ + + +class TestDetectColonies: + def test_uniform_plate_no_colonies(self) -> None: + plate = _uniform_plate() + result = detect_colonies(plate) + assert isinstance(result, ColonyDetectionResult) + assert result.has_colonies is False + assert result.colonies == [] + assert result.mask.shape == result.contrast.shape + assert not result.mask.any() + + def test_plate_with_colonies_detected(self) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + assert result.has_colonies is True + assert len(result.colonies) > 0 + + def test_summary_schema(self) -> None: + plate = _plate_with_colonies(n_colonies=2) + result = detect_colonies(plate) + summary = result.summary() + assert "has_colonies" in summary + assert "colony_count" in summary + assert isinstance(summary["colonies"], list) + if summary["colonies"]: + c = summary["colonies"][0] + assert "label" in c + assert "area_px" in c + assert "centroid" in c + assert "bbox" in c + assert "eccentricity" in c + assert "equivalent_diameter" in c + + def test_cropped_smaller_than_input(self) -> None: + plate = _uniform_plate(400, 400) + result = detect_colonies(plate) + assert result.cropped.shape[0] < 400 + assert result.cropped.shape[1] < 400 + + def test_grayscale_input(self) -> None: + plate = np.full((200, 200), 128, dtype=np.uint8) + result = detect_colonies(plate) + assert isinstance(result, ColonyDetectionResult) From 8245899653c2771d03f024cf442beee32ac72bc0 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 11:55:24 -0700 Subject: [PATCH 07/27] Epson V700 Scanner: Add colony overlay visualization and CSV export Draw cyan contour outlines on per-plate overlay JPEGs and export a combined colony-properties CSV (via pandas). Wired into both the CLI (--detect-colonies now emits overlay JPEGs + CSV) and the Lambda pipeline (uploads overlays and CSV to S3 as processed files). Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 20 +++ .../epson_v700_scanner/colony_detection.py | 123 ++++++++++++++++ .../epson_v700_scanner/process_file.py | 56 ++++++- .../test_colony_detection.py | 139 ++++++++++++++++++ .../epson_v700_scanner/test_process_file.py | 1 + 5 files changed, 335 insertions(+), 4 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index e3a7dca6..4eb418ec 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -137,17 +137,37 @@ def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> from data_hub_lambda.epson_v700_scanner.colony_detection import ( detect_colonies as run_colony_detection, ) + from data_hub_lambda.epson_v700_scanner.colony_detection import ( + draw_colony_overlay, + export_colony_csv, + export_colony_overlay, + ) plate_crops = processor.crop_plates() if not plate_crops: click.echo("No plates detected — skipping colony detection.") return + dest_dir = output_dir or file.parent + dest_dir.mkdir(parents=True, exist_ok=True) + + dataframes = [] for i, crop in enumerate(plate_crops): result = run_colony_detection(crop) click.echo(f"\nPlate {i + 1}:") click.echo(json.dumps(result.summary(), indent=2)) + overlay = draw_colony_overlay(crop, result.mask) + overlay_path = dest_dir / f"{file.stem}_plate{i + 1}_colonies.jpg" + export_colony_overlay(overlay, overlay_path) + click.echo(f"Colony overlay: {overlay_path}") + + dataframes.append(result.to_dataframe(plate_index=i + 1)) + + csv_path = dest_dir / f"{file.stem}_colonies.csv" + export_colony_csv(dataframes, csv_path) + click.echo(f"\nColony CSV: {csv_path}") + # --------------------------------------------------------------------------- # Azure Cielo qPCR diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index b663e3e7..52d86f28 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -15,9 +15,12 @@ from __future__ import annotations import logging from dataclasses import dataclass, field +from pathlib import Path from typing import Any +import imageio.v3 as iio import numpy as np +import pandas as pd import skimage as ski from numpy.typing import NDArray @@ -30,6 +33,11 @@ _CONTRAST_PRESENCE_THRESHOLD = 5.0 """Minimum 95th-percentile contrast value to declare colonies present.""" +JPEG_QUALITY = 85 + +_CONTOUR_COLOR: tuple[int, int, int] = (0, 255, 255) +_CONTOUR_THICKNESS = 3 + @dataclass class ColonyProperties: @@ -72,6 +80,43 @@ def summary(self) -> dict[str, Any]: ], } + def to_dataframe(self, plate_index: int = 0) -> pd.DataFrame: + """Return a :class:`~pandas.DataFrame` with one row per colony.""" + if not self.colonies: + return pd.DataFrame( + columns=[ + "plate_index", + "label", + "area_px", + "centroid_row", + "centroid_col", + "bbox_min_row", + "bbox_min_col", + "bbox_max_row", + "bbox_max_col", + "eccentricity", + "equivalent_diameter", + ] + ) + rows = [] + for c in self.colonies: + rows.append( + { + "plate_index": plate_index, + "label": c.label, + "area_px": c.area_px, + "centroid_row": round(c.centroid_row, 1), + "centroid_col": round(c.centroid_col, 1), + "bbox_min_row": c.bbox[0], + "bbox_min_col": c.bbox[1], + "bbox_max_row": c.bbox[2], + "bbox_max_col": c.bbox[3], + "eccentricity": round(c.eccentricity, 4), + "equivalent_diameter": round(c.equivalent_diameter, 2), + } + ) + return pd.DataFrame(rows) + # ------------------------------------------------------------------ # Pipeline steps @@ -176,6 +221,84 @@ def measure_colonies(mask: NDArray[np.bool_]) -> list[ColonyProperties]: return colonies +# ------------------------------------------------------------------ +# Visualisation & export +# ------------------------------------------------------------------ + + +def draw_colony_overlay( + plate_image: NDArray[np.uint8], + mask: NDArray[np.bool_], + margin: float = MARGIN_FRACTION, +) -> NDArray[np.uint8]: + """Draw colony contour outlines on the plate image. + + The *mask* lives in the coordinate space of the margin-cropped image, + so contours are offset back to the original plate-image coordinates. + + Args: + plate_image: Original (H, W, 3) RGB uint8 plate crop. + mask: Binary colony mask in cropped coordinates. + margin: The same margin fraction used during detection. + + Returns: + Copy of *plate_image* with cyan contour outlines. + """ + out = plate_image.copy() + h, w = plate_image.shape[:2] + row_offset = int(h * margin) + col_offset = int(w * margin) + + contours = ski.measure.find_contours(mask.astype(float), level=0.5) + t = _CONTOUR_THICKNESS + for contour in contours: + for r_f, c_f in contour: + r = int(round(r_f)) + row_offset + c = int(round(c_f)) + col_offset + r0, r1 = max(r - t, 0), min(r + t + 1, h) + c0, c1 = max(c - t, 0), min(c + t + 1, w) + out[r0:r1, c0:c1] = _CONTOUR_COLOR + return out + + +def export_colony_overlay( + overlay: NDArray[np.uint8], + path: Path, +) -> Path: + """Write a colony-overlay image as JPEG.""" + iio.imwrite(path, overlay, quality=JPEG_QUALITY) + logger.debug("Wrote colony overlay: %s", path) + return path + + +def export_colony_csv( + frames: list[pd.DataFrame], + path: Path, +) -> Path: + """Concatenate per-plate DataFrames and write a CSV.""" + if frames: + combined = pd.concat(frames, ignore_index=True) + else: + combined = pd.DataFrame( + columns=[ + "plate_index", + "label", + "area_px", + "centroid_row", + "centroid_col", + "bbox_min_row", + "bbox_min_col", + "bbox_max_row", + "bbox_max_col", + "eccentricity", + "equivalent_diameter", + ] + ) + combined.to_csv(path, index=False) + logger.debug("Wrote colony CSV: %s", path) + return path + + # ------------------------------------------------------------------ # Orchestrator # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 7bfb3b2b..285d3b80 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -75,14 +75,62 @@ def process_file(run_id: str, filename: str) -> None: plate_crops = processor.crop_plates() if plate_crops: - from data_hub_lambda.epson_v700_scanner.colony_detection import detect_colonies + from data_hub_lambda.epson_v700_scanner.colony_detection import ( + detect_colonies, + draw_colony_overlay, + export_colony_csv, + export_colony_overlay, + ) + + colony_summaries = [] + dataframes = [] + for i, crop in enumerate(plate_crops): + result = detect_colonies(crop) + colony_summaries.append(result.summary()) + dataframes.append(result.to_dataframe(plate_index=i + 1)) + + overlay = draw_colony_overlay(crop, result.mask) + overlay_name = f"{processor.path.stem}_plate{i + 1}_colonies.jpg" + overlay_path = export_colony_overlay(overlay, raw_data_dir / overlay_name) + overlay_s3_key = f"{INSTRUMENT_ID}/{run_id}/{overlay_name}" + s3_utils.upload_file(overlay_path, f"s3://{processed_bucket}/{overlay_s3_key}") + overlay_file = client.create_file( + instrument_id=INSTRUMENT_ID, + run_id=run_id, + s3_bucket=processed_bucket or "", + s3_key=overlay_s3_key, + filename=overlay_name, + category="processed", + ) + client.update_file( + overlay_file.id, + size_bytes=overlay_path.stat().st_size, + content_type="image/jpeg", + ) + + csv_name = f"{processor.path.stem}_colonies.csv" + csv_path = export_colony_csv(dataframes, raw_data_dir / csv_name) + csv_s3_key = f"{INSTRUMENT_ID}/{run_id}/{csv_name}" + s3_utils.upload_file(csv_path, f"s3://{processed_bucket}/{csv_s3_key}") + csv_file = client.create_file( + instrument_id=INSTRUMENT_ID, + run_id=run_id, + s3_bucket=processed_bucket or "", + s3_key=csv_s3_key, + filename=csv_name, + category="processed", + ) + client.update_file( + csv_file.id, + size_bytes=csv_path.stat().st_size, + content_type="text/csv", + ) - colony_results = [detect_colonies(crop).summary() for crop in plate_crops] - metadata["colony_detection"] = colony_results + metadata["colony_detection"] = colony_summaries logger.info( "Colony detection complete for %d plate(s): %s", len(plate_crops), - [r["colony_count"] for r in colony_results], + [r["colony_count"] for r in colony_summaries], ) logger.info("Parsed metadata: %s", metadata) diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index e5a4d707..b6676149 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -1,8 +1,10 @@ """Unit tests for `epson_v700_scanner.colony_detection`.""" from __future__ import annotations +from pathlib import Path import numpy as np +import pandas as pd from data_hub_lambda.epson_v700_scanner.colony_detection import ( MARGIN_FRACTION, @@ -11,6 +13,9 @@ crop_margin, detect_colonies, detect_colony_presence, + draw_colony_overlay, + export_colony_csv, + export_colony_overlay, measure_colonies, optimize_colony_contrast, smooth, @@ -258,3 +263,137 @@ def test_grayscale_input(self) -> None: plate = np.full((200, 200), 128, dtype=np.uint8) result = detect_colonies(plate) assert isinstance(result, ColonyDetectionResult) + + +# ------------------------------------------------------------------ +# draw_colony_overlay +# ------------------------------------------------------------------ + + +class TestDrawColonyOverlay: + def test_output_shape_matches_input(self) -> None: + plate = _plate_with_colonies(h=300, w=300, n_colonies=3, colony_radius=20) + result = detect_colonies(plate) + overlay = draw_colony_overlay(plate, result.mask) + assert overlay.shape == plate.shape + assert overlay.dtype == np.uint8 + + def test_does_not_mutate_input(self) -> None: + plate = _uniform_plate(200, 200) + mask = np.zeros((160, 160), dtype=bool) + original = plate.copy() + draw_colony_overlay(plate, mask) + np.testing.assert_array_equal(plate, original) + + def test_empty_mask_returns_copy(self) -> None: + plate = _uniform_plate(200, 200) + mask = np.zeros((160, 160), dtype=bool) + overlay = draw_colony_overlay(plate, mask) + np.testing.assert_array_equal(overlay, plate) + + def test_contours_drawn_when_colonies_present(self) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + assert result.has_colonies + overlay = draw_colony_overlay(plate, result.mask) + diff = (overlay != plate).any(axis=-1) + assert diff.any(), "Expected overlay to differ from original" + + +# ------------------------------------------------------------------ +# export_colony_overlay +# ------------------------------------------------------------------ + + +class TestExportColonyOverlay: + def test_writes_jpeg(self, tmp_path: Path) -> None: + img = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + out = export_colony_overlay(img, tmp_path / "overlay.jpg") + assert out.exists() + assert out.suffix == ".jpg" + + +# ------------------------------------------------------------------ +# to_dataframe +# ------------------------------------------------------------------ + + +class TestToDataframe: + def test_columns_present(self) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + df = result.to_dataframe(plate_index=1) + assert isinstance(df, pd.DataFrame) + expected_cols = { + "plate_index", + "label", + "area_px", + "centroid_row", + "centroid_col", + "bbox_min_row", + "bbox_min_col", + "bbox_max_row", + "bbox_max_col", + "eccentricity", + "equivalent_diameter", + } + assert expected_cols == set(df.columns) + + def test_row_count_matches_colonies(self) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + df = result.to_dataframe() + assert len(df) == len(result.colonies) + + def test_empty_result_gives_empty_df(self) -> None: + plate = _uniform_plate() + result = detect_colonies(plate) + df = result.to_dataframe() + assert len(df) == 0 + assert "plate_index" in df.columns + + def test_plate_index_propagated(self) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + df = result.to_dataframe(plate_index=3) + assert (df["plate_index"] == 3).all() + + +# ------------------------------------------------------------------ +# export_colony_csv +# ------------------------------------------------------------------ + + +class TestExportColonyCsv: + def test_writes_csv(self, tmp_path: Path) -> None: + plate = _plate_with_colonies( + h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 + ) + result = detect_colonies(plate) + frames = [result.to_dataframe(plate_index=1)] + csv_path = export_colony_csv(frames, tmp_path / "colonies.csv") + assert csv_path.exists() + loaded = pd.read_csv(csv_path) + assert len(loaded) == len(result.colonies) + + def test_concatenates_multiple_plates(self, tmp_path: Path) -> None: + df1 = pd.DataFrame({"plate_index": [1], "label": [1], "area_px": [100]}) + df2 = pd.DataFrame({"plate_index": [2], "label": [1], "area_px": [200]}) + csv_path = export_colony_csv([df1, df2], tmp_path / "multi.csv") + loaded = pd.read_csv(csv_path) + assert len(loaded) == 2 + assert list(loaded["plate_index"]) == [1, 2] + + def test_empty_frames_writes_empty_csv(self, tmp_path: Path) -> None: + csv_path = export_colony_csv([], tmp_path / "empty.csv") + assert csv_path.exists() + loaded = pd.read_csv(csv_path) + assert len(loaded) == 0 diff --git a/lambda/tests/epson_v700_scanner/test_process_file.py b/lambda/tests/epson_v700_scanner/test_process_file.py index ae07835e..cbdc80cd 100644 --- a/lambda/tests/epson_v700_scanner/test_process_file.py +++ b/lambda/tests/epson_v700_scanner/test_process_file.py @@ -80,6 +80,7 @@ def patched_converter(patched_jpg_path: Path) -> MagicMock: "OriginalHeight": 4800, "OriginalWidth": 6400, } + converter.crop_plates.return_value = [] return converter From ce086b7fa167f0710d2d044f9ab62e576f040189 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 12:04:12 -0700 Subject: [PATCH 08/27] Epson V700 Scanner: Draw colony contours on the main export JPEG Instead of producing separate per-plate colony overlay images, colony contours are now drawn directly on the original export figure alongside the plate bounding boxes before resizing. Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 59 +++++++++---------- .../epson_v700_scanner/colony_detection.py | 51 ---------------- .../epson_v700_scanner/image_processing.py | 56 +++++++++++++++++- .../epson_v700_scanner/process_file.py | 54 +++++++---------- .../test_colony_detection.py | 52 ---------------- 5 files changed, 103 insertions(+), 169 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 4eb418ec..615c8a9a 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -120,53 +120,48 @@ def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> processor = TiffProcessor(file) processor.load() - jpg_path = processor.export_jpg() - - if output_dir is not None: - output_dir.mkdir(parents=True, exist_ok=True) - dest = output_dir / jpg_path.name - shutil.move(str(jpg_path), str(dest)) - jpg_path = dest - - click.echo(f"Exported JPG: {jpg_path}") - - metadata = processor.parse_metadata() - click.echo(json.dumps(metadata, indent=2)) + colony_masks: list | None = None if detect_colonies: from data_hub_lambda.epson_v700_scanner.colony_detection import ( detect_colonies as run_colony_detection, ) from data_hub_lambda.epson_v700_scanner.colony_detection import ( - draw_colony_overlay, export_colony_csv, - export_colony_overlay, ) + processor.detect_plates() plate_crops = processor.crop_plates() if not plate_crops: click.echo("No plates detected — skipping colony detection.") - return - - dest_dir = output_dir or file.parent - dest_dir.mkdir(parents=True, exist_ok=True) - - dataframes = [] - for i, crop in enumerate(plate_crops): - result = run_colony_detection(crop) - click.echo(f"\nPlate {i + 1}:") - click.echo(json.dumps(result.summary(), indent=2)) + else: + colony_masks = [] + dataframes = [] + for i, crop in enumerate(plate_crops): + result = run_colony_detection(crop) + colony_masks.append(result.mask) + click.echo(f"\nPlate {i + 1}:") + click.echo(json.dumps(result.summary(), indent=2)) + dataframes.append(result.to_dataframe(plate_index=i + 1)) + + dest_dir = output_dir or file.parent + dest_dir.mkdir(parents=True, exist_ok=True) + csv_path = dest_dir / f"{file.stem}_colonies.csv" + export_colony_csv(dataframes, csv_path) + click.echo(f"\nColony CSV: {csv_path}") + + jpg_path = processor.export_jpg(colony_masks=colony_masks) - overlay = draw_colony_overlay(crop, result.mask) - overlay_path = dest_dir / f"{file.stem}_plate{i + 1}_colonies.jpg" - export_colony_overlay(overlay, overlay_path) - click.echo(f"Colony overlay: {overlay_path}") + if output_dir is not None: + output_dir.mkdir(parents=True, exist_ok=True) + dest = output_dir / jpg_path.name + shutil.move(str(jpg_path), str(dest)) + jpg_path = dest - dataframes.append(result.to_dataframe(plate_index=i + 1)) + click.echo(f"Exported JPG: {jpg_path}") - csv_path = dest_dir / f"{file.stem}_colonies.csv" - export_colony_csv(dataframes, csv_path) - click.echo(f"\nColony CSV: {csv_path}") + metadata = processor.parse_metadata() + click.echo(json.dumps(metadata, indent=2)) # --------------------------------------------------------------------------- diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 52d86f28..8f13bcd2 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -18,7 +18,6 @@ from pathlib import Path from typing import Any -import imageio.v3 as iio import numpy as np import pandas as pd import skimage as ski @@ -33,11 +32,6 @@ _CONTRAST_PRESENCE_THRESHOLD = 5.0 """Minimum 95th-percentile contrast value to declare colonies present.""" -JPEG_QUALITY = 85 - -_CONTOUR_COLOR: tuple[int, int, int] = (0, 255, 255) -_CONTOUR_THICKNESS = 3 - @dataclass class ColonyProperties: @@ -226,51 +220,6 @@ def measure_colonies(mask: NDArray[np.bool_]) -> list[ColonyProperties]: # ------------------------------------------------------------------ -def draw_colony_overlay( - plate_image: NDArray[np.uint8], - mask: NDArray[np.bool_], - margin: float = MARGIN_FRACTION, -) -> NDArray[np.uint8]: - """Draw colony contour outlines on the plate image. - - The *mask* lives in the coordinate space of the margin-cropped image, - so contours are offset back to the original plate-image coordinates. - - Args: - plate_image: Original (H, W, 3) RGB uint8 plate crop. - mask: Binary colony mask in cropped coordinates. - margin: The same margin fraction used during detection. - - Returns: - Copy of *plate_image* with cyan contour outlines. - """ - out = plate_image.copy() - h, w = plate_image.shape[:2] - row_offset = int(h * margin) - col_offset = int(w * margin) - - contours = ski.measure.find_contours(mask.astype(float), level=0.5) - t = _CONTOUR_THICKNESS - for contour in contours: - for r_f, c_f in contour: - r = int(round(r_f)) + row_offset - c = int(round(c_f)) + col_offset - r0, r1 = max(r - t, 0), min(r + t + 1, h) - c0, c1 = max(c - t, 0), min(c + t + 1, w) - out[r0:r1, c0:c1] = _CONTOUR_COLOR - return out - - -def export_colony_overlay( - overlay: NDArray[np.uint8], - path: Path, -) -> Path: - """Write a colony-overlay image as JPEG.""" - iio.imwrite(path, overlay, quality=JPEG_QUALITY) - logger.debug("Wrote colony overlay: %s", path) - return path - - def export_colony_csv( frames: list[pd.DataFrame], path: Path, diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 71022497..73303add 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -33,6 +33,9 @@ _OVERLAY_COLOR: tuple[int, int, int] = (0, 255, 0) _OVERLAY_THICKNESS = 6 +_COLONY_CONTOUR_COLOR: tuple[int, int, int] = (0, 255, 255) +_COLONY_CONTOUR_THICKNESS = 3 + # PhotometricInterpretation values: 2 = RGB, others (0, 1, 3) are grayscale or # palette and are treated as B&W for our display purposes. _PHOTOMETRIC_RGB = 2 @@ -102,11 +105,19 @@ def intensities(self) -> NDArray[Any]: raise RuntimeError("Call load() first.") return self._intensities - def export_jpg(self) -> Path: + def export_jpg( + self, + colony_masks: list[NDArray[np.bool_]] | None = None, + ) -> Path: """Detect plates, draw overlays, resize, and write a JPEG. If no gold frames are detected the full image is exported as-is (the pre-detection fallback behaviour). + + Args: + colony_masks: Optional per-plate binary masks (one per entry in + ``plate_boxes``). When provided, colony contour outlines + are drawn on the export image. """ img = self._to_rgb_uint8(self.intensities) if self.plate_boxes is None: @@ -114,6 +125,8 @@ def export_jpg(self) -> Path: if self.plate_boxes: img = self._draw_plate_overlays(img, self.plate_boxes) + if colony_masks: + img = self._draw_colony_contours(img, self.plate_boxes, colony_masks) img = self._resize(img) @@ -174,14 +187,20 @@ def parse_metadata(self) -> dict[str, Any]: # Plate detection # ------------------------------------------------------------------ - def detect_plates(self, img: NDArray[np.uint8]) -> None: + def detect_plates(self, img: NDArray[np.uint8] | None = None) -> None: """Detect agar plates inside gold 3D-printed frames. Runs detection on a downsampled copy for speed, then scales bounding boxes back to original coordinates. Results are stored in ``self.plate_boxes`` as ``(min_row, min_col, max_row, max_col)`` tuples sorted left-to-right. + + Args: + img: Optional RGB uint8 image. When *None* the image is + derived from ``self.intensities``. """ + if img is None: + img = self._to_rgb_uint8(self.intensities) h_orig, w_orig = img.shape[:2] s = _DETECTION_DOWNSAMPLE small: NDArray[np.uint8] = img[::s, ::s] @@ -254,6 +273,39 @@ def _draw_plate_overlays( out[min_row:max_row, min_col:max_col] = img[min_row:max_row, min_col:max_col] return out + @staticmethod + def _draw_colony_contours( + img: NDArray[np.uint8], + boxes: list[_PlateBox], + colony_masks: list[NDArray[np.bool_]], + margin: float = 0.10, + ) -> NDArray[np.uint8]: + """Draw colony contour outlines onto *img* for each plate. + + Each mask lives in the margin-cropped coordinate space of its + plate crop, so contours are offset by the plate box origin plus + the crop margin. + """ + out = img.copy() + h, w = out.shape[:2] + t = _COLONY_CONTOUR_THICKNESS + for box, mask in zip(boxes, colony_masks, strict=True): + min_row, min_col, max_row, max_col = box + plate_h = max_row - min_row + plate_w = max_col - min_col + row_offset = min_row + int(plate_h * margin) + col_offset = min_col + int(plate_w * margin) + + contours = ski.measure.find_contours(mask.astype(float), level=0.5) + for contour in contours: + for r_f, c_f in contour: + r = int(round(r_f)) + row_offset + c = int(round(c_f)) + col_offset + r0, r1 = max(r - t, 0), min(r + t + 1, h) + c0, c1 = max(c - t, 0), min(c + t + 1, w) + out[r0:r1, c0:c1] = _COLONY_CONTOUR_COLOR + return out + def crop_plates(self) -> list[NDArray[np.uint8]]: """Return RGB uint8 crops for each detected plate. diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 285d3b80..2554208c 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -50,7 +50,27 @@ def process_file(run_id: str, filename: str) -> None: processor = TiffProcessor(local_file_path) processor.load() - jpg_file_path = processor.export_jpg() + processor.detect_plates() + + colony_masks: list | None = None + colony_summaries: list | None = None + dataframes: list | None = None + plate_crops = processor.crop_plates() + if plate_crops: + from data_hub_lambda.epson_v700_scanner.colony_detection import ( + detect_colonies, + ) + + colony_summaries = [] + colony_masks = [] + dataframes = [] + for i, crop in enumerate(plate_crops): + result = detect_colonies(crop) + colony_summaries.append(result.summary()) + colony_masks.append(result.mask) + dataframes.append(result.to_dataframe(plate_index=i + 1)) + + jpg_file_path = processor.export_jpg(colony_masks=colony_masks) processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET jpg_s3_key = f"{INSTRUMENT_ID}/{run_id}/{jpg_file_path.name}" @@ -73,41 +93,11 @@ def process_file(run_id: str, filename: str) -> None: metadata = processor.parse_metadata() - plate_crops = processor.crop_plates() - if plate_crops: + if plate_crops and dataframes and colony_summaries: from data_hub_lambda.epson_v700_scanner.colony_detection import ( - detect_colonies, - draw_colony_overlay, export_colony_csv, - export_colony_overlay, ) - colony_summaries = [] - dataframes = [] - for i, crop in enumerate(plate_crops): - result = detect_colonies(crop) - colony_summaries.append(result.summary()) - dataframes.append(result.to_dataframe(plate_index=i + 1)) - - overlay = draw_colony_overlay(crop, result.mask) - overlay_name = f"{processor.path.stem}_plate{i + 1}_colonies.jpg" - overlay_path = export_colony_overlay(overlay, raw_data_dir / overlay_name) - overlay_s3_key = f"{INSTRUMENT_ID}/{run_id}/{overlay_name}" - s3_utils.upload_file(overlay_path, f"s3://{processed_bucket}/{overlay_s3_key}") - overlay_file = client.create_file( - instrument_id=INSTRUMENT_ID, - run_id=run_id, - s3_bucket=processed_bucket or "", - s3_key=overlay_s3_key, - filename=overlay_name, - category="processed", - ) - client.update_file( - overlay_file.id, - size_bytes=overlay_path.stat().st_size, - content_type="image/jpeg", - ) - csv_name = f"{processor.path.stem}_colonies.csv" csv_path = export_colony_csv(dataframes, raw_data_dir / csv_name) csv_s3_key = f"{INSTRUMENT_ID}/{run_id}/{csv_name}" diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index b6676149..623e9d90 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -13,9 +13,7 @@ crop_margin, detect_colonies, detect_colony_presence, - draw_colony_overlay, export_colony_csv, - export_colony_overlay, measure_colonies, optimize_colony_contrast, smooth, @@ -265,56 +263,6 @@ def test_grayscale_input(self) -> None: assert isinstance(result, ColonyDetectionResult) -# ------------------------------------------------------------------ -# draw_colony_overlay -# ------------------------------------------------------------------ - - -class TestDrawColonyOverlay: - def test_output_shape_matches_input(self) -> None: - plate = _plate_with_colonies(h=300, w=300, n_colonies=3, colony_radius=20) - result = detect_colonies(plate) - overlay = draw_colony_overlay(plate, result.mask) - assert overlay.shape == plate.shape - assert overlay.dtype == np.uint8 - - def test_does_not_mutate_input(self) -> None: - plate = _uniform_plate(200, 200) - mask = np.zeros((160, 160), dtype=bool) - original = plate.copy() - draw_colony_overlay(plate, mask) - np.testing.assert_array_equal(plate, original) - - def test_empty_mask_returns_copy(self) -> None: - plate = _uniform_plate(200, 200) - mask = np.zeros((160, 160), dtype=bool) - overlay = draw_colony_overlay(plate, mask) - np.testing.assert_array_equal(overlay, plate) - - def test_contours_drawn_when_colonies_present(self) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) - result = detect_colonies(plate) - assert result.has_colonies - overlay = draw_colony_overlay(plate, result.mask) - diff = (overlay != plate).any(axis=-1) - assert diff.any(), "Expected overlay to differ from original" - - -# ------------------------------------------------------------------ -# export_colony_overlay -# ------------------------------------------------------------------ - - -class TestExportColonyOverlay: - def test_writes_jpeg(self, tmp_path: Path) -> None: - img = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) - out = export_colony_overlay(img, tmp_path / "overlay.jpg") - assert out.exists() - assert out.suffix == ".jpg" - - # ------------------------------------------------------------------ # to_dataframe # ------------------------------------------------------------------ From 8fe5c8ea93637fb368400788ba0354b580613b1d Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 12:29:08 -0700 Subject: [PATCH 09/27] Epson V700 Scanner: Use fixed pixel margin for colony detection crop Replace the 10% fractional margin with a fixed 200px crop from each edge, which better handles varying plate sizes. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 15 +++++----- .../epson_v700_scanner/image_processing.py | 12 ++++---- .../test_colony_detection.py | 28 ++++++++----------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 8f13bcd2..b7165f59 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -5,7 +5,7 @@ Pipeline -------- -1. Crop ~10 % margin to remove plate edges / frame artefacts. +1. Crop a fixed pixel margin to remove plate edges / frame artefacts. 2. Optimise contrast (Euclidean distance from estimated background colour). 3. Decide whether colonies are present (contrast above noise floor). 4. Gaussian smooth. @@ -25,7 +25,8 @@ logger = logging.getLogger(__name__) -MARGIN_FRACTION = 0.10 +MARGIN_PX = 200 +"""Fixed pixel margin cropped from each edge before colony detection.""" _GAUSSIAN_SIGMA = 2.0 @@ -117,20 +118,18 @@ def to_dataframe(self, plate_index: int = 0) -> pd.DataFrame: # ------------------------------------------------------------------ -def crop_margin(image: NDArray[Any], margin: float = MARGIN_FRACTION) -> NDArray[Any]: - """Remove a fractional margin from each edge of *image*. +def crop_margin(image: NDArray[Any], margin_px: int = MARGIN_PX) -> NDArray[Any]: + """Remove a fixed pixel margin from each edge of *image*. Args: image: (H, W) or (H, W, C) array. - margin: Fraction of each dimension to remove per side. + margin_px: Number of pixels to remove from each side. Returns: Cropped view of the original array. """ h, w = image.shape[:2] - row_margin = int(h * margin) - col_margin = int(w * margin) - return image[row_margin : h - row_margin, col_margin : w - col_margin] + return image[margin_px : h - margin_px, margin_px : w - margin_px] def optimize_colony_contrast(image: NDArray[Any]) -> NDArray[np.floating[Any]]: diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 73303add..2e0e8a4c 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -13,7 +13,7 @@ JPEG_QUALITY = 85 -MAX_DIMENSION = 1000 +MAX_DIMENSION = 2000 _TIFF_SUFFIXES = {".tif", ".tiff"} @@ -278,7 +278,7 @@ def _draw_colony_contours( img: NDArray[np.uint8], boxes: list[_PlateBox], colony_masks: list[NDArray[np.bool_]], - margin: float = 0.10, + margin_px: int = 150, ) -> NDArray[np.uint8]: """Draw colony contour outlines onto *img* for each plate. @@ -290,11 +290,9 @@ def _draw_colony_contours( h, w = out.shape[:2] t = _COLONY_CONTOUR_THICKNESS for box, mask in zip(boxes, colony_masks, strict=True): - min_row, min_col, max_row, max_col = box - plate_h = max_row - min_row - plate_w = max_col - min_col - row_offset = min_row + int(plate_h * margin) - col_offset = min_col + int(plate_w * margin) + min_row, min_col, _max_row, _max_col = box + row_offset = min_row + margin_px + col_offset = min_col + margin_px contours = ski.measure.find_contours(mask.astype(float), level=0.5) for contour in contours: diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 623e9d90..275a5069 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -7,7 +7,7 @@ import pandas as pd from data_hub_lambda.epson_v700_scanner.colony_detection import ( - MARGIN_FRACTION, + MARGIN_PX, ColonyDetectionResult, ColonyProperties, crop_margin, @@ -37,7 +37,7 @@ def _plate_with_colonies( """Create a synthetic plate with circular colonies inside the margin.""" img = np.full((h, w, 3), bg, dtype=np.uint8) rng = np.random.RandomState(42) - margin = int(max(h, w) * MARGIN_FRACTION) + colony_radius + 5 + margin = MARGIN_PX + colony_radius + 5 for _ in range(n_colonies): cy = rng.randint(margin, h - margin) cx = rng.randint(margin, w - margin) @@ -60,29 +60,25 @@ def _disk(cy: int, cx: int, radius: int, h: int, w: int) -> tuple[np.ndarray, np class TestCropMargin: - def test_default_removes_10pct(self) -> None: - img = np.zeros((200, 300, 3), dtype=np.uint8) + def test_default_removes_150px(self) -> None: + img = np.zeros((500, 600, 3), dtype=np.uint8) cropped = crop_margin(img) - expected_h = 200 - 2 * int(200 * MARGIN_FRACTION) - expected_w = 300 - 2 * int(300 * MARGIN_FRACTION) - assert cropped.shape == (expected_h, expected_w, 3) + assert cropped.shape == (500 - 2 * MARGIN_PX, 600 - 2 * MARGIN_PX, 3) def test_grayscale(self) -> None: - img = np.zeros((100, 100), dtype=np.uint8) + img = np.zeros((500, 500), dtype=np.uint8) cropped = crop_margin(img) - margin = int(100 * MARGIN_FRACTION) - assert cropped.shape == (100 - 2 * margin, 100 - 2 * margin) + assert cropped.shape == (500 - 2 * MARGIN_PX, 500 - 2 * MARGIN_PX) def test_custom_margin(self) -> None: img = np.zeros((200, 200, 3), dtype=np.uint8) - cropped = crop_margin(img, margin=0.25) + cropped = crop_margin(img, margin_px=50) assert cropped.shape == (100, 100, 3) def test_preserves_content(self) -> None: - img = np.arange(100 * 100, dtype=np.uint8).reshape(100, 100) - cropped = crop_margin(img, margin=0.10) - m = int(100 * 0.10) - np.testing.assert_array_equal(cropped, img[m : 100 - m, m : 100 - m]) + img = np.arange(500 * 500, dtype=np.uint8).reshape(500, 500) + cropped = crop_margin(img, margin_px=20) + np.testing.assert_array_equal(cropped, img[20:480, 20:480]) # ------------------------------------------------------------------ @@ -258,7 +254,7 @@ def test_cropped_smaller_than_input(self) -> None: assert result.cropped.shape[1] < 400 def test_grayscale_input(self) -> None: - plate = np.full((200, 200), 128, dtype=np.uint8) + plate = np.full((400, 400), 128, dtype=np.uint8) result = detect_colonies(plate) assert isinstance(result, ColonyDetectionResult) From 198a3c510e2eb3de4c975d25119fd9583878903f Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 12:34:25 -0700 Subject: [PATCH 10/27] Epson V700 Scanner: Source colony margin from single constant Have _draw_colony_contours import MARGIN_PX from colony_detection instead of hardcoding its own default, so the crop offset stays in sync. Co-authored-by: Cursor --- .../data_hub_lambda/epson_v700_scanner/image_processing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 2e0e8a4c..e2ac3238 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -278,7 +278,7 @@ def _draw_colony_contours( img: NDArray[np.uint8], boxes: list[_PlateBox], colony_masks: list[NDArray[np.bool_]], - margin_px: int = 150, + margin_px: int | None = None, ) -> NDArray[np.uint8]: """Draw colony contour outlines onto *img* for each plate. @@ -286,6 +286,11 @@ def _draw_colony_contours( plate crop, so contours are offset by the plate box origin plus the crop margin. """ + if margin_px is None: + from data_hub_lambda.epson_v700_scanner.colony_detection import MARGIN_PX + + margin_px = MARGIN_PX + out = img.copy() h, w = out.shape[:2] t = _COLONY_CONTOUR_THICKNESS From ec96c3fcfe5b7f90f0dfce8c811a01467d82d039 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 13:14:47 -0700 Subject: [PATCH 11/27] Epson V700 Scanner: Use 99.5th percentile for colony presence check The 95th percentile was too sensitive, causing empty plates to be misclassified as having colonies due to noise/edge artefacts. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index b7165f59..f1733b28 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -30,7 +30,7 @@ _GAUSSIAN_SIGMA = 2.0 -_CONTRAST_PRESENCE_THRESHOLD = 5.0 +_CONTRAST_PRESENCE_THRESHOLD = 20.0 """Minimum 95th-percentile contrast value to declare colonies present.""" @@ -168,12 +168,12 @@ def detect_colony_presence( ) -> bool: """Return ``True`` if there is enough contrast to indicate colonies. - Uses the 95th percentile of the contrast image; plates without + Uses the 99.5th percentile of the contrast image; plates without colonies have near-uniform background. """ - p95 = float(np.percentile(contrast, 95)) - logger.debug("Colony-presence p95 contrast = %.2f (threshold %.2f)", p95, threshold) - return p95 > threshold + p = float(np.percentile(contrast, 99.5)) + logger.debug("Colony-presence p99.5 contrast = %.2f (threshold %.2f)", p, threshold) + return p > threshold def smooth( From 265793b506e0c0ef998171a0701f20de5cb0508e Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 14:47:24 -0700 Subject: [PATCH 12/27] Epson V700 Scanner: Replace Gaussian smoothing with Difference-of-Gaussians Use a DoG band-pass filter (low_sigma=0.6, high_sigma=64) followed by 10th-percentile baseline subtraction for colony detection pre-processing. This suppresses both high-frequency noise and low-frequency background gradients more effectively than simple Gaussian smoothing. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 33 ++++++++++++++++--- .../test_colony_detection.py | 8 ++++- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index f1733b28..93d2a899 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -8,7 +8,7 @@ 1. Crop a fixed pixel margin to remove plate edges / frame artefacts. 2. Optimise contrast (Euclidean distance from estimated background colour). 3. Decide whether colonies are present (contrast above noise floor). -4. Gaussian smooth. +4. Difference-of-Gaussians band-pass filter with 10th-percentile subtraction. 5. Otsu threshold to produce a binary colony mask. """ @@ -28,7 +28,14 @@ MARGIN_PX = 200 """Fixed pixel margin cropped from each edge before colony detection.""" -_GAUSSIAN_SIGMA = 2.0 +_DOG_LOW_SIGMA = 0.6 +"""Low sigma for the Difference-of-Gaussians band-pass filter.""" + +_DOG_HIGH_SIGMA = 64.0 +"""High sigma for the Difference-of-Gaussians band-pass filter.""" + +_PERCENTILE_FLOOR = 10.0 +"""Percentile used for baseline subtraction after DoG filtering.""" _CONTRAST_PRESENCE_THRESHOLD = 20.0 """Minimum 95th-percentile contrast value to declare colonies present.""" @@ -178,10 +185,26 @@ def detect_colony_presence( def smooth( contrast: NDArray[np.floating[Any]], - sigma: float = _GAUSSIAN_SIGMA, + low_sigma: float = _DOG_LOW_SIGMA, + high_sigma: float = _DOG_HIGH_SIGMA, + percentile: float = _PERCENTILE_FLOOR, ) -> NDArray[np.floating[Any]]: - """Apply Gaussian smoothing.""" - return ski.filters.gaussian(contrast, sigma=sigma, preserve_range=True) # type: ignore[no-any-return] + """Apply Difference-of-Gaussians band-pass filter with percentile subtraction. + + The DoG filter retains structures between *low_sigma* and *high_sigma* + scale, suppressing both high-frequency noise and low-frequency background + gradients. A 10th-percentile baseline is then subtracted and negative + values are clipped to zero so the result stays non-negative for Otsu + thresholding. + """ + dog: NDArray[np.floating[Any]] = ski.filters.difference_of_gaussians( + contrast, + low_sigma=low_sigma, + high_sigma=high_sigma, + ) + floor = np.percentile(dog, percentile) + result: NDArray[np.floating[Any]] = np.clip(dog - floor, 0, None) + return result def threshold_colonies(smoothed: NDArray[np.floating[Any]]) -> NDArray[np.bool_]: diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 275a5069..33c516c0 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -148,9 +148,15 @@ def test_output_shape_preserved(self) -> None: def test_smoothing_reduces_noise(self) -> None: rng = np.random.RandomState(0) img = rng.rand(200, 200) * 100 - result = smooth(img, sigma=3.0) + result = smooth(img, low_sigma=0.6, high_sigma=64.0) assert result.std() < img.std() + def test_result_is_non_negative(self) -> None: + rng = np.random.RandomState(42) + img = rng.rand(200, 200) * 100 + result = smooth(img) + assert np.all(result >= 0) + # ------------------------------------------------------------------ # threshold_colonies From f60d13685b5ae60d94d73168e8648998568f88f5 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 14:48:23 -0700 Subject: [PATCH 13/27] Update colony_detection.py - tweak DoG parameters --- .../data_hub_lambda/epson_v700_scanner/colony_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 93d2a899..45cc5b5b 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -28,10 +28,10 @@ MARGIN_PX = 200 """Fixed pixel margin cropped from each edge before colony detection.""" -_DOG_LOW_SIGMA = 0.6 +_DOG_LOW_SIGMA = 1.0 """Low sigma for the Difference-of-Gaussians band-pass filter.""" -_DOG_HIGH_SIGMA = 64.0 +_DOG_HIGH_SIGMA = 128.0 """High sigma for the Difference-of-Gaussians band-pass filter.""" _PERCENTILE_FLOOR = 10.0 From 9b07bab7fb745b19075c06fb8e21feae315d0b40 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 14:54:48 -0700 Subject: [PATCH 14/27] Fix colony detection tests broken by MARGIN_PX increase to 200 Test helpers defaulted to 400x400 images which became empty after cropping 200px from each side. Increase defaults to 600x600 and add an empty-array guard in detect_colony_presence. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 2 ++ .../epson_v700_scanner/test_colony_detection.py | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 45cc5b5b..fc2864e6 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -178,6 +178,8 @@ def detect_colony_presence( Uses the 99.5th percentile of the contrast image; plates without colonies have near-uniform background. """ + if contrast.size == 0: + return False p = float(np.percentile(contrast, 99.5)) logger.debug("Colony-presence p99.5 contrast = %.2f (threshold %.2f)", p, threshold) return p > threshold diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 33c516c0..7739a14f 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -21,14 +21,14 @@ ) -def _uniform_plate(h: int = 400, w: int = 400, value: int = 120) -> np.ndarray: # type: ignore[type-arg] +def _uniform_plate(h: int = 600, w: int = 600, value: int = 120) -> np.ndarray: # type: ignore[type-arg] """Create a uniform-colour plate image (no colonies).""" return np.full((h, w, 3), value, dtype=np.uint8) def _plate_with_colonies( - h: int = 400, - w: int = 400, + h: int = 600, + w: int = 600, bg: int = 120, colony_color: tuple[int, int, int] = (255, 255, 255), n_colonies: int = 3, @@ -254,13 +254,13 @@ def test_summary_schema(self) -> None: assert "equivalent_diameter" in c def test_cropped_smaller_than_input(self) -> None: - plate = _uniform_plate(400, 400) + plate = _uniform_plate(600, 600) result = detect_colonies(plate) - assert result.cropped.shape[0] < 400 - assert result.cropped.shape[1] < 400 + assert result.cropped.shape[0] < 600 + assert result.cropped.shape[1] < 600 def test_grayscale_input(self) -> None: - plate = np.full((400, 400), 128, dtype=np.uint8) + plate = np.full((600, 600), 128, dtype=np.uint8) result = detect_colonies(plate) assert isinstance(result, ColonyDetectionResult) From 4e9b9b5e00a33dd3282eb06a22f064784093f7db Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 11 May 2026 16:33:36 -0700 Subject: [PATCH 15/27] Update colony_detection.py - minor correction to docstring --- .../src/data_hub_lambda/epson_v700_scanner/colony_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index fc2864e6..66ce14e3 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -38,7 +38,7 @@ """Percentile used for baseline subtraction after DoG filtering.""" _CONTRAST_PRESENCE_THRESHOLD = 20.0 -"""Minimum 95th-percentile contrast value to declare colonies present.""" +"""Minimum 99.5th-percentile contrast value to declare colonies present.""" @dataclass From e2222a299be34ac59b0987b3491cbc9d5e4907a1 Mon Sep 17 00:00:00 2001 From: lanery Date: Fri, 15 May 2026 14:24:27 -0700 Subject: [PATCH 16/27] Update image_processing.py - modify display parameters --- .../data_hub_lambda/epson_v700_scanner/image_processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index e2ac3238..44c9a304 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -31,10 +31,10 @@ _CLOSING_RADIUS = 5 _OVERLAY_COLOR: tuple[int, int, int] = (0, 255, 0) -_OVERLAY_THICKNESS = 6 +_OVERLAY_THICKNESS = 8 -_COLONY_CONTOUR_COLOR: tuple[int, int, int] = (0, 255, 255) -_COLONY_CONTOUR_THICKNESS = 3 +_COLONY_CONTOUR_COLOR: tuple[int, int, int] = (255, 0, 255) +_COLONY_CONTOUR_THICKNESS = 2 # PhotometricInterpretation values: 2 = RGB, others (0, 1, 3) are grayscale or # palette and are treated as B&W for our display purposes. From c9d5bf7be7a0987a44d5543bd14cdba536a55048 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 09:28:13 -0700 Subject: [PATCH 17/27] Colony detection: switch to physical units (mm) and collect RGB Convert all spatial colony properties from pixels to millimetres using scan DPI (25.4 / dpi). Centroid and bbox coordinates are now plate-relative (margin offset added back) for downstream colony picking. Per-colony mean RGB extracted via regionprops intensity_image. Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 3 +- .../epson_v700_scanner/colony_detection.py | 153 +++++++++++------- .../epson_v700_scanner/image_processing.py | 12 ++ .../epson_v700_scanner/process_file.py | 3 +- .../test_colony_detection.py | 98 +++++++---- 5 files changed, 176 insertions(+), 93 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 615c8a9a..31d09b5d 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -135,10 +135,11 @@ def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> if not plate_crops: click.echo("No plates detected — skipping colony detection.") else: + dpi = processor.dpi colony_masks = [] dataframes = [] for i, crop in enumerate(plate_crops): - result = run_colony_detection(crop) + result = run_colony_detection(crop, dpi=dpi) colony_masks.append(result.mask) click.echo(f"\nPlate {i + 1}:") click.echo(json.dumps(result.summary(), indent=2)) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 66ce14e3..1af989ce 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -43,15 +43,34 @@ @dataclass class ColonyProperties: - """Measured properties for a single colony.""" + """Measured properties for a single colony (physical units, plate-relative).""" label: int - area_px: int - centroid_row: float - centroid_col: float - bbox: tuple[int, int, int, int] + area_mm2: float + centroid_row_mm: float + centroid_col_mm: float + bbox_mm: tuple[float, float, float, float] eccentricity: float - equivalent_diameter: float + equivalent_diameter_mm: float + mean_rgb: tuple[float, float, float] + + +_DATAFRAME_COLUMNS = [ + "plate_index", + "label", + "area_mm2", + "centroid_row_mm", + "centroid_col_mm", + "bbox_min_row_mm", + "bbox_min_col_mm", + "bbox_max_row_mm", + "bbox_max_col_mm", + "eccentricity", + "equivalent_diameter_mm", + "mean_r", + "mean_g", + "mean_b", +] @dataclass @@ -72,11 +91,15 @@ def summary(self) -> dict[str, Any]: "colonies": [ { "label": c.label, - "area_px": c.area_px, - "centroid": [round(c.centroid_row, 1), round(c.centroid_col, 1)], - "bbox": list(c.bbox), + "area_mm2": round(c.area_mm2, 4), + "centroid_mm": [ + round(c.centroid_row_mm, 3), + round(c.centroid_col_mm, 3), + ], + "bbox_mm": [round(v, 3) for v in c.bbox_mm], "eccentricity": round(c.eccentricity, 4), - "equivalent_diameter": round(c.equivalent_diameter, 2), + "equivalent_diameter_mm": round(c.equivalent_diameter_mm, 4), + "mean_rgb": [round(v, 1) for v in c.mean_rgb], } for c in self.colonies ], @@ -85,36 +108,25 @@ def summary(self) -> dict[str, Any]: def to_dataframe(self, plate_index: int = 0) -> pd.DataFrame: """Return a :class:`~pandas.DataFrame` with one row per colony.""" if not self.colonies: - return pd.DataFrame( - columns=[ - "plate_index", - "label", - "area_px", - "centroid_row", - "centroid_col", - "bbox_min_row", - "bbox_min_col", - "bbox_max_row", - "bbox_max_col", - "eccentricity", - "equivalent_diameter", - ] - ) + return pd.DataFrame(columns=_DATAFRAME_COLUMNS) rows = [] for c in self.colonies: rows.append( { "plate_index": plate_index, "label": c.label, - "area_px": c.area_px, - "centroid_row": round(c.centroid_row, 1), - "centroid_col": round(c.centroid_col, 1), - "bbox_min_row": c.bbox[0], - "bbox_min_col": c.bbox[1], - "bbox_max_row": c.bbox[2], - "bbox_max_col": c.bbox[3], + "area_mm2": round(c.area_mm2, 4), + "centroid_row_mm": round(c.centroid_row_mm, 3), + "centroid_col_mm": round(c.centroid_col_mm, 3), + "bbox_min_row_mm": round(c.bbox_mm[0], 3), + "bbox_min_col_mm": round(c.bbox_mm[1], 3), + "bbox_max_row_mm": round(c.bbox_mm[2], 3), + "bbox_max_col_mm": round(c.bbox_mm[3], 3), "eccentricity": round(c.eccentricity, 4), - "equivalent_diameter": round(c.equivalent_diameter, 2), + "equivalent_diameter_mm": round(c.equivalent_diameter_mm, 4), + "mean_r": round(c.mean_rgb[0], 1), + "mean_g": round(c.mean_rgb[1], 1), + "mean_b": round(c.mean_rgb[2], 1), } ) return pd.DataFrame(rows) @@ -216,26 +228,61 @@ def threshold_colonies(smoothed: NDArray[np.floating[Any]]) -> NDArray[np.bool_] return mask -def measure_colonies(mask: NDArray[np.bool_]) -> list[ColonyProperties]: - """Label connected components and extract per-colony measurements.""" +def measure_colonies( + mask: NDArray[np.bool_], + rgb_image: NDArray[np.uint8], + dpi: int, + margin_px: int = MARGIN_PX, +) -> list[ColonyProperties]: + """Label connected components and extract per-colony measurements. + + All spatial measurements are returned in millimetres. Centroids and + bounding boxes are expressed relative to the plate crop origin (i.e. + the margin offset is added back before conversion). + + Args: + mask: Binary colony mask (margin-cropped coordinate space). + rgb_image: RGB uint8 image with the same shape as *mask* + (margin-cropped). Used as ``intensity_image`` so that + ``regionprops`` can compute per-colony mean colour. + dpi: Image resolution in dots-per-inch. + margin_px: Pixel margin that was removed from the plate crop. + """ + mm_per_px = 25.4 / dpi + labels = ski.measure.label(mask) - regions = ski.measure.regionprops(labels) + regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] for region in regions: + row_px = float(region.centroid[0]) + margin_px + col_px = float(region.centroid[1]) + margin_px + + min_r, min_c, max_r, max_c = region.bbox + bbox_mm = ( + (min_r + margin_px) * mm_per_px, + (min_c + margin_px) * mm_per_px, + (max_r + margin_px) * mm_per_px, + (max_c + margin_px) * mm_per_px, + ) + + rgb_mean = region.intensity_mean + mean_rgb = (float(rgb_mean[0]), float(rgb_mean[1]), float(rgb_mean[2])) + colonies.append( ColonyProperties( label=int(region.label), - area_px=int(region.area), - centroid_row=float(region.centroid[0]), - centroid_col=float(region.centroid[1]), - bbox=tuple(region.bbox), # type: ignore[arg-type] + area_mm2=float(region.area) * mm_per_px**2, + centroid_row_mm=row_px * mm_per_px, + centroid_col_mm=col_px * mm_per_px, + bbox_mm=bbox_mm, eccentricity=float(region.eccentricity), - equivalent_diameter=float(region.equivalent_diameter_area), + equivalent_diameter_mm=float(region.equivalent_diameter_area) * mm_per_px, + mean_rgb=mean_rgb, ) ) - colonies.sort(key=lambda c: c.area_px, reverse=True) + colonies.sort(key=lambda c: c.area_mm2, reverse=True) return colonies @@ -252,21 +299,7 @@ def export_colony_csv( if frames: combined = pd.concat(frames, ignore_index=True) else: - combined = pd.DataFrame( - columns=[ - "plate_index", - "label", - "area_px", - "centroid_row", - "centroid_col", - "bbox_min_row", - "bbox_min_col", - "bbox_max_row", - "bbox_max_col", - "eccentricity", - "equivalent_diameter", - ] - ) + combined = pd.DataFrame(columns=_DATAFRAME_COLUMNS) combined.to_csv(path, index=False) logger.debug("Wrote colony CSV: %s", path) return path @@ -277,12 +310,14 @@ def export_colony_csv( # ------------------------------------------------------------------ -def detect_colonies(plate_image: NDArray[Any]) -> ColonyDetectionResult: +def detect_colonies(plate_image: NDArray[Any], dpi: int) -> ColonyDetectionResult: """Run the full colony-detection pipeline on a single plate crop. Args: plate_image: RGB uint8 plate image produced by :class:`~data_hub_lambda.epson_v700_scanner.image_processing.TiffProcessor`. + dpi: Image resolution in dots-per-inch, used to convert pixel + measurements to millimetres. Returns: A :class:`ColonyDetectionResult` with all intermediate arrays @@ -303,7 +338,7 @@ def detect_colonies(plate_image: NDArray[Any]) -> ColonyDetectionResult: smoothed = smooth(contrast) mask = threshold_colonies(smoothed) - colonies = measure_colonies(mask) + colonies = measure_colonies(mask, rgb_image=cropped, dpi=dpi) logger.info("Detected %d colony/ies.", len(colonies)) return ColonyDetectionResult( diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 44c9a304..0ff40240 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -89,8 +89,20 @@ class TiffProcessor: def __init__(self, path: Path) -> None: self.path = path self._intensities: NDArray[Any] | None = None + self._dpi: int | None = None self.plate_boxes: list[_PlateBox] | None = None + @property + def dpi(self) -> int: + """Integer DPI derived from the TIFF ``XResolution`` tag.""" + if self._dpi is None: + with tifffile.TiffFile(self.path) as tif: + x_res_tag = tif.pages.first.tags.get("XResolution") + self._dpi = _derive_dpi(x_res_tag.value if x_res_tag else None) + if self._dpi is None: + raise ValueError(f"Cannot determine DPI for {self.path}") + return self._dpi + def load(self) -> None: if not self.path.exists(): raise FileNotFoundError(f"TIFF file not found: {self.path}") diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 2554208c..78a6e330 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -61,11 +61,12 @@ def process_file(run_id: str, filename: str) -> None: detect_colonies, ) + dpi = processor.dpi colony_summaries = [] colony_masks = [] dataframes = [] for i, crop in enumerate(plate_crops): - result = detect_colonies(crop) + result = detect_colonies(crop, dpi=dpi) colony_summaries.append(result.summary()) colony_masks.append(result.mask) dataframes.append(result.to_dataframe(plate_index=i + 1)) diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 7739a14f..1a44bfdd 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -183,35 +183,65 @@ def test_bimodal_separation(self) -> None: # ------------------------------------------------------------------ +_TEST_DPI = 1200 + + class TestMeasureColonies: def test_empty_mask_returns_empty(self) -> None: mask = np.zeros((100, 100), dtype=bool) - assert measure_colonies(mask) == [] + rgb = np.zeros((100, 100, 3), dtype=np.uint8) + assert measure_colonies(mask, rgb, dpi=_TEST_DPI) == [] def test_single_blob(self) -> None: mask = np.zeros((100, 100), dtype=bool) mask[40:60, 40:60] = True - colonies = measure_colonies(mask) + rgb = np.full((100, 100, 3), 128, dtype=np.uint8) + colonies = measure_colonies(mask, rgb, dpi=_TEST_DPI) assert len(colonies) == 1 - assert colonies[0].area_px == 20 * 20 + mm_per_px = 25.4 / _TEST_DPI + expected_area_mm2 = 20 * 20 * mm_per_px**2 + assert abs(colonies[0].area_mm2 - expected_area_mm2) < 1e-6 def test_two_blobs_sorted_by_area(self) -> None: mask = np.zeros((200, 200), dtype=bool) mask[10:20, 10:20] = True # 100 px mask[50:80, 50:80] = True # 900 px - colonies = measure_colonies(mask) + rgb = np.full((200, 200, 3), 128, dtype=np.uint8) + colonies = measure_colonies(mask, rgb, dpi=_TEST_DPI) assert len(colonies) == 2 - assert colonies[0].area_px > colonies[1].area_px + assert colonies[0].area_mm2 > colonies[1].area_mm2 def test_colony_properties_populated(self) -> None: mask = np.zeros((100, 100), dtype=bool) mask[40:60, 40:60] = True - colony = measure_colonies(mask)[0] + rgb = np.full((100, 100, 3), 128, dtype=np.uint8) + colony = measure_colonies(mask, rgb, dpi=_TEST_DPI)[0] assert isinstance(colony, ColonyProperties) assert colony.label >= 1 assert colony.eccentricity >= 0.0 - assert colony.equivalent_diameter > 0.0 - assert len(colony.bbox) == 4 + assert colony.equivalent_diameter_mm > 0.0 + assert len(colony.bbox_mm) == 4 + + def test_centroid_includes_margin_offset(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + mask[40:60, 40:60] = True + rgb = np.full((100, 100, 3), 128, dtype=np.uint8) + mm_per_px = 25.4 / _TEST_DPI + colony = measure_colonies(mask, rgb, dpi=_TEST_DPI, margin_px=MARGIN_PX)[0] + expected_row_mm = (50.0 + MARGIN_PX) * mm_per_px + expected_col_mm = (50.0 + MARGIN_PX) * mm_per_px + assert abs(colony.centroid_row_mm - expected_row_mm) < 0.1 + assert abs(colony.centroid_col_mm - expected_col_mm) < 0.1 + + def test_mean_rgb_populated(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + mask[40:60, 40:60] = True + rgb = np.zeros((100, 100, 3), dtype=np.uint8) + rgb[40:60, 40:60] = [200, 100, 50] + colony = measure_colonies(mask, rgb, dpi=_TEST_DPI)[0] + assert abs(colony.mean_rgb[0] - 200.0) < 1.0 + assert abs(colony.mean_rgb[1] - 100.0) < 1.0 + assert abs(colony.mean_rgb[2] - 50.0) < 1.0 # ------------------------------------------------------------------ @@ -222,7 +252,7 @@ def test_colony_properties_populated(self) -> None: class TestDetectColonies: def test_uniform_plate_no_colonies(self) -> None: plate = _uniform_plate() - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) assert isinstance(result, ColonyDetectionResult) assert result.has_colonies is False assert result.colonies == [] @@ -233,13 +263,13 @@ def test_plate_with_colonies_detected(self) -> None: plate = _plate_with_colonies( h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 ) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) assert result.has_colonies is True assert len(result.colonies) > 0 def test_summary_schema(self) -> None: plate = _plate_with_colonies(n_colonies=2) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) summary = result.summary() assert "has_colonies" in summary assert "colony_count" in summary @@ -247,21 +277,22 @@ def test_summary_schema(self) -> None: if summary["colonies"]: c = summary["colonies"][0] assert "label" in c - assert "area_px" in c - assert "centroid" in c - assert "bbox" in c + assert "area_mm2" in c + assert "centroid_mm" in c + assert "bbox_mm" in c assert "eccentricity" in c - assert "equivalent_diameter" in c + assert "equivalent_diameter_mm" in c + assert "mean_rgb" in c def test_cropped_smaller_than_input(self) -> None: plate = _uniform_plate(600, 600) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) assert result.cropped.shape[0] < 600 assert result.cropped.shape[1] < 600 def test_grayscale_input(self) -> None: plate = np.full((600, 600), 128, dtype=np.uint8) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) assert isinstance(result, ColonyDetectionResult) @@ -275,21 +306,24 @@ def test_columns_present(self) -> None: plate = _plate_with_colonies( h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 ) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe(plate_index=1) assert isinstance(df, pd.DataFrame) expected_cols = { "plate_index", "label", - "area_px", - "centroid_row", - "centroid_col", - "bbox_min_row", - "bbox_min_col", - "bbox_max_row", - "bbox_max_col", + "area_mm2", + "centroid_row_mm", + "centroid_col_mm", + "bbox_min_row_mm", + "bbox_min_col_mm", + "bbox_max_row_mm", + "bbox_max_col_mm", "eccentricity", - "equivalent_diameter", + "equivalent_diameter_mm", + "mean_r", + "mean_g", + "mean_b", } assert expected_cols == set(df.columns) @@ -297,13 +331,13 @@ def test_row_count_matches_colonies(self) -> None: plate = _plate_with_colonies( h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 ) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe() assert len(df) == len(result.colonies) def test_empty_result_gives_empty_df(self) -> None: plate = _uniform_plate() - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe() assert len(df) == 0 assert "plate_index" in df.columns @@ -312,7 +346,7 @@ def test_plate_index_propagated(self) -> None: plate = _plate_with_colonies( h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 ) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe(plate_index=3) assert (df["plate_index"] == 3).all() @@ -327,7 +361,7 @@ def test_writes_csv(self, tmp_path: Path) -> None: plate = _plate_with_colonies( h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 ) - result = detect_colonies(plate) + result = detect_colonies(plate, dpi=_TEST_DPI) frames = [result.to_dataframe(plate_index=1)] csv_path = export_colony_csv(frames, tmp_path / "colonies.csv") assert csv_path.exists() @@ -335,8 +369,8 @@ def test_writes_csv(self, tmp_path: Path) -> None: assert len(loaded) == len(result.colonies) def test_concatenates_multiple_plates(self, tmp_path: Path) -> None: - df1 = pd.DataFrame({"plate_index": [1], "label": [1], "area_px": [100]}) - df2 = pd.DataFrame({"plate_index": [2], "label": [1], "area_px": [200]}) + df1 = pd.DataFrame({"plate_index": [1], "label": [1], "area_mm2": [0.5]}) + df2 = pd.DataFrame({"plate_index": [2], "label": [1], "area_mm2": [1.0]}) csv_path = export_colony_csv([df1, df2], tmp_path / "multi.csv") loaded = pd.read_csv(csv_path) assert len(loaded) == 2 From 9464bf3c4210a7baef5302a3a49b9844204b5ca9 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 09:31:14 -0700 Subject: [PATCH 18/27] Colony detection: exclude border-touching colonies via clear_border Co-authored-by: Cursor --- .../src/data_hub_lambda/epson_v700_scanner/colony_detection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 1af989ce..e444c12b 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -251,6 +251,7 @@ def measure_colonies( mm_per_px = 25.4 / dpi labels = ski.measure.label(mask) + labels = ski.segmentation.clear_border(labels) regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] From 1aa24f374007b74bb127e1980baa5cbefbf54b2f Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 09:35:59 -0700 Subject: [PATCH 19/27] Revert "Colony detection: exclude border-touching colonies via clear_border" This reverts commit 9464bf3c4210a7baef5302a3a49b9844204b5ca9. --- .../src/data_hub_lambda/epson_v700_scanner/colony_detection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index e444c12b..1af989ce 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -251,7 +251,6 @@ def measure_colonies( mm_per_px = 25.4 / dpi labels = ski.measure.label(mask) - labels = ski.segmentation.clear_border(labels) regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] From 57764a60713fad419b30f5156b8882a23137e917 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 09:42:09 -0700 Subject: [PATCH 20/27] Colony detection: remove border-touching colonies with clear_border Colonies that touch the edge of the margin-cropped region are partial and have distorted properties. Also bump MARGIN_PX to 250 and increase test plate sizes accordingly. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 3 +- .../test_colony_detection.py | 44 +++++++------------ 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 1af989ce..529a2247 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -MARGIN_PX = 200 +MARGIN_PX = 250 """Fixed pixel margin cropped from each edge before colony detection.""" _DOG_LOW_SIGMA = 1.0 @@ -251,6 +251,7 @@ def measure_colonies( mm_per_px = 25.4 / dpi labels = ski.measure.label(mask) + labels = ski.segmentation.clear_border(labels) regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 1a44bfdd..7701e860 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -21,14 +21,14 @@ ) -def _uniform_plate(h: int = 600, w: int = 600, value: int = 120) -> np.ndarray: # type: ignore[type-arg] +def _uniform_plate(h: int = 1000, w: int = 1000, value: int = 120) -> np.ndarray: # type: ignore[type-arg] """Create a uniform-colour plate image (no colonies).""" return np.full((h, w, 3), value, dtype=np.uint8) def _plate_with_colonies( - h: int = 600, - w: int = 600, + h: int = 1000, + w: int = 1000, bg: int = 120, colony_color: tuple[int, int, int] = (255, 255, 255), n_colonies: int = 3, @@ -60,15 +60,15 @@ def _disk(cy: int, cx: int, radius: int, h: int, w: int) -> tuple[np.ndarray, np class TestCropMargin: - def test_default_removes_150px(self) -> None: - img = np.zeros((500, 600, 3), dtype=np.uint8) + def test_default_removes_margin(self) -> None: + img = np.zeros((1000, 1200, 3), dtype=np.uint8) cropped = crop_margin(img) - assert cropped.shape == (500 - 2 * MARGIN_PX, 600 - 2 * MARGIN_PX, 3) + assert cropped.shape == (1000 - 2 * MARGIN_PX, 1200 - 2 * MARGIN_PX, 3) def test_grayscale(self) -> None: - img = np.zeros((500, 500), dtype=np.uint8) + img = np.zeros((1000, 1000), dtype=np.uint8) cropped = crop_margin(img) - assert cropped.shape == (500 - 2 * MARGIN_PX, 500 - 2 * MARGIN_PX) + assert cropped.shape == (1000 - 2 * MARGIN_PX, 1000 - 2 * MARGIN_PX) def test_custom_margin(self) -> None: img = np.zeros((200, 200, 3), dtype=np.uint8) @@ -260,9 +260,7 @@ def test_uniform_plate_no_colonies(self) -> None: assert not result.mask.any() def test_plate_with_colonies_detected(self) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) + plate = _plate_with_colonies(colony_color=(255, 255, 255), n_colonies=8, colony_radius=30) result = detect_colonies(plate, dpi=_TEST_DPI) assert result.has_colonies is True assert len(result.colonies) > 0 @@ -285,13 +283,13 @@ def test_summary_schema(self) -> None: assert "mean_rgb" in c def test_cropped_smaller_than_input(self) -> None: - plate = _uniform_plate(600, 600) + plate = _uniform_plate(1000, 1000) result = detect_colonies(plate, dpi=_TEST_DPI) - assert result.cropped.shape[0] < 600 - assert result.cropped.shape[1] < 600 + assert result.cropped.shape[0] < 1000 + assert result.cropped.shape[1] < 1000 def test_grayscale_input(self) -> None: - plate = np.full((600, 600), 128, dtype=np.uint8) + plate = np.full((1000, 1000), 128, dtype=np.uint8) result = detect_colonies(plate, dpi=_TEST_DPI) assert isinstance(result, ColonyDetectionResult) @@ -303,9 +301,7 @@ def test_grayscale_input(self) -> None: class TestToDataframe: def test_columns_present(self) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) + plate = _plate_with_colonies(colony_color=(255, 255, 255), n_colonies=8, colony_radius=30) result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe(plate_index=1) assert isinstance(df, pd.DataFrame) @@ -328,9 +324,7 @@ def test_columns_present(self) -> None: assert expected_cols == set(df.columns) def test_row_count_matches_colonies(self) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) + plate = _plate_with_colonies(colony_color=(255, 255, 255), n_colonies=8, colony_radius=30) result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe() assert len(df) == len(result.colonies) @@ -343,9 +337,7 @@ def test_empty_result_gives_empty_df(self) -> None: assert "plate_index" in df.columns def test_plate_index_propagated(self) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) + plate = _plate_with_colonies(colony_color=(255, 255, 255), n_colonies=8, colony_radius=30) result = detect_colonies(plate, dpi=_TEST_DPI) df = result.to_dataframe(plate_index=3) assert (df["plate_index"] == 3).all() @@ -358,9 +350,7 @@ def test_plate_index_propagated(self) -> None: class TestExportColonyCsv: def test_writes_csv(self, tmp_path: Path) -> None: - plate = _plate_with_colonies( - h=600, w=600, colony_color=(255, 255, 255), n_colonies=8, colony_radius=30 - ) + plate = _plate_with_colonies(colony_color=(255, 255, 255), n_colonies=8, colony_radius=30) result = detect_colonies(plate, dpi=_TEST_DPI) frames = [result.to_dataframe(plate_index=1)] csv_path = export_colony_csv(frames, tmp_path / "colonies.csv") From 501fffbb9768b5266dcb34dd441996e0adcae4cd Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 09:57:57 -0700 Subject: [PATCH 21/27] Return cleaned mask from measure_colonies for contour overlay The JPG export draws contours from result.mask. Previously it used the raw Otsu mask which still contained border-touching colonies that were filtered from the properties list. Now measure_colonies returns the post-clear_border mask so contours and properties stay consistent. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 17 +++++++----- .../test_colony_detection.py | 27 ++++++++++++++----- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 529a2247..3c6c4ff1 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -MARGIN_PX = 250 +MARGIN_PX = 200 """Fixed pixel margin cropped from each edge before colony detection.""" _DOG_LOW_SIGMA = 1.0 @@ -233,7 +233,7 @@ def measure_colonies( rgb_image: NDArray[np.uint8], dpi: int, margin_px: int = MARGIN_PX, -) -> list[ColonyProperties]: +) -> tuple[list[ColonyProperties], NDArray[np.bool_]]: """Label connected components and extract per-colony measurements. All spatial measurements are returned in millimetres. Centroids and @@ -247,11 +247,16 @@ def measure_colonies( ``regionprops`` can compute per-colony mean colour. dpi: Image resolution in dots-per-inch. margin_px: Pixel margin that was removed from the plate crop. + + Returns: + A tuple of (colony list, cleaned mask). The cleaned mask has + border-touching regions removed. """ mm_per_px = 25.4 / dpi labels = ski.measure.label(mask) labels = ski.segmentation.clear_border(labels) + cleaned_mask: NDArray[np.bool_] = labels > 0 regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] @@ -284,7 +289,7 @@ def measure_colonies( ) colonies.sort(key=lambda c: c.area_mm2, reverse=True) - return colonies + return colonies, cleaned_mask # ------------------------------------------------------------------ @@ -338,14 +343,14 @@ def detect_colonies(plate_image: NDArray[Any], dpi: int) -> ColonyDetectionResul ) smoothed = smooth(contrast) - mask = threshold_colonies(smoothed) - colonies = measure_colonies(mask, rgb_image=cropped, dpi=dpi) + raw_mask = threshold_colonies(smoothed) + colonies, cleaned_mask = measure_colonies(raw_mask, rgb_image=cropped, dpi=dpi) logger.info("Detected %d colony/ies.", len(colonies)) return ColonyDetectionResult( cropped=cropped, contrast=contrast, has_colonies=True, - mask=mask, + mask=cleaned_mask, colonies=colonies, ) diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 7701e860..159e1c14 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -190,13 +190,15 @@ class TestMeasureColonies: def test_empty_mask_returns_empty(self) -> None: mask = np.zeros((100, 100), dtype=bool) rgb = np.zeros((100, 100, 3), dtype=np.uint8) - assert measure_colonies(mask, rgb, dpi=_TEST_DPI) == [] + colonies, cleaned = measure_colonies(mask, rgb, dpi=_TEST_DPI) + assert colonies == [] + assert not cleaned.any() def test_single_blob(self) -> None: mask = np.zeros((100, 100), dtype=bool) mask[40:60, 40:60] = True rgb = np.full((100, 100, 3), 128, dtype=np.uint8) - colonies = measure_colonies(mask, rgb, dpi=_TEST_DPI) + colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI) assert len(colonies) == 1 mm_per_px = 25.4 / _TEST_DPI expected_area_mm2 = 20 * 20 * mm_per_px**2 @@ -207,7 +209,7 @@ def test_two_blobs_sorted_by_area(self) -> None: mask[10:20, 10:20] = True # 100 px mask[50:80, 50:80] = True # 900 px rgb = np.full((200, 200, 3), 128, dtype=np.uint8) - colonies = measure_colonies(mask, rgb, dpi=_TEST_DPI) + colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI) assert len(colonies) == 2 assert colonies[0].area_mm2 > colonies[1].area_mm2 @@ -215,7 +217,8 @@ def test_colony_properties_populated(self) -> None: mask = np.zeros((100, 100), dtype=bool) mask[40:60, 40:60] = True rgb = np.full((100, 100, 3), 128, dtype=np.uint8) - colony = measure_colonies(mask, rgb, dpi=_TEST_DPI)[0] + colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI) + colony = colonies[0] assert isinstance(colony, ColonyProperties) assert colony.label >= 1 assert colony.eccentricity >= 0.0 @@ -227,7 +230,8 @@ def test_centroid_includes_margin_offset(self) -> None: mask[40:60, 40:60] = True rgb = np.full((100, 100, 3), 128, dtype=np.uint8) mm_per_px = 25.4 / _TEST_DPI - colony = measure_colonies(mask, rgb, dpi=_TEST_DPI, margin_px=MARGIN_PX)[0] + colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI, margin_px=MARGIN_PX) + colony = colonies[0] expected_row_mm = (50.0 + MARGIN_PX) * mm_per_px expected_col_mm = (50.0 + MARGIN_PX) * mm_per_px assert abs(colony.centroid_row_mm - expected_row_mm) < 0.1 @@ -238,11 +242,22 @@ def test_mean_rgb_populated(self) -> None: mask[40:60, 40:60] = True rgb = np.zeros((100, 100, 3), dtype=np.uint8) rgb[40:60, 40:60] = [200, 100, 50] - colony = measure_colonies(mask, rgb, dpi=_TEST_DPI)[0] + colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI) + colony = colonies[0] assert abs(colony.mean_rgb[0] - 200.0) < 1.0 assert abs(colony.mean_rgb[1] - 100.0) < 1.0 assert abs(colony.mean_rgb[2] - 50.0) < 1.0 + def test_border_colonies_excluded(self) -> None: + mask = np.zeros((100, 100), dtype=bool) + mask[0:10, 40:60] = True # touches top border + mask[40:60, 40:60] = True # interior + rgb = np.full((100, 100, 3), 128, dtype=np.uint8) + colonies, cleaned = measure_colonies(mask, rgb, dpi=_TEST_DPI) + assert len(colonies) == 1 + assert not cleaned[0:10, 40:60].any() + assert cleaned[40:60, 40:60].all() + # ------------------------------------------------------------------ # detect_colonies (full pipeline) From 6d2f51ab8948d89fc7f6ca4ef7f72a60da83cc68 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 10:34:58 -0700 Subject: [PATCH 22/27] Refactor colony overlay from contour drawing to labeled bounding boxes Replace pixel-level contour stamping with matplotlib-rendered bounding boxes and numeric labels. Clean up inline imports (move MARGIN_PX to top-level, use Figure/FigureCanvasAgg directly instead of pyplot with global backend side effect) and replace BytesIO roundtrip with direct canvas buffer read. Co-authored-by: Cursor --- .../epson_v700_scanner/image_processing.py | 99 +++++++++++++------ 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 0ff40240..4831a4a2 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -9,6 +9,8 @@ import tifffile from numpy.typing import NDArray +from data_hub_lambda.epson_v700_scanner.colony_detection import MARGIN_PX + logger = logging.getLogger(__name__) JPEG_QUALITY = 85 @@ -33,8 +35,9 @@ _OVERLAY_COLOR: tuple[int, int, int] = (0, 255, 0) _OVERLAY_THICKNESS = 8 -_COLONY_CONTOUR_COLOR: tuple[int, int, int] = (255, 0, 255) -_COLONY_CONTOUR_THICKNESS = 2 +_COLONY_BBOX_COLOR: tuple[int, int, int] = (255, 255, 255) +_COLONY_BBOX_THICKNESS = 3 +_COLONY_LABEL_FONT_SIZE = 28 # PhotometricInterpretation values: 2 = RGB, others (0, 1, 3) are grayscale or # palette and are treated as B&W for our display purposes. @@ -128,8 +131,8 @@ def export_jpg( Args: colony_masks: Optional per-plate binary masks (one per entry in - ``plate_boxes``). When provided, colony contour outlines - are drawn on the export image. + ``plate_boxes``). When provided, colony bounding boxes + and labels are drawn on the export image. """ img = self._to_rgb_uint8(self.intensities) if self.plate_boxes is None: @@ -138,7 +141,7 @@ def export_jpg( if self.plate_boxes: img = self._draw_plate_overlays(img, self.plate_boxes) if colony_masks: - img = self._draw_colony_contours(img, self.plate_boxes, colony_masks) + img = self._draw_colony_bboxes(img, self.plate_boxes, colony_masks) img = self._resize(img) @@ -286,40 +289,76 @@ def _draw_plate_overlays( return out @staticmethod - def _draw_colony_contours( + def _draw_colony_bboxes( img: NDArray[np.uint8], boxes: list[_PlateBox], colony_masks: list[NDArray[np.bool_]], - margin_px: int | None = None, + margin_px: int = MARGIN_PX, ) -> NDArray[np.uint8]: - """Draw colony contour outlines onto *img* for each plate. + """Draw colony bounding boxes and labels onto *img* for each plate. Each mask lives in the margin-cropped coordinate space of its - plate crop, so contours are offset by the plate box origin plus - the crop margin. - """ - if margin_px is None: - from data_hub_lambda.epson_v700_scanner.colony_detection import MARGIN_PX + plate crop. Bounding boxes are offset by the plate box origin + plus the crop margin to map into full-image coordinates. - margin_px = MARGIN_PX + Uses matplotlib for anti-aliased rectangle and text rendering. + """ + from matplotlib.backends.backend_agg import FigureCanvasAgg + from matplotlib.figure import Figure + from matplotlib.patches import Rectangle + + img_h, img_w = img.shape[:2] + dpi = 100 + fig = Figure(figsize=(img_w / dpi, img_h / dpi), dpi=dpi) + canvas = FigureCanvasAgg(fig) + ax = fig.add_subplot(1, 1, 1) + ax.imshow(img) + ax.set_axis_off() + ax.set_xlim(0, img_w) + ax.set_ylim(img_h, 0) + + bbox_color = tuple(c / 255.0 for c in _COLONY_BBOX_COLOR) - out = img.copy() - h, w = out.shape[:2] - t = _COLONY_CONTOUR_THICKNESS for box, mask in zip(boxes, colony_masks, strict=True): - min_row, min_col, _max_row, _max_col = box - row_offset = min_row + margin_px - col_offset = min_col + margin_px - - contours = ski.measure.find_contours(mask.astype(float), level=0.5) - for contour in contours: - for r_f, c_f in contour: - r = int(round(r_f)) + row_offset - c = int(round(c_f)) + col_offset - r0, r1 = max(r - t, 0), min(r + t + 1, h) - c0, c1 = max(c - t, 0), min(c + t + 1, w) - out[r0:r1, c0:c1] = _COLONY_CONTOUR_COLOR - return out + plate_min_row, plate_min_col, _max_row, _max_col = box + row_offset = plate_min_row + margin_px + col_offset = plate_min_col + margin_px + + labels = ski.measure.label(mask) + regions = ski.measure.regionprops(labels) + + for region in regions: + min_r, min_c, max_r, max_c = region.bbox + r0 = min_r + row_offset + c0 = min_c + col_offset + r1 = max_r + row_offset + c1 = max_c + col_offset + + rect = Rectangle( + (c0, r0), + c1 - c0, + r1 - r0, + linewidth=_COLONY_BBOX_THICKNESS, + edgecolor=bbox_color, + facecolor="none", + ) + ax.add_patch(rect) + ax.text( + (c0 + c1) / 2, + r0 - 4, + str(region.label), + color=bbox_color, + fontsize=_COLONY_LABEL_FONT_SIZE, + fontweight="bold", + horizontalalignment="center", + verticalalignment="bottom", + clip_on=True, + ) + + fig.subplots_adjust(left=0, right=1, top=1, bottom=0) + canvas.draw() + buf: NDArray[np.uint8] = np.asarray(canvas.buffer_rgba())[:, :, :3].copy() + return buf def crop_plates(self) -> list[NDArray[np.uint8]]: """Return RGB uint8 crops for each detected plate. From dad1f82f1c32a9b2441890e40e363997cd17570c Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 10:45:39 -0700 Subject: [PATCH 23/27] =?UTF-8?q?Colony=20detection:=20discard=20colonies?= =?UTF-8?q?=20smaller=20than=200.05=20mm=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use skimage remove_small_objects to filter noise artifacts from the binary mask before labeling. The threshold is converted from mm² to pixels using the scan DPI. Co-authored-by: Cursor --- .../data_hub_lambda/epson_v700_scanner/colony_detection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 3c6c4ff1..dbcd6c58 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -40,6 +40,9 @@ _CONTRAST_PRESENCE_THRESHOLD = 20.0 """Minimum 99.5th-percentile contrast value to declare colonies present.""" +_MIN_COLONY_AREA_MM2 = 0.05 +"""Colonies smaller than this (in mm²) are discarded as noise.""" + @dataclass class ColonyProperties: @@ -253,7 +256,9 @@ def measure_colonies( border-touching regions removed. """ mm_per_px = 25.4 / dpi + min_area_px = int(np.ceil(_MIN_COLONY_AREA_MM2 / mm_per_px**2)) + mask = ski.morphology.remove_small_objects(mask, min_size=min_area_px) labels = ski.measure.label(mask) labels = ski.segmentation.clear_border(labels) cleaned_mask: NDArray[np.bool_] = labels > 0 From 26b3a53436c023f66879ecb226a1d959736e6354 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 10:46:19 -0700 Subject: [PATCH 24/27] Hide colony labels when plate has >300 colonies to reduce clutter Co-authored-by: Cursor --- .../epson_v700_scanner/image_processing.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index 4831a4a2..ba30c9c2 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -38,6 +38,7 @@ _COLONY_BBOX_COLOR: tuple[int, int, int] = (255, 255, 255) _COLONY_BBOX_THICKNESS = 3 _COLONY_LABEL_FONT_SIZE = 28 +_COLONY_LABEL_MAX_COUNT = 300 # PhotometricInterpretation values: 2 = RGB, others (0, 1, 3) are grayscale or # palette and are treated as B&W for our display purposes. @@ -326,6 +327,7 @@ def _draw_colony_bboxes( labels = ski.measure.label(mask) regions = ski.measure.regionprops(labels) + show_labels = len(regions) <= _COLONY_LABEL_MAX_COUNT for region in regions: min_r, min_c, max_r, max_c = region.bbox @@ -343,17 +345,18 @@ def _draw_colony_bboxes( facecolor="none", ) ax.add_patch(rect) - ax.text( - (c0 + c1) / 2, - r0 - 4, - str(region.label), - color=bbox_color, - fontsize=_COLONY_LABEL_FONT_SIZE, - fontweight="bold", - horizontalalignment="center", - verticalalignment="bottom", - clip_on=True, - ) + if show_labels: + ax.text( + (c0 + c1) / 2, + r0 - 4, + str(region.label), + color=bbox_color, + fontsize=_COLONY_LABEL_FONT_SIZE, + fontweight="bold", + horizontalalignment="center", + verticalalignment="bottom", + clip_on=True, + ) fig.subplots_adjust(left=0, right=1, top=1, bottom=0) canvas.draw() From 84e27f186dd0316b876b376f183b3a3341a227eb Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 10:57:06 -0700 Subject: [PATCH 25/27] Update colony_detection.py - label after clear border --- .../data_hub_lambda/epson_v700_scanner/colony_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index dbcd6c58..b6b45b79 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -259,9 +259,9 @@ def measure_colonies( min_area_px = int(np.ceil(_MIN_COLONY_AREA_MM2 / mm_per_px**2)) mask = ski.morphology.remove_small_objects(mask, min_size=min_area_px) - labels = ski.measure.label(mask) - labels = ski.segmentation.clear_border(labels) + labels = ski.segmentation.clear_border(mask) cleaned_mask: NDArray[np.bool_] = labels > 0 + labels = ski.measure.label(cleaned_mask) regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] From c2e93bfaa17d816c8dcecc8c2bcade617d581540 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 11:33:30 -0700 Subject: [PATCH 26/27] Unify colony detection pipeline into single shared entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract run_colony_pipeline() so both the Lambda handler and CLI delegate to one function instead of duplicating the plate-loop logic. Fix remove_small_objects deprecation (min_size → max_size) and a test that used a blob below the minimum-area threshold. Co-authored-by: Cursor --- lambda/src/data_hub_lambda/cli.py | 21 +++---- .../epson_v700_scanner/colony_detection.py | 41 ++++++++++++- .../epson_v700_scanner/image_processing.py | 60 ++++++++++--------- .../epson_v700_scanner/process_file.py | 34 ++++------- .../test_colony_detection.py | 2 +- 5 files changed, 92 insertions(+), 66 deletions(-) diff --git a/lambda/src/data_hub_lambda/cli.py b/lambda/src/data_hub_lambda/cli.py index 31d09b5d..3b8cf920 100644 --- a/lambda/src/data_hub_lambda/cli.py +++ b/lambda/src/data_hub_lambda/cli.py @@ -121,13 +121,11 @@ def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> processor = TiffProcessor(file) processor.load() - colony_masks: list | None = None + pipeline = None if detect_colonies: - from data_hub_lambda.epson_v700_scanner.colony_detection import ( - detect_colonies as run_colony_detection, - ) from data_hub_lambda.epson_v700_scanner.colony_detection import ( export_colony_csv, + run_colony_pipeline, ) processor.detect_plates() @@ -135,23 +133,20 @@ def epson_scanner(file: Path, output_dir: Path | None, detect_colonies: bool) -> if not plate_crops: click.echo("No plates detected — skipping colony detection.") else: - dpi = processor.dpi - colony_masks = [] - dataframes = [] - for i, crop in enumerate(plate_crops): - result = run_colony_detection(crop, dpi=dpi) - colony_masks.append(result.mask) + pipeline = run_colony_pipeline(plate_crops, dpi=processor.dpi) + for i, result in enumerate(pipeline.results): click.echo(f"\nPlate {i + 1}:") click.echo(json.dumps(result.summary(), indent=2)) - dataframes.append(result.to_dataframe(plate_index=i + 1)) dest_dir = output_dir or file.parent dest_dir.mkdir(parents=True, exist_ok=True) csv_path = dest_dir / f"{file.stem}_colonies.csv" - export_colony_csv(dataframes, csv_path) + export_colony_csv(pipeline.to_dataframes(), csv_path) click.echo(f"\nColony CSV: {csv_path}") - jpg_path = processor.export_jpg(colony_masks=colony_masks) + jpg_path = processor.export_jpg( + colony_results=pipeline.results if pipeline else None, + ) if output_dir is not None: output_dir.mkdir(parents=True, exist_ok=True) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index b6b45b79..5e5702d5 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -78,7 +78,7 @@ class ColonyProperties: @dataclass class ColonyDetectionResult: - """Full result of the colony-detection pipeline.""" + """Full result of the colony-detection pipeline for a single plate.""" cropped: NDArray[Any] contrast: NDArray[np.floating[Any]] @@ -135,6 +135,20 @@ def to_dataframe(self, plate_index: int = 0) -> pd.DataFrame: return pd.DataFrame(rows) +@dataclass +class ColonyPipelineResult: + """Aggregated colony-detection results across all plates in a scan.""" + + results: list[ColonyDetectionResult] + + @property + def summaries(self) -> list[dict[str, Any]]: + return [r.summary() for r in self.results] + + def to_dataframes(self) -> list[pd.DataFrame]: + return [r.to_dataframe(plate_index=i + 1) for i, r in enumerate(self.results)] + + # ------------------------------------------------------------------ # Pipeline steps # ------------------------------------------------------------------ @@ -258,7 +272,7 @@ def measure_colonies( mm_per_px = 25.4 / dpi min_area_px = int(np.ceil(_MIN_COLONY_AREA_MM2 / mm_per_px**2)) - mask = ski.morphology.remove_small_objects(mask, min_size=min_area_px) + mask = ski.morphology.remove_small_objects(mask, max_size=min_area_px) labels = ski.segmentation.clear_border(mask) cleaned_mask: NDArray[np.bool_] = labels > 0 labels = ski.measure.label(cleaned_mask) @@ -359,3 +373,26 @@ def detect_colonies(plate_image: NDArray[Any], dpi: int) -> ColonyDetectionResul mask=cleaned_mask, colonies=colonies, ) + + +def run_colony_pipeline( + plate_crops: list[NDArray[np.uint8]], + dpi: int, +) -> ColonyPipelineResult: + """Run colony detection on every plate crop and return aggregated results. + + This is the single entry point for colony detection across all plates in + a scan. Both the Lambda handler and the CLI delegate to this function. + + Args: + plate_crops: RGB uint8 plate images from + :meth:`~data_hub_lambda.epson_v700_scanner.image_processing.TiffProcessor.crop_plates`. + dpi: Image resolution in dots-per-inch. + """ + results = [detect_colonies(crop, dpi=dpi) for crop in plate_crops] + logger.info( + "Colony pipeline complete for %d plate(s): %s", + len(results), + [r.summary()["colony_count"] for r in results], + ) + return ColonyPipelineResult(results=results) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index ba30c9c2..c7ac130d 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -9,7 +9,9 @@ import tifffile from numpy.typing import NDArray -from data_hub_lambda.epson_v700_scanner.colony_detection import MARGIN_PX +from data_hub_lambda.epson_v700_scanner.colony_detection import ( + ColonyDetectionResult, +) logger = logging.getLogger(__name__) @@ -123,7 +125,7 @@ def intensities(self) -> NDArray[Any]: def export_jpg( self, - colony_masks: list[NDArray[np.bool_]] | None = None, + colony_results: list[ColonyDetectionResult] | None = None, ) -> Path: """Detect plates, draw overlays, resize, and write a JPEG. @@ -131,9 +133,9 @@ def export_jpg( (the pre-detection fallback behaviour). Args: - colony_masks: Optional per-plate binary masks (one per entry in - ``plate_boxes``). When provided, colony bounding boxes - and labels are drawn on the export image. + colony_results: Optional per-plate detection results (one per + entry in ``plate_boxes``). When provided, colony bounding + boxes and labels are drawn on the export image. """ img = self._to_rgb_uint8(self.intensities) if self.plate_boxes is None: @@ -141,8 +143,13 @@ def export_jpg( if self.plate_boxes: img = self._draw_plate_overlays(img, self.plate_boxes) - if colony_masks: - img = self._draw_colony_bboxes(img, self.plate_boxes, colony_masks) + if colony_results: + img = self._draw_colony_bboxes( + img, + self.plate_boxes, + colony_results, + dpi=self.dpi, + ) img = self._resize(img) @@ -293,14 +300,15 @@ def _draw_plate_overlays( def _draw_colony_bboxes( img: NDArray[np.uint8], boxes: list[_PlateBox], - colony_masks: list[NDArray[np.bool_]], - margin_px: int = MARGIN_PX, + colony_results: list[ColonyDetectionResult], + dpi: int, ) -> NDArray[np.uint8]: """Draw colony bounding boxes and labels onto *img* for each plate. - Each mask lives in the margin-cropped coordinate space of its - plate crop. Bounding boxes are offset by the plate box origin - plus the crop margin to map into full-image coordinates. + Uses the pre-computed :class:`ColonyProperties` from each + detection result rather than re-labelling the masks. Bounding + boxes (stored in mm, plate-crop-relative) are converted back to + full-image pixel coordinates. Uses matplotlib for anti-aliased rectangle and text rendering. """ @@ -308,9 +316,10 @@ def _draw_colony_bboxes( from matplotlib.figure import Figure from matplotlib.patches import Rectangle + mm_per_px = 25.4 / dpi img_h, img_w = img.shape[:2] - dpi = 100 - fig = Figure(figsize=(img_w / dpi, img_h / dpi), dpi=dpi) + fig_dpi = 100 + fig = Figure(figsize=(img_w / fig_dpi, img_h / fig_dpi), dpi=fig_dpi) canvas = FigureCanvasAgg(fig) ax = fig.add_subplot(1, 1, 1) ax.imshow(img) @@ -320,21 +329,16 @@ def _draw_colony_bboxes( bbox_color = tuple(c / 255.0 for c in _COLONY_BBOX_COLOR) - for box, mask in zip(boxes, colony_masks, strict=True): + for box, result in zip(boxes, colony_results, strict=True): plate_min_row, plate_min_col, _max_row, _max_col = box - row_offset = plate_min_row + margin_px - col_offset = plate_min_col + margin_px - - labels = ski.measure.label(mask) - regions = ski.measure.regionprops(labels) - show_labels = len(regions) <= _COLONY_LABEL_MAX_COUNT + show_labels = len(result.colonies) <= _COLONY_LABEL_MAX_COUNT - for region in regions: - min_r, min_c, max_r, max_c = region.bbox - r0 = min_r + row_offset - c0 = min_c + col_offset - r1 = max_r + row_offset - c1 = max_c + col_offset + for colony in result.colonies: + min_r_mm, min_c_mm, max_r_mm, max_c_mm = colony.bbox_mm + r0 = min_r_mm / mm_per_px + plate_min_row + c0 = min_c_mm / mm_per_px + plate_min_col + r1 = max_r_mm / mm_per_px + plate_min_row + c1 = max_c_mm / mm_per_px + plate_min_col rect = Rectangle( (c0, r0), @@ -349,7 +353,7 @@ def _draw_colony_bboxes( ax.text( (c0 + c1) / 2, r0 - 4, - str(region.label), + str(colony.label), color=bbox_color, fontsize=_COLONY_LABEL_FONT_SIZE, fontweight="bold", diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 78a6e330..5dd59382 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -52,26 +52,18 @@ def process_file(run_id: str, filename: str) -> None: processor.load() processor.detect_plates() - colony_masks: list | None = None - colony_summaries: list | None = None - dataframes: list | None = None + pipeline = None plate_crops = processor.crop_plates() if plate_crops: from data_hub_lambda.epson_v700_scanner.colony_detection import ( - detect_colonies, + run_colony_pipeline, ) - dpi = processor.dpi - colony_summaries = [] - colony_masks = [] - dataframes = [] - for i, crop in enumerate(plate_crops): - result = detect_colonies(crop, dpi=dpi) - colony_summaries.append(result.summary()) - colony_masks.append(result.mask) - dataframes.append(result.to_dataframe(plate_index=i + 1)) + pipeline = run_colony_pipeline(plate_crops, dpi=processor.dpi) - jpg_file_path = processor.export_jpg(colony_masks=colony_masks) + jpg_file_path = processor.export_jpg( + colony_results=pipeline.results if pipeline else None, + ) processed_bucket = config.AWS_S3_PROCESSED_DATA_BUCKET jpg_s3_key = f"{INSTRUMENT_ID}/{run_id}/{jpg_file_path.name}" @@ -94,13 +86,16 @@ def process_file(run_id: str, filename: str) -> None: metadata = processor.parse_metadata() - if plate_crops and dataframes and colony_summaries: + if pipeline: from data_hub_lambda.epson_v700_scanner.colony_detection import ( export_colony_csv, ) csv_name = f"{processor.path.stem}_colonies.csv" - csv_path = export_colony_csv(dataframes, raw_data_dir / csv_name) + csv_path = export_colony_csv( + pipeline.to_dataframes(), + raw_data_dir / csv_name, + ) csv_s3_key = f"{INSTRUMENT_ID}/{run_id}/{csv_name}" s3_utils.upload_file(csv_path, f"s3://{processed_bucket}/{csv_s3_key}") csv_file = client.create_file( @@ -117,12 +112,7 @@ def process_file(run_id: str, filename: str) -> None: content_type="text/csv", ) - metadata["colony_detection"] = colony_summaries - logger.info( - "Colony detection complete for %d plate(s): %s", - len(plate_crops), - [r["colony_count"] for r in colony_summaries], - ) + metadata["colony_detection"] = pipeline.summaries logger.info("Parsed metadata: %s", metadata) diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index 159e1c14..b68fd5f9 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -206,7 +206,7 @@ def test_single_blob(self) -> None: def test_two_blobs_sorted_by_area(self) -> None: mask = np.zeros((200, 200), dtype=bool) - mask[10:20, 10:20] = True # 100 px + mask[10:22, 10:22] = True # 144 px (above min-area threshold at 1200 DPI) mask[50:80, 50:80] = True # 900 px rgb = np.full((200, 200, 3), 128, dtype=np.uint8) colonies, _ = measure_colonies(mask, rgb, dpi=_TEST_DPI) From 13d35e8e224254dd107519a313365dd404ebd1a9 Mon Sep 17 00:00:00 2001 From: lanery Date: Mon, 18 May 2026 14:33:17 -0700 Subject: [PATCH 27/27] Fix RGBA conversion, cache RGB, harden parse_metadata, and add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Guard _to_rgb_uint8 against unintended rescale_intensity on RGBA uint8 inputs by tracking the original dtype before rgba2rgb conversion. - Cache _to_rgb_uint8 result on TiffProcessor to avoid redundant ~100 MB allocations across detect_plates/crop_plates/export_jpg. - Make parse_metadata() call detect_plates() itself when plate_boxes is None, removing the order-dependent API. - Fix variable shadowing in measure_colonies (labels → cleared_mask). - Promote lazy colony_detection imports to top-level in process_file.py. - Update module docstring to document all 6 pipeline steps. - Add test for sub-threshold colony removal. - Add smoke tests for _draw_plate_overlays and _draw_colony_bboxes. Co-authored-by: Cursor --- .../epson_v700_scanner/colony_detection.py | 9 ++-- .../epson_v700_scanner/image_processing.py | 40 ++++++++------ .../epson_v700_scanner/process_file.py | 12 ++--- .../test_colony_detection.py | 13 +++++ .../test_image_processing.py | 52 +++++++++++++++++++ 5 files changed, 98 insertions(+), 28 deletions(-) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py index 5e5702d5..9b3632bf 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/colony_detection.py @@ -10,6 +10,8 @@ 3. Decide whether colonies are present (contrast above noise floor). 4. Difference-of-Gaussians band-pass filter with 10th-percentile subtraction. 5. Otsu threshold to produce a binary colony mask. +6. Morphological cleanup: remove sub-threshold objects and border-touching + regions, then label and measure surviving colonies. """ from __future__ import annotations @@ -273,9 +275,8 @@ def measure_colonies( min_area_px = int(np.ceil(_MIN_COLONY_AREA_MM2 / mm_per_px**2)) mask = ski.morphology.remove_small_objects(mask, max_size=min_area_px) - labels = ski.segmentation.clear_border(mask) - cleaned_mask: NDArray[np.bool_] = labels > 0 - labels = ski.measure.label(cleaned_mask) + cleared_mask = ski.segmentation.clear_border(mask) + labels = ski.measure.label(cleared_mask) regions = ski.measure.regionprops(labels, intensity_image=rgb_image) colonies: list[ColonyProperties] = [] @@ -308,7 +309,7 @@ def measure_colonies( ) colonies.sort(key=lambda c: c.area_mm2, reverse=True) - return colonies, cleaned_mask + return colonies, cleared_mask # ------------------------------------------------------------------ diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py index c7ac130d..c3fdd04a 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/image_processing.py @@ -96,6 +96,7 @@ def __init__(self, path: Path) -> None: self.path = path self._intensities: NDArray[Any] | None = None self._dpi: int | None = None + self._rgb_cache: NDArray[np.uint8] | None = None self.plate_boxes: list[_PlateBox] | None = None @property @@ -137,7 +138,7 @@ def export_jpg( entry in ``plate_boxes``). When provided, colony bounding boxes and labels are drawn on the export image. """ - img = self._to_rgb_uint8(self.intensities) + img = self._get_rgb_uint8().copy() if self.plate_boxes is None: self.detect_plates(img) @@ -164,11 +165,10 @@ def parse_metadata(self) -> dict[str, Any]: - ``dpi``: integer DPI from ``XResolution``. - ``color_mode``: ``"rgb"`` or ``"bw"``. - - ``plate_count``: number of detected plates (only present after - :meth:`export_jpg` has been called). + - ``plate_count``: number of detected plates (runs detection if + not already performed). - ``plate_boxes``: list of ``[min_row, min_col, max_row, max_col]`` - bounding boxes in original-image coordinates (only present after - :meth:`export_jpg` has been called). + bounding boxes in original-image coordinates. """ metadata: dict[str, Any] = {} with tifffile.TiffFile(self.path) as tif: @@ -195,14 +195,11 @@ def parse_metadata(self) -> dict[str, Any]: if color_mode is not None: metadata["color_mode"] = color_mode - if self.plate_boxes is not None: - metadata["plate_count"] = len(self.plate_boxes) - metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] - else: - logger.warning( - "plate_boxes is None; call export_jpg() before" - " parse_metadata() to include plate data" - ) + if self.plate_boxes is None: + self.detect_plates() + assert self.plate_boxes is not None # populated by detect_plates + metadata["plate_count"] = len(self.plate_boxes) + metadata["plate_boxes"] = [list(b) for b in self.plate_boxes] return metadata @@ -223,7 +220,7 @@ def detect_plates(self, img: NDArray[np.uint8] | None = None) -> None: derived from ``self.intensities``. """ if img is None: - img = self._to_rgb_uint8(self.intensities) + img = self._get_rgb_uint8() h_orig, w_orig = img.shape[:2] s = _DETECTION_DOWNSAMPLE small: NDArray[np.uint8] = img[::s, ::s] @@ -375,7 +372,7 @@ def crop_plates(self) -> list[NDArray[np.uint8]]: """ if self.plate_boxes is None: raise RuntimeError("Call export_jpg() or detect_plates() first.") - img = self._to_rgb_uint8(self.intensities) + img = self._get_rgb_uint8() crops: list[NDArray[np.uint8]] = [] for min_row, min_col, max_row, max_col in self.plate_boxes: crops.append(img[min_row:max_row, min_col:max_col].copy()) @@ -385,14 +382,25 @@ def crop_plates(self) -> list[NDArray[np.uint8]]: # Internal helpers # ------------------------------------------------------------------ + def _get_rgb_uint8(self) -> NDArray[np.uint8]: + """Return the cached RGB uint8 version of the loaded intensities.""" + if self._rgb_cache is None: + self._rgb_cache = self._to_rgb_uint8(self.intensities) + return self._rgb_cache + @staticmethod def _to_rgb_uint8(img: NDArray[Any]) -> NDArray[np.uint8]: """Normalize to 8-bit RGB regardless of input dtype/channels.""" + orig_dtype = img.dtype + if img.ndim == 3 and img.shape[2] == 4: img = ski.color.rgba2rgb(img) - if img.dtype != np.uint8: + if orig_dtype != np.uint8: img = ski.util.img_as_ubyte(ski.exposure.rescale_intensity(img)) + elif img.dtype != np.uint8: + # rgba2rgb converted uint8 -> float; scale back without rescaling + img = ski.util.img_as_ubyte(img) if img.ndim == 2: img = ski.color.gray2rgb(img) diff --git a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py index 5dd59382..10c36c3f 100644 --- a/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py +++ b/lambda/src/data_hub_lambda/epson_v700_scanner/process_file.py @@ -2,6 +2,10 @@ import logging from data_hub_lambda.api_client import get_client +from data_hub_lambda.epson_v700_scanner.colony_detection import ( + export_colony_csv, + run_colony_pipeline, +) from data_hub_lambda.epson_v700_scanner.image_processing import TiffProcessor from data_hub_shared import s3_utils from data_hub_shared.config import config @@ -55,10 +59,6 @@ def process_file(run_id: str, filename: str) -> None: pipeline = None plate_crops = processor.crop_plates() if plate_crops: - from data_hub_lambda.epson_v700_scanner.colony_detection import ( - run_colony_pipeline, - ) - pipeline = run_colony_pipeline(plate_crops, dpi=processor.dpi) jpg_file_path = processor.export_jpg( @@ -87,10 +87,6 @@ def process_file(run_id: str, filename: str) -> None: metadata = processor.parse_metadata() if pipeline: - from data_hub_lambda.epson_v700_scanner.colony_detection import ( - export_colony_csv, - ) - csv_name = f"{processor.path.stem}_colonies.csv" csv_path = export_colony_csv( pipeline.to_dataframes(), diff --git a/lambda/tests/epson_v700_scanner/test_colony_detection.py b/lambda/tests/epson_v700_scanner/test_colony_detection.py index b68fd5f9..f486fa46 100644 --- a/lambda/tests/epson_v700_scanner/test_colony_detection.py +++ b/lambda/tests/epson_v700_scanner/test_colony_detection.py @@ -248,6 +248,19 @@ def test_mean_rgb_populated(self) -> None: assert abs(colony.mean_rgb[1] - 100.0) < 1.0 assert abs(colony.mean_rgb[2] - 50.0) < 1.0 + def test_subthreshold_colony_removed(self) -> None: + """Objects smaller than _MIN_COLONY_AREA_MM2 should be discarded.""" + mask = np.zeros((200, 200), dtype=bool) + # Small blob: 5x5 = 25 px² (well below 112 px threshold at 1200 DPI) + mask[90:95, 90:95] = True + # Large blob: 20x25 = 500 px² (well above threshold) + mask[30:50, 30:55] = True + rgb = np.full((200, 200, 3), 128, dtype=np.uint8) + colonies, cleaned = measure_colonies(mask, rgb, dpi=_TEST_DPI) + assert len(colonies) == 1 + assert colonies[0].area_mm2 > 0.05 + assert not cleaned[90:95, 90:95].any() + def test_border_colonies_excluded(self) -> None: mask = np.zeros((100, 100), dtype=bool) mask[0:10, 40:60] = True # touches top border diff --git a/lambda/tests/epson_v700_scanner/test_image_processing.py b/lambda/tests/epson_v700_scanner/test_image_processing.py index 834959e9..0bd1e550 100644 --- a/lambda/tests/epson_v700_scanner/test_image_processing.py +++ b/lambda/tests/epson_v700_scanner/test_image_processing.py @@ -270,3 +270,55 @@ def test_metadata_includes_plate_count(self, tmp_path: Path) -> None: assert metadata["plate_count"] == 1 assert len(metadata["plate_boxes"]) == 1 + + +class TestDrawPlateOverlays: + def test_shape_and_dtype_preserved(self) -> None: + img = np.random.randint(0, 255, (200, 300, 3), dtype=np.uint8) + boxes = [(50, 50, 150, 250)] + result = TiffProcessor._draw_plate_overlays(img, boxes) + assert result.shape == img.shape + assert result.dtype == np.uint8 + + def test_interior_pixels_unchanged(self) -> None: + img = np.full((200, 300, 3), 128, dtype=np.uint8) + boxes = [(50, 50, 150, 250)] + result = TiffProcessor._draw_plate_overlays(img, boxes) + np.testing.assert_array_equal(result[50:150, 50:250], img[50:150, 50:250]) + + def test_border_pixels_modified(self) -> None: + img = np.full((200, 300, 3), 128, dtype=np.uint8) + boxes = [(50, 50, 150, 250)] + result = TiffProcessor._draw_plate_overlays(img, boxes) + assert not np.array_equal(result[42:50, 50:250], img[42:50, 50:250]) + + +class TestDrawColonyBboxes: + def test_shape_and_dtype_preserved(self) -> None: + from data_hub_lambda.epson_v700_scanner.colony_detection import ( + ColonyDetectionResult, + ColonyProperties, + ) + + img = np.random.randint(0, 255, (400, 400, 3), dtype=np.uint8) + boxes = [(50, 50, 350, 350)] + colony = ColonyProperties( + label=1, + area_mm2=1.0, + centroid_row_mm=5.0, + centroid_col_mm=5.0, + bbox_mm=(4.0, 4.0, 6.0, 6.0), + eccentricity=0.0, + equivalent_diameter_mm=1.13, + mean_rgb=(128.0, 128.0, 128.0), + ) + result_obj = ColonyDetectionResult( + cropped=img[50:350, 50:350], + contrast=np.zeros((300, 300)), + has_colonies=True, + mask=np.zeros((300, 300), dtype=bool), + colonies=[colony], + ) + result = TiffProcessor._draw_colony_bboxes(img, boxes, [result_obj], dpi=600) + assert result.shape == img.shape + assert result.dtype == np.uint8