From ae2dbf5c619b372bf3a57a820136aa3792a4f449 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 10 Oct 2025 14:06:26 +0100 Subject: [PATCH 01/21] Add indexing to service --- src/ffs/service.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/ffs/service.py b/src/ffs/service.py index b11ae383..617d2df9 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -13,11 +13,16 @@ from pathlib import Path from typing import Iterator, Optional +import gemmi +import numpy as np import workflows.recipe from pydantic import BaseModel, ValidationError from rich.logging import RichHandler from workflows.services.common_service import CommonService +import ffs.index +from ffs.ssx_index import GPUIndexer + logger = logging.getLogger(__name__) logger.level = logging.DEBUG @@ -38,14 +43,19 @@ class PiaRequest(BaseModel): detector_distance: float d_min: float | None = None d_max: float | None = None + cell: tuple[float, float, float, float, float, float] | None = None class DetectorGeometry(BaseModel): pixel_size_x: float = 0.075 # Default value for Eiger pixel_size_y: float = 0.075 # Default value for Eiger + image_size_x: int = 2068 # Default value for Eiger + image_size_y: int = 2162 # Default value for Eiger distance: float beam_center_x: float beam_center_y: float + thickness: float = 0.45 # Default value for Eiger (Silicon) + mu: float = 3.9220781 # Default value for Eiger (Silicon) def to_json(self): return json.dumps(self.dict(), indent=4) @@ -175,6 +185,15 @@ def initializing(self): ) self._spotfinder_executable = _find_spotfinder() self._order_resolver = MessageOrderResolver(self.log) + ## Initialise the fast-feedback-indexer + self.indexer = None + self.output_for_index = False # Only turn on when we have confirmed all the things we need (cell, etc) + try: + self.indexer = GPUIndexer() + except ModuleNotFoundError: + self.log.debug( + "ffbidx not found, has the fast-feedback-indexer module been built and sourced?" + ) def gpu_per_image_analysis( self, @@ -204,6 +223,30 @@ def gpu_per_image_analysis( f"Rejecting PIA request for {parameters.dcgid}/{parameters.message_index}({parameters.dcid}): Invalid detector parameters \n{e}" ) + if self.indexer and parameters.cell and parameters.wavelength: + ## We have all we need to index, so make up to date models. + cell = gemmi.UnitCell(*parameters.cell) + self.indexer.cell = np.reshape( + np.array(cell.orth.mat, dtype="float32"), (3, 3) + ) ## Cell as an orthogonalisation matrix + ## convert beam centre to correct units (given in mm, want in px). + self.indexer.panel = ffs.index.make_panel( + detector_geometry.distance, + detector_geometry.beam_center_x / detector_geometry.pixel_size_x, + detector_geometry.beam_center_y / detector_geometry.pixel_size_y, + detector_geometry.pixel_size_x, + detector_geometry.pixel_size_y, + detector_geometry.image_size_x, + detector_geometry.image_size_y, + detector_geometry.thickness, + detector_geometry.mu, + ) + self.indexer.wavelength = parameters.wavelength + self.output_for_index = ( + True # The indexer has been configured, so can run the spotfinder + ) + # with --output-for-index and capture the results in read_and_send. + start_time = time.monotonic() self.log.info( f"Gotten PIA request for {parameters.dcgid}/{parameters.message_index}({parameters.dcid}): {parameters.filename}/:{parameters.start_frame_index}-{parameters.start_frame_index + parameters.number_of_frames}" @@ -271,6 +314,8 @@ def gpu_per_image_analysis( command.extend(["--dmin", str(parameters.d_min)]) if parameters.d_max: command.extend(["--dmax", str(parameters.d_max)]) + if self.output_for_index: + command.extend(["--output-for-index"]) self.log.info(f"Running: {' '.join(str(x) for x in command)}") @@ -309,6 +354,14 @@ def read_and_send() -> None: data["file-seen-at"] = time.time() # XRC has one-based-indexing data["file-number"] += 1 + ## Do indexing + if self.output_for_index: + xyzobs_px = np.array(data["spot_centers"]) + indexing_result = self.indexer.index(xyzobs_px) + self.log.info(indexing_result.model_dump_json(indent=2)) + result = indexing_result.model_dump() + data.update(result) + del data["spot_centers"] # don't send this data array onwards. self.log.info(f"Sending: {data}") rw.set_default_channel("result") rw.send_to("result", data) From 67b6e52f42e0ab1ba5410cba428b616492e1429c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 13:57:29 +0000 Subject: [PATCH 02/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/docker-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3f53646d..a05e3711 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -2,9 +2,9 @@ name: Docker Image CI on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] jobs: @@ -13,6 +13,6 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Build the Docker image - run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) + - uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) From 42e103c8e9a32ed089f40b2df406d54fea8a23b4 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Thu, 23 Oct 2025 13:35:45 +0100 Subject: [PATCH 03/21] Add unit cell validator to convert input from ispyb --- src/ffs/service.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/ffs/service.py b/src/ffs/service.py index 617d2df9..bd774021 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -12,6 +12,7 @@ from datetime import datetime from pathlib import Path from typing import Iterator, Optional +import pydantic import gemmi import numpy as np @@ -43,7 +44,21 @@ class PiaRequest(BaseModel): detector_distance: float d_min: float | None = None d_max: float | None = None - cell: tuple[float, float, float, float, float, float] | None = None + unit_cell: tuple[float, float, float, float, float, float] | None = None + + @pydantic.validator("unit_cell", pre=True) + def check_unit_cell(cls, v): + if not v: + return None + orig_v = v + if isinstance(v, str): + v = v.replace(",", " ").split() + v = [float(v) for v in v] + try: + assert len(v) == 6 + except Exception: + raise ValueError(f"Invalid unit_cell {orig_v}") + return v class DetectorGeometry(BaseModel): @@ -223,9 +238,9 @@ def gpu_per_image_analysis( f"Rejecting PIA request for {parameters.dcgid}/{parameters.message_index}({parameters.dcid}): Invalid detector parameters \n{e}" ) - if self.indexer and parameters.cell and parameters.wavelength: + if self.indexer and parameters.unit_cell and parameters.wavelength: ## We have all we need to index, so make up to date models. - cell = gemmi.UnitCell(*parameters.cell) + cell = gemmi.UnitCell(*parameters.unit_cell) self.indexer.cell = np.reshape( np.array(cell.orth.mat, dtype="float32"), (3, 3) ) ## Cell as an orthogonalisation matrix From b2640aeae2dab4268af515ea70d42677483edbec Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:51:41 +0100 Subject: [PATCH 04/21] Updates to detector metadata handling --- src/ffs/service.py | 72 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/src/ffs/service.py b/src/ffs/service.py index bd774021..736fb936 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -11,13 +11,13 @@ import time from datetime import datetime from pathlib import Path -from typing import Iterator, Optional +from typing import Iterator, Optional, Union, Literal import pydantic import gemmi import numpy as np import workflows.recipe -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, ValidationError, Field from rich.logging import RichHandler from workflows.services.common_service import CommonService @@ -45,6 +45,7 @@ class PiaRequest(BaseModel): d_min: float | None = None d_max: float | None = None unit_cell: tuple[float, float, float, float, float, float] | None = None + detector: str = "Eiger16M" @pydantic.validator("unit_cell", pre=True) def check_unit_cell(cls, v): @@ -52,6 +53,7 @@ def check_unit_cell(cls, v): return None orig_v = v if isinstance(v, str): + v = v.replace(",", " ").split() v = [float(v) for v in v] try: @@ -61,19 +63,58 @@ def check_unit_cell(cls, v): return v -class DetectorGeometry(BaseModel): +class DetectorParameters(BaseModel): + type: str + thickness: float + material: str + pixel_size_x: float + pixel_size_y: float + image_size_x: int + image_size_y: int + _mu_cache: dict = {} + + class Config: + extra = "forbid" # Don't allow instantiation of this base class + + def calculate_mu(self, wavelength: float) -> float: + if wavelength not in self._mu_cache: + self._mu_cache[wavelength] = calculate_mu_from_parameters( + self.thickness, self.material, wavelength + ) + return self._mu_cache[wavelength] + + +class Eiger16M(DetectorParameters): + type: Literal["Eiger16M"] + thickness: float = 0.45 + material: str = "Si" # atomic no 14 pixel_size_x: float = 0.075 # Default value for Eiger pixel_size_y: float = 0.075 # Default value for Eiger image_size_x: int = 2068 # Default value for Eiger image_size_y: int = 2162 # Default value for Eiger + mu: float = 3.9220781 # Default value for Eiger (Silicon) #FIXME replace with mu calculation + + +class Eiger9M(DetectorParameters): + type: Literal["Eiger9M"] + thickness: float = 0.45 + material: str = "Si" + pixel_size_x: float = 0.075 # Default value for Eiger + pixel_size_y: float = 0.075 # Default value for Eiger + image_size_x: int = 1000 # Default value for Eiger9M #FIXME + image_size_y: int = 1000 # Default value for Eiger9M #FIXME + mu: float = 3.9220781 # Default value for Eiger (Silicon) #FIXME replace with mu calculation + +class DetectorGeometry(BaseModel): distance: float beam_center_x: float beam_center_y: float - thickness: float = 0.45 # Default value for Eiger (Silicon) - mu: float = 3.9220781 # Default value for Eiger (Silicon) + detector: Union[Eiger9M, Eiger16M] = Field(..., discriminator="type") def to_json(self): - return json.dumps(self.dict(), indent=4) + d = self.dict() + d.update(self.detector.dict()) + return json.dumps(d, indent=4) def _setup_rich_logging(level=logging.DEBUG): @@ -231,6 +272,7 @@ def gpu_per_image_analysis( distance=parameters.detector_distance, beam_center_x=parameters.xBeam, beam_center_y=parameters.yBeam, + detector={"type":parameters.detector}, ) self.log.debug("{detector_geometry.to_json()=}") except ValidationError as e: @@ -245,16 +287,18 @@ def gpu_per_image_analysis( np.array(cell.orth.mat, dtype="float32"), (3, 3) ) ## Cell as an orthogonalisation matrix ## convert beam centre to correct units (given in mm, want in px). + px_size_x = detector_geometry.detector.pixel_size_x + px_size_y = detector_geometry.detector.pixel_size_y self.indexer.panel = ffs.index.make_panel( detector_geometry.distance, - detector_geometry.beam_center_x / detector_geometry.pixel_size_x, - detector_geometry.beam_center_y / detector_geometry.pixel_size_y, - detector_geometry.pixel_size_x, - detector_geometry.pixel_size_y, - detector_geometry.image_size_x, - detector_geometry.image_size_y, - detector_geometry.thickness, - detector_geometry.mu, + detector_geometry.beam_center_x / px_size_x, + detector_geometry.beam_center_y / px_size_y, + px_size_x, + px_size_y, + detector_geometry.detector.image_size_x, + detector_geometry.detector.image_size_y, + detector_geometry.detector.thickness, + detector_geometry.detector.mu, ) self.indexer.wavelength = parameters.wavelength self.output_for_index = ( From 62f366cc165c5e2caa6d52682796b88e276c53f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Oct 2025 13:52:50 +0000 Subject: [PATCH 05/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/ffs/service.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ffs/service.py b/src/ffs/service.py index 736fb936..b5f01a6d 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -11,13 +11,13 @@ import time from datetime import datetime from pathlib import Path -from typing import Iterator, Optional, Union, Literal -import pydantic +from typing import Iterator, Literal, Optional, Union import gemmi import numpy as np +import pydantic import workflows.recipe -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError from rich.logging import RichHandler from workflows.services.common_service import CommonService @@ -53,7 +53,6 @@ def check_unit_cell(cls, v): return None orig_v = v if isinstance(v, str): - v = v.replace(",", " ").split() v = [float(v) for v in v] try: @@ -74,7 +73,7 @@ class DetectorParameters(BaseModel): _mu_cache: dict = {} class Config: - extra = "forbid" # Don't allow instantiation of this base class + extra = "forbid" # Don't allow instantiation of this base class def calculate_mu(self, wavelength: float) -> float: if wavelength not in self._mu_cache: @@ -87,7 +86,7 @@ def calculate_mu(self, wavelength: float) -> float: class Eiger16M(DetectorParameters): type: Literal["Eiger16M"] thickness: float = 0.45 - material: str = "Si" # atomic no 14 + material: str = "Si" # atomic no 14 pixel_size_x: float = 0.075 # Default value for Eiger pixel_size_y: float = 0.075 # Default value for Eiger image_size_x: int = 2068 # Default value for Eiger @@ -105,6 +104,7 @@ class Eiger9M(DetectorParameters): image_size_y: int = 1000 # Default value for Eiger9M #FIXME mu: float = 3.9220781 # Default value for Eiger (Silicon) #FIXME replace with mu calculation + class DetectorGeometry(BaseModel): distance: float beam_center_x: float @@ -272,7 +272,7 @@ def gpu_per_image_analysis( distance=parameters.detector_distance, beam_center_x=parameters.xBeam, beam_center_y=parameters.yBeam, - detector={"type":parameters.detector}, + detector={"type": parameters.detector}, ) self.log.debug("{detector_geometry.to_json()=}") except ValidationError as e: From 8aa4aba7a0a222585ff018281e5a23c0230d1308 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 24 Oct 2025 12:05:03 +0100 Subject: [PATCH 06/21] Calculate mu based on detector --- baseline/indexer/index_module.cpp | 3 ++ dx2 | 2 +- src/ffs/service.py | 54 ++++++++++++++++++------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/baseline/indexer/index_module.cpp b/baseline/indexer/index_module.cpp index d2989fc7..e0fcb921 100644 --- a/baseline/indexer/index_module.cpp +++ b/baseline/indexer/index_module.cpp @@ -3,9 +3,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -220,6 +222,7 @@ NB_MODULE(index, m) { .def_prop_ro("delpsi", [](const IndexingResult &r) { return r.delpsi; }) .def_prop_ro("rmsds", [](const IndexingResult &r) { return r.rmsds; }); m.def("make_panel", &make_panel, "Create a configured Panel object"); + m.def("calculate_mu_for_material_at_wavelength", &calculate_mu_for_material_at_wavelength, "Calculate the absorption coefficient from material and wavelength"); m.def("ssx_xyz_to_rlp", &ssx_xyz_to_rlp, nb::arg("xyzobs_px"), diff --git a/dx2 b/dx2 index 393f538a..dd4f1fa0 160000 --- a/dx2 +++ b/dx2 @@ -1 +1 @@ -Subproject commit 393f538a7d730b9a815a456e362bf88c837648e8 +Subproject commit dd4f1fa0442b198f01e0785ec5ea0a029bec0d55 diff --git a/src/ffs/service.py b/src/ffs/service.py index b5f01a6d..c1a07674 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -11,13 +11,13 @@ import time from datetime import datetime from pathlib import Path -from typing import Iterator, Literal, Optional, Union +from typing import Iterator, Optional, Union, Literal +import pydantic import gemmi import numpy as np -import pydantic import workflows.recipe -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, ValidationError, Field from rich.logging import RichHandler from workflows.services.common_service import CommonService @@ -73,12 +73,12 @@ class DetectorParameters(BaseModel): _mu_cache: dict = {} class Config: - extra = "forbid" # Don't allow instantiation of this base class + extra = "forbid" # Don't allow instantiation of this base class def calculate_mu(self, wavelength: float) -> float: if wavelength not in self._mu_cache: - self._mu_cache[wavelength] = calculate_mu_from_parameters( - self.thickness, self.material, wavelength + self._mu_cache[wavelength] = ffs.index.calculate_mu_for_material_at_wavelength( + self.material, wavelength ) return self._mu_cache[wavelength] @@ -86,30 +86,37 @@ def calculate_mu(self, wavelength: float) -> float: class Eiger16M(DetectorParameters): type: Literal["Eiger16M"] thickness: float = 0.45 - material: str = "Si" # atomic no 14 - pixel_size_x: float = 0.075 # Default value for Eiger - pixel_size_y: float = 0.075 # Default value for Eiger - image_size_x: int = 2068 # Default value for Eiger - image_size_y: int = 2162 # Default value for Eiger - mu: float = 3.9220781 # Default value for Eiger (Silicon) #FIXME replace with mu calculation - + material: str = "Si" + pixel_size_x: float = 0.075 + pixel_size_y: float = 0.075 + image_size_x: int = 4148 + image_size_y: int = 4362 -class Eiger9M(DetectorParameters): - type: Literal["Eiger9M"] +class Eiger4M(DetectorParameters): + type: Literal["Eiger4M"] thickness: float = 0.45 material: str = "Si" - pixel_size_x: float = 0.075 # Default value for Eiger - pixel_size_y: float = 0.075 # Default value for Eiger - image_size_x: int = 1000 # Default value for Eiger9M #FIXME - image_size_y: int = 1000 # Default value for Eiger9M #FIXME - mu: float = 3.9220781 # Default value for Eiger (Silicon) #FIXME replace with mu calculation + pixel_size_x: float = 0.075 + pixel_size_y: float = 0.075 + image_size_x: int = 2068 + image_size_y: int = 2162 + + +class Eiger9MCdTe(DetectorParameters): + type: Literal["Eiger9MCdTe"] + thickness: float = 0.75 + material: str = "CdTe" + pixel_size_x: float = 0.075 + pixel_size_y: float = 0.075 + image_size_x: int = 3108 + image_size_y: int = 3262 class DetectorGeometry(BaseModel): distance: float beam_center_x: float beam_center_y: float - detector: Union[Eiger9M, Eiger16M] = Field(..., discriminator="type") + detector: Union[Eiger9MCdTe, Eiger16M, Eiger4M] = Field(..., discriminator="type") def to_json(self): d = self.dict() @@ -272,7 +279,7 @@ def gpu_per_image_analysis( distance=parameters.detector_distance, beam_center_x=parameters.xBeam, beam_center_y=parameters.yBeam, - detector={"type": parameters.detector}, + detector={"type" : parameters.detector}, ) self.log.debug("{detector_geometry.to_json()=}") except ValidationError as e: @@ -289,6 +296,7 @@ def gpu_per_image_analysis( ## convert beam centre to correct units (given in mm, want in px). px_size_x = detector_geometry.detector.pixel_size_x px_size_y = detector_geometry.detector.pixel_size_y + mu = detector_geometry.detector.calculate_mu(parameters.wavelength) self.indexer.panel = ffs.index.make_panel( detector_geometry.distance, detector_geometry.beam_center_x / px_size_x, @@ -298,7 +306,7 @@ def gpu_per_image_analysis( detector_geometry.detector.image_size_x, detector_geometry.detector.image_size_y, detector_geometry.detector.thickness, - detector_geometry.detector.mu, + mu, ) self.indexer.wavelength = parameters.wavelength self.output_for_index = ( From 40b16b84a83f854d8bb2200dc5c0a6e0f90fc155 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:57:05 +0100 Subject: [PATCH 07/21] Tidy pydantic models --- dx2 | 2 +- src/ffs/service.py | 49 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/dx2 b/dx2 index dd4f1fa0..deabef17 160000 --- a/dx2 +++ b/dx2 @@ -1 +1 @@ -Subproject commit dd4f1fa0442b198f01e0785ec5ea0a029bec0d55 +Subproject commit deabef177fa63dcd041998ec7053b040c0746ad8 diff --git a/src/ffs/service.py b/src/ffs/service.py index c1a07674..4c105c71 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -10,6 +10,7 @@ import threading import time from datetime import datetime +from enum import Enum from pathlib import Path from typing import Iterator, Optional, Union, Literal import pydantic @@ -17,7 +18,7 @@ import gemmi import numpy as np import workflows.recipe -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, ValidationError, Field, PrivateAttr from rich.logging import RichHandler from workflows.services.common_service import CommonService @@ -61,19 +62,37 @@ def check_unit_cell(cls, v): raise ValueError(f"Invalid unit_cell {orig_v}") return v +class Material(str, Enum): + Si = "Si" + CdTe = "CdTe" class DetectorParameters(BaseModel): - type: str + """ + Define a set of detector metadata that derived classes + need to provide. + This class is not to be instantiated directly. + """ + detector_type: str thickness: float - material: str + material: Material pixel_size_x: float pixel_size_y: float image_size_x: int image_size_y: int - _mu_cache: dict = {} - - class Config: - extra = "forbid" # Don't allow instantiation of this base class + # mu cache not serialized to dict/json, ok as mu not needed for spotfinder + _mu_cache: dict = PrivateAttr(default_factory=dict) + + def __init_subclass__(cls, **kwargs): + # enforce setting of defaults for all fields in subclasses. + super().__init_subclass__(**kwargs) + missing_defaults = [ + name for name, field in cls.__fields__.items() + if field.default is None and field.default_factory is None + ] + if missing_defaults: + raise TypeError( + f"{cls.__name__} must define default values for all fields: missing {missing_defaults}" + ) def calculate_mu(self, wavelength: float) -> float: if wavelength not in self._mu_cache: @@ -84,18 +103,18 @@ def calculate_mu(self, wavelength: float) -> float: class Eiger16M(DetectorParameters): - type: Literal["Eiger16M"] + detector_type: Literal["Eiger16M"] thickness: float = 0.45 - material: str = "Si" + material : Material = Material.Si pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 4148 image_size_y: int = 4362 class Eiger4M(DetectorParameters): - type: Literal["Eiger4M"] + detector_type: Literal["Eiger4M"] thickness: float = 0.45 - material: str = "Si" + material : Material = Material.Si pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 2068 @@ -103,9 +122,9 @@ class Eiger4M(DetectorParameters): class Eiger9MCdTe(DetectorParameters): - type: Literal["Eiger9MCdTe"] + detector_type: Literal["Eiger9MCdTe"] thickness: float = 0.75 - material: str = "CdTe" + material : Material = Material.CdTe pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 3108 @@ -116,7 +135,7 @@ class DetectorGeometry(BaseModel): distance: float beam_center_x: float beam_center_y: float - detector: Union[Eiger9MCdTe, Eiger16M, Eiger4M] = Field(..., discriminator="type") + detector: Union[Eiger9MCdTe, Eiger16M, Eiger4M] = Field(..., discriminator="detector_type") def to_json(self): d = self.dict() @@ -279,7 +298,7 @@ def gpu_per_image_analysis( distance=parameters.detector_distance, beam_center_x=parameters.xBeam, beam_center_y=parameters.yBeam, - detector={"type" : parameters.detector}, + detector={"detector_type" : parameters.detector}, ) self.log.debug("{detector_geometry.to_json()=}") except ValidationError as e: From 1f5d7260a6399c02aaf78cb1daf86fb9488c6a91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:58:01 +0000 Subject: [PATCH 08/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- baseline/indexer/index_module.cpp | 6 ++++-- src/ffs/service.py | 31 ++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/baseline/indexer/index_module.cpp b/baseline/indexer/index_module.cpp index e0fcb921..c782ae87 100644 --- a/baseline/indexer/index_module.cpp +++ b/baseline/indexer/index_module.cpp @@ -1,9 +1,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -222,7 +222,9 @@ NB_MODULE(index, m) { .def_prop_ro("delpsi", [](const IndexingResult &r) { return r.delpsi; }) .def_prop_ro("rmsds", [](const IndexingResult &r) { return r.rmsds; }); m.def("make_panel", &make_panel, "Create a configured Panel object"); - m.def("calculate_mu_for_material_at_wavelength", &calculate_mu_for_material_at_wavelength, "Calculate the absorption coefficient from material and wavelength"); + m.def("calculate_mu_for_material_at_wavelength", + &calculate_mu_for_material_at_wavelength, + "Calculate the absorption coefficient from material and wavelength"); m.def("ssx_xyz_to_rlp", &ssx_xyz_to_rlp, nb::arg("xyzobs_px"), diff --git a/src/ffs/service.py b/src/ffs/service.py index 4c105c71..a0e9ea5f 100644 --- a/src/ffs/service.py +++ b/src/ffs/service.py @@ -12,13 +12,13 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Iterator, Optional, Union, Literal -import pydantic +from typing import Iterator, Literal, Optional, Union import gemmi import numpy as np +import pydantic import workflows.recipe -from pydantic import BaseModel, ValidationError, Field, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr, ValidationError from rich.logging import RichHandler from workflows.services.common_service import CommonService @@ -62,16 +62,19 @@ def check_unit_cell(cls, v): raise ValueError(f"Invalid unit_cell {orig_v}") return v + class Material(str, Enum): Si = "Si" CdTe = "CdTe" + class DetectorParameters(BaseModel): """ Define a set of detector metadata that derived classes need to provide. This class is not to be instantiated directly. """ + detector_type: str thickness: float material: Material @@ -86,7 +89,8 @@ def __init_subclass__(cls, **kwargs): # enforce setting of defaults for all fields in subclasses. super().__init_subclass__(**kwargs) missing_defaults = [ - name for name, field in cls.__fields__.items() + name + for name, field in cls.__fields__.items() if field.default is None and field.default_factory is None ] if missing_defaults: @@ -96,8 +100,10 @@ def __init_subclass__(cls, **kwargs): def calculate_mu(self, wavelength: float) -> float: if wavelength not in self._mu_cache: - self._mu_cache[wavelength] = ffs.index.calculate_mu_for_material_at_wavelength( - self.material, wavelength + self._mu_cache[wavelength] = ( + ffs.index.calculate_mu_for_material_at_wavelength( + self.material, wavelength + ) ) return self._mu_cache[wavelength] @@ -105,16 +111,17 @@ def calculate_mu(self, wavelength: float) -> float: class Eiger16M(DetectorParameters): detector_type: Literal["Eiger16M"] thickness: float = 0.45 - material : Material = Material.Si + material: Material = Material.Si pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 4148 image_size_y: int = 4362 + class Eiger4M(DetectorParameters): detector_type: Literal["Eiger4M"] thickness: float = 0.45 - material : Material = Material.Si + material: Material = Material.Si pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 2068 @@ -124,7 +131,7 @@ class Eiger4M(DetectorParameters): class Eiger9MCdTe(DetectorParameters): detector_type: Literal["Eiger9MCdTe"] thickness: float = 0.75 - material : Material = Material.CdTe + material: Material = Material.CdTe pixel_size_x: float = 0.075 pixel_size_y: float = 0.075 image_size_x: int = 3108 @@ -135,7 +142,9 @@ class DetectorGeometry(BaseModel): distance: float beam_center_x: float beam_center_y: float - detector: Union[Eiger9MCdTe, Eiger16M, Eiger4M] = Field(..., discriminator="detector_type") + detector: Union[Eiger9MCdTe, Eiger16M, Eiger4M] = Field( + ..., discriminator="detector_type" + ) def to_json(self): d = self.dict() @@ -298,7 +307,7 @@ def gpu_per_image_analysis( distance=parameters.detector_distance, beam_center_x=parameters.xBeam, beam_center_y=parameters.yBeam, - detector={"detector_type" : parameters.detector}, + detector={"detector_type": parameters.detector}, ) self.log.debug("{detector_geometry.to_json()=}") except ValidationError as e: From 003d302e0b8afccc3090268951d9747ebdbf8a44 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Thu, 30 Oct 2025 14:04:27 +0000 Subject: [PATCH 09/21] Fetch and build ffbidx with cmake --- CMakeLists.txt | 31 ++++++++++++++++++++++++++++++- build.sh | 11 +++++++++++ tests/test_gpu_ssx_index.py | 4 ++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c32ffad1..9c15d617 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,9 +22,14 @@ include(AlwaysColourCompilation) include_directories(include) # Find Python -find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) include_directories(${Python3_INCLUDE_DIRS}) +# Make sure we can install ffbidx into correct location. +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Install path" FORCE) +endif() + # Dependency fetching set(FETCHCONTENT_QUIET OFF) include(FetchContent) @@ -48,6 +53,21 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(argparse) +# Required for Python module build of ffbidx +set(BUILD_FAST_INDEXER ON CACHE BOOL "" FORCE) +set(PYTHON_MODULE ON CACHE BOOL "" FORCE) +set(PYTHON_MODULE_RPATH ON CACHE BOOL "" FORCE) + +# Override install path for ffbidx to site-packages +set(PYTHON_MODULE_PATH ${Python3_SITEARCH} CACHE PATH "Python install path" FORCE) + +FetchContent_Declare( + ffbidx + GIT_REPOSITORY https://github.com/paulscherrerinstitute/fast-feedback-indexer.git + GIT_TAG main +) +FetchContent_MakeAvailable(ffbidx) + find_package(fmt REQUIRED) find_package(spdlog REQUIRED) @@ -128,4 +148,13 @@ add_subdirectory(h5read) add_subdirectory(baseline) add_subdirectory(spotfinder) +# define a ssx_index 'executable' that actually just runs the python code (for style consistency) configure_file(${CMAKE_SOURCE_DIR}/scripts/ssx_index ${CMAKE_BINARY_DIR}/bin/ssx_index COPYONLY) + +# run 'pip install .' to install the c++ extension for ssx_index +install(CODE " + execute_process( + COMMAND ${Python3_EXECUTABLE} -m pip install . + WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\" + ) +") diff --git a/build.sh b/build.sh index 1b937ff5..3218ed32 100755 --- a/build.sh +++ b/build.sh @@ -161,6 +161,17 @@ build_directory() { make -j"$JOBS" fi ) + + # Install + print_status "Installing $description..." + ( + cd "$build_dir" + if [[ "$BUILD_CMD" == "ninja" ]]; then + ninja "install" + else + make "install" -j"$JOBS" + fi + ) print_success "Successfully built $description" } diff --git a/tests/test_gpu_ssx_index.py b/tests/test_gpu_ssx_index.py index fea21144..a2cb39d3 100644 --- a/tests/test_gpu_ssx_index.py +++ b/tests/test_gpu_ssx_index.py @@ -157,8 +157,8 @@ def test_gpu_ssx_index(dials_data, tmp_path): cwd=tmp_path, ) assert not proc.stderr - assert tmp_path / "indexed.refl" - assert tmp_path / "indexed_crystals.json" + assert (tmp_path / "indexed.refl").exists() + assert (tmp_path / "indexed_crystals.json").exists() with open(tmp_path / "indexed_crystals.json", "r") as f: crystals = json.load(f) for i, (crystal, expected) in enumerate( From 1cd2892ce8d31dbcb8c830a28a443e9453bb95bc Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Mon, 2 Feb 2026 08:47:36 +0000 Subject: [PATCH 10/21] Add numpy to env to make available in docker image --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index bb4a1e71..f05a02e3 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,7 @@ dependencies: - hdf5=1.14.6 - hdf5-external-filter-plugins=0.1.0 - nlohmann_json=3.12.0 + - numpy=2.3.4 - pre-commit=4.3.0 - pydantic=2.12.3 - pytest=8.4.2 From d0ff4eb2b8d6f079e4b886f5717db908285dce9f Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:41:42 +0000 Subject: [PATCH 11/21] Update for ffbidx inclusion --- Dockerfile | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 55744e54..87cf2ea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,15 +26,30 @@ RUN micromamba create -y -f /opt/runtime-environment.yml -p /opt/ffs COPY . /opt/ffs_src # Build the C++/CUDA backend +# Make Numpy headers available for ffbidx build (via -DPython3_EXECUTABLE=/opt/build_env/bin/python) +# Make ffbidx install into runtime env (-DPython3_SITELIB=$RT_SITE \ -DPython3_SITEARCH=$RT_SITE) WORKDIR /opt/build -RUN cmake /opt/ffs_src -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/ffs -DHDF5_ROOT=/opt/ffs +RUN RT_SITE=$(/opt/ffs/bin/python -c "import site; print(site.getsitepackages()[0])") +RUN cmake /opt/ffs_src \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/ffs \ + -DHDF5_ROOT=/opt/ffs \ + -DPython3_EXECUTABLE=/opt/build_env/bin/python \ +  -DPython3_SITELIB=$RT_SITE \ + -DPython3_SITEARCH=$RT_SITE + RUN cmake --build . --target spotfinder --target spotfinder32 RUN cmake --install . --component Runtime # Build and install dx2 submodule WORKDIR /opt/build_dx2 -RUN cmake /opt/ffs_src/dx2 -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/ffs +RUN cmake /opt/ffs_src/dx2 \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/ffs \ + -DHDF5_ROOT=/opt/ffs RUN cmake --build . && cmake --install . # Install Python package From 0b8c3d58c549461f5584803f9ea4a6b9c654ff1e Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:30:32 +0000 Subject: [PATCH 12/21] No ifs --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd8c88d1..75b2b8f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,9 +26,7 @@ find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) include_directories(${Python3_INCLUDE_DIRS}) # Make sure we can install ffbidx into correct location. -if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Install path" FORCE) -endif() +set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Install path" FORCE) # Dependency fetching # Find packages that are in conda environment From 7964253e41d6b360f1520fafe71119fa992fb746 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:02:36 +0000 Subject: [PATCH 13/21] change tag for testing --- chart/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index 3db44ca4..275bdacf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/diamondlightsource/fast-feedback-service - tag: latest + tag: pr-71 # Beamline configuration beamline: "i24" From 31880ffd9e049ebca4701a4a1c1d449dc31d6078 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:08:42 +0000 Subject: [PATCH 14/21] add numpy to runtime env --- runtime-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime-environment.yml b/runtime-environment.yml index 6082a291..2786d48c 100644 --- a/runtime-environment.yml +++ b/runtime-environment.yml @@ -7,6 +7,7 @@ dependencies: - gemmi=0.7.3 - hdf5=1.14.6 - hdf5-external-filter-plugins=0.1.0 + - numpy=2.3.4 - python=3.13.7 - pip - zocalo From a2d43ee76db4ef85b6f03584762339bebad7f490 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:24:49 +0000 Subject: [PATCH 15/21] Revert "add numpy to runtime env" This reverts commit 31880ffd9e049ebca4701a4a1c1d449dc31d6078. --- runtime-environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-environment.yml b/runtime-environment.yml index 2786d48c..6082a291 100644 --- a/runtime-environment.yml +++ b/runtime-environment.yml @@ -7,7 +7,6 @@ dependencies: - gemmi=0.7.3 - hdf5=1.14.6 - hdf5-external-filter-plugins=0.1.0 - - numpy=2.3.4 - python=3.13.7 - pip - zocalo From 61f3830bca1b73492f1d1cb376fb5bc5898fc39b Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Wed, 4 Feb 2026 10:40:29 +0000 Subject: [PATCH 16/21] Make sure python install paths set --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 87cf2ea3..9ced296b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,15 +29,15 @@ COPY . /opt/ffs_src # Make Numpy headers available for ffbidx build (via -DPython3_EXECUTABLE=/opt/build_env/bin/python) # Make ffbidx install into runtime env (-DPython3_SITELIB=$RT_SITE \ -DPython3_SITEARCH=$RT_SITE) WORKDIR /opt/build -RUN RT_SITE=$(/opt/ffs/bin/python -c "import site; print(site.getsitepackages()[0])") -RUN cmake /opt/ffs_src \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/ffs \ - -DHDF5_ROOT=/opt/ffs \ - -DPython3_EXECUTABLE=/opt/build_env/bin/python \ -  -DPython3_SITELIB=$RT_SITE \ - -DPython3_SITEARCH=$RT_SITE +RUN RT_SITE=$(/opt/ffs/bin/python -c "import site; print(site.getsitepackages()[0])") && \ + cmake /opt/ffs_src \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/ffs \ + -DHDF5_ROOT=/opt/ffs \ + -DPython3_EXECUTABLE=/opt/build_env/bin/python \ +   -DPython3_SITELIB=$RT_SITE \ + -DPython3_SITEARCH=$RT_SITE RUN cmake --build . --target spotfinder --target spotfinder32 From 1460e952cb0314fa9a3762c48a1e596a7579a7fb Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:26:11 +0000 Subject: [PATCH 17/21] Install properly --- Dockerfile | 2 +- baseline/indexer/CMakeLists.txt | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ced296b..d5b9c2d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN RT_SITE=$(/opt/ffs/bin/python -c "import site; print(site.getsitepackages()[   -DPython3_SITELIB=$RT_SITE \ -DPython3_SITEARCH=$RT_SITE -RUN cmake --build . --target spotfinder --target spotfinder32 +RUN cmake --build . --target spotfinder --target spotfinder32 --target index RUN cmake --install . --component Runtime diff --git a/baseline/indexer/CMakeLists.txt b/baseline/indexer/CMakeLists.txt index d9fee60b..59c600e5 100644 --- a/baseline/indexer/CMakeLists.txt +++ b/baseline/indexer/CMakeLists.txt @@ -62,7 +62,9 @@ nanobind_add_module(index index_module.cpp) target_include_directories(index PRIVATE ${CMAKE_SOURCE_DIR}/baseline/predictor) target_link_libraries(index PRIVATE Eigen3::Eigen dx2 ${Python3_LIBRARIES}) -set(PYTHON_MODULE_DIR ${CMAKE_SOURCE_DIR}/src/ffs) -add_custom_command(TARGET index POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy $ ${PYTHON_MODULE_DIR} + +# Install into the runtime Python sitearch +install(TARGETS index + LIBRARY DESTINATION ${Python3_SITEARCH} + COMPONENT Runtime ) From cc348e81513bcf1d49457b579080f8a006482654 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 4 Feb 2026 15:01:35 +0000 Subject: [PATCH 18/21] Fix installation of indexing module, add ffbidx --- .dockerignore | 2 ++ CMakeLists.txt | 13 ++++--------- Dockerfile | 27 ++++++++------------------- baseline/indexer/CMakeLists.txt | 4 ++-- runtime-environment.yml | 1 + 5 files changed, 17 insertions(+), 30 deletions(-) diff --git a/.dockerignore b/.dockerignore index 687c8fe3..6a56063f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,7 @@ Makefile *.tar .*_cache ENV/ +*.so # This is better, but doesn't work with podman. sigh. # * # !**/CMakeLists.txt @@ -24,3 +25,4 @@ ENV/ # !**/*.cc # !**/*.cpp # !**/*.h + diff --git a/CMakeLists.txt b/CMakeLists.txt index 75b2b8f5..757f49e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,12 +21,7 @@ include(AlwaysColourCompilation) include_directories(include) -# Find Python -find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) -include_directories(${Python3_INCLUDE_DIRS}) - -# Make sure we can install ffbidx into correct location. -set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Install path" FORCE) +find_package(Python COMPONENTS Interpreter Development.Module NumPy REQUIRED) # Dependency fetching # Find packages that are in conda environment @@ -59,8 +54,8 @@ set(BUILD_FAST_INDEXER ON CACHE BOOL "" FORCE) set(PYTHON_MODULE ON CACHE BOOL "" FORCE) set(PYTHON_MODULE_RPATH ON CACHE BOOL "" FORCE) -# Override install path for ffbidx to site-packages -set(PYTHON_MODULE_PATH ${Python3_SITEARCH} CACHE PATH "Python install path" FORCE) +# ffbidx uses this to find site-packages +set(PYTHON_MODULE_PATH ${Python_SITEARCH} CACHE PATH "ffbidx python site-packages path" FORCE) FetchContent_Declare( ffbidx @@ -165,7 +160,7 @@ configure_file(${CMAKE_SOURCE_DIR}/scripts/ssx_index ${CMAKE_BINARY_DIR}/bin/ssx # run 'pip install .' to install the c++ extension for ssx_index install(CODE " execute_process( - COMMAND ${Python3_EXECUTABLE} -m pip install . + COMMAND ${Python_EXECUTABLE} -m pip install . WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\" ) ") diff --git a/Dockerfile b/Dockerfile index d5b9c2d1..5e9eff12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,38 +24,27 @@ RUN micromamba create -y -f /opt/runtime-environment.yml -p /opt/ffs # Copy source COPY . /opt/ffs_src - +ENV CMAKE_GENERATOR Ninja # Build the C++/CUDA backend # Make Numpy headers available for ffbidx build (via -DPython3_EXECUTABLE=/opt/build_env/bin/python) # Make ffbidx install into runtime env (-DPython3_SITELIB=$RT_SITE \ -DPython3_SITEARCH=$RT_SITE) WORKDIR /opt/build -RUN RT_SITE=$(/opt/ffs/bin/python -c "import site; print(site.getsitepackages()[0])") && \ - cmake /opt/ffs_src \ - -G Ninja \ +RUN cmake /opt/ffs_src \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/opt/ffs \ -DHDF5_ROOT=/opt/ffs \ - -DPython3_EXECUTABLE=/opt/build_env/bin/python \ -   -DPython3_SITELIB=$RT_SITE \ - -DPython3_SITEARCH=$RT_SITE - -RUN cmake --build . --target spotfinder --target spotfinder32 --target index + -DPython3_ROOT_DIR=/opt/ffs -RUN cmake --install . --component Runtime +RUN cmake --build . --target spotfinder --target spotfinder32 --target index --target ffbidx --target fast_indexer -# Build and install dx2 submodule -WORKDIR /opt/build_dx2 -RUN cmake /opt/ffs_src/dx2 \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/ffs \ - -DHDF5_ROOT=/opt/ffs -RUN cmake --build . && cmake --install . +RUN cmake --install . --component Runtime && \ + cmake --install . --component ffbidx_python && \ + cmake --install . --component ffbidx_libraries # Install Python package -WORKDIR /opt/build RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FFS=1.0 /opt/ffs/bin/pip3 install /opt/ffs_src + # Now copy this into an isolated runtime container FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu24.04 diff --git a/baseline/indexer/CMakeLists.txt b/baseline/indexer/CMakeLists.txt index 59c600e5..6cdc8e34 100644 --- a/baseline/indexer/CMakeLists.txt +++ b/baseline/indexer/CMakeLists.txt @@ -60,11 +60,11 @@ target_link_libraries(baseline_indexer # Create Python module nanobind_add_module(index index_module.cpp) target_include_directories(index PRIVATE ${CMAKE_SOURCE_DIR}/baseline/predictor) -target_link_libraries(index PRIVATE Eigen3::Eigen dx2 ${Python3_LIBRARIES}) +target_link_libraries(index PRIVATE Eigen3::Eigen dx2 Python::Module) # Install into the runtime Python sitearch install(TARGETS index - LIBRARY DESTINATION ${Python3_SITEARCH} + LIBRARY DESTINATION ${Python_SITEARCH}/ffs COMPONENT Runtime ) diff --git a/runtime-environment.yml b/runtime-environment.yml index 6082a291..ae897fc5 100644 --- a/runtime-environment.yml +++ b/runtime-environment.yml @@ -14,3 +14,4 @@ dependencies: - pydantic - rich - spdlog=1.15.3 + - numpy=2.3.4 From 8dfcaacaeecd5834c4c903bbbd29b0610bef04f4 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 4 Feb 2026 15:36:21 +0000 Subject: [PATCH 19/21] Build/install everything, and install ssx_index as a script Rather than manually copying/setting shebang, leverage python packaging entry_points. Install everything, because the number of dependent parts of the library is much larger now, and when we were only building spotfinder we didn't need to build the other parts. --- CMakeLists.txt | 11 ----------- Dockerfile | 9 +++------ pyproject.toml | 3 +++ runtime-environment.yml | 1 + src/ffs/ssx_index.py | 2 +- 5 files changed, 8 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 757f49e6..21c62226 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,14 +153,3 @@ add_subdirectory(h5read) add_subdirectory(baseline) add_subdirectory(spotfinder) add_subdirectory(integrator) - -# define a ssx_index 'executable' that actually just runs the python code (for style consistency) -configure_file(${CMAKE_SOURCE_DIR}/scripts/ssx_index ${CMAKE_BINARY_DIR}/bin/ssx_index COPYONLY) - -# run 'pip install .' to install the c++ extension for ssx_index -install(CODE " - execute_process( - COMMAND ${Python_EXECUTABLE} -m pip install . - WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\" - ) -") diff --git a/Dockerfile b/Dockerfile index 5e9eff12..c5c0a664 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,9 +25,8 @@ RUN micromamba create -y -f /opt/runtime-environment.yml -p /opt/ffs # Copy source COPY . /opt/ffs_src ENV CMAKE_GENERATOR Ninja + # Build the C++/CUDA backend -# Make Numpy headers available for ffbidx build (via -DPython3_EXECUTABLE=/opt/build_env/bin/python) -# Make ffbidx install into runtime env (-DPython3_SITELIB=$RT_SITE \ -DPython3_SITEARCH=$RT_SITE) WORKDIR /opt/build RUN cmake /opt/ffs_src \ -DCMAKE_BUILD_TYPE=Release \ @@ -35,11 +34,9 @@ RUN cmake /opt/ffs_src \ -DHDF5_ROOT=/opt/ffs \ -DPython3_ROOT_DIR=/opt/ffs -RUN cmake --build . --target spotfinder --target spotfinder32 --target index --target ffbidx --target fast_indexer +RUN cmake --build . -RUN cmake --install . --component Runtime && \ - cmake --install . --component ffbidx_python && \ - cmake --install . --component ffbidx_libraries +RUN cmake --install . # Install Python package RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FFS=1.0 /opt/ffs/bin/pip3 install /opt/ffs_src diff --git a/pyproject.toml b/pyproject.toml index 551870d6..26a5bccd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ "zocalo>=1.2.0", ] +[project.scripts] +ssx_index = "ffs.ssx_index:run" + [dependency-groups] dev = [ "pytest", diff --git a/runtime-environment.yml b/runtime-environment.yml index ae897fc5..0b3d54e9 100644 --- a/runtime-environment.yml +++ b/runtime-environment.yml @@ -7,6 +7,7 @@ dependencies: - gemmi=0.7.3 - hdf5=1.14.6 - hdf5-external-filter-plugins=0.1.0 + - h5py - python=3.13.7 - pip - zocalo diff --git a/src/ffs/ssx_index.py b/src/ffs/ssx_index.py index 72a954b7..ea8e195e 100644 --- a/src/ffs/ssx_index.py +++ b/src/ffs/ssx_index.py @@ -240,7 +240,7 @@ def write_table(self, filename): ## rlp, flags, xyzobs.mm.value -def run(args): +def run(args=None): st = time.time() parser = argparse.ArgumentParser( prog="index", From 18d1d12afc502714011e7e8dbe627be271f7ebcf Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Thu, 5 Feb 2026 08:42:30 +0000 Subject: [PATCH 20/21] Add JBE to authors --- Dockerfile | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c5c0a664..c7b090a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu24.04 LABEL org.opencontainers.image.title="fast-feedback-service" \ org.opencontainers.image.description="GPU-accelerated fast-feedback X-ray diffraction analysis service" \ - org.opencontainers.image.authors="Nicholas Devenish , Dimitrios Vlachos " \ + org.opencontainers.image.authors="Nicholas Devenish , Dimitrios Vlachos , James Beilsten-Edmands " \ org.opencontainers.image.source="https://github.com/DiamondLightSource/fast-feedback-service" \ org.opencontainers.image.licenses="BSD-3-Clause" diff --git a/pyproject.toml b/pyproject.toml index 26a5bccd..dce4a512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ readme = "README.md" authors = [ { name = "Nicholas Devenish", email = "ndevenish@gmail.com" }, { name = "Dimitri Vlachos", email = "dimitrios.vlachos@diamond.ac.uk" }, + { name = "James Beilsten-Edmands", email = "james.beilsten-edmands@diamond.ac.uk"}, ] requires-python = ">=3.12" From acc63aea3c6970a0968629d76eac1671309dd64c Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 6 Feb 2026 15:19:56 +0000 Subject: [PATCH 21/21] revert image tag to latest --- chart/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 275bdacf..281ff602 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/diamondlightsource/fast-feedback-service - tag: pr-71 + tag: latest # Beamline configuration beamline: "i24" @@ -27,4 +27,4 @@ podSecurityContext: # Container-level security context containerSecurityContext: # k8s-i24-beamline user - runAsUser: 36145 \ No newline at end of file + runAsUser: 36145