diff --git a/src/odemis/acq/feature.py b/src/odemis/acq/feature.py index 90a7747357..73f08beb2b 100644 --- a/src/odemis/acq/feature.py +++ b/src/odemis/acq/feature.py @@ -21,7 +21,6 @@ """ import copy -import glob import logging import os import threading @@ -45,12 +44,13 @@ from odemis.acq.milling import DEFAULT_MILLING_TASKS_PATH from odemis.acq.move import Posture, MicroscopePostureManager from odemis.acq.stitching._tiledacq import SAFE_REL_RANGE_DEFAULT -from odemis.acq.stream import Stream, StaticFluoStream +from odemis.acq.stream import Stream, StaticFluoStream, StaticSEMStream, StaticFIBStream from odemis.dataio import find_fittest_converter from odemis.gui.cont.cryo_project import IMG_FILENAME, IMG_IN_FILE_IDS, add_image from odemis.model import MD_IN_FILE_INDEX from odemis.util import dataio, executeAsyncTask from odemis.util.comp import generate_zlevels +from odemis.util import datacollector from odemis.util.dataio import data_to_static_streams, open_acquisition, splitext from odemis.util.driver import estimate_stage_movement_time from odemis.util.filename import create_filename @@ -168,13 +168,18 @@ class CryoFeature(object): def __init__(self, name: str, stage_position: Dict[str, float], fm_focus_position: Dict[str, float], - milling_tasks: Optional[Dict[str, MillingTaskSettings]] = None, correlation_data=None): + milling_tasks: Optional[Dict[str, MillingTaskSettings]] = None, + correlation_data=None, + collect: bool = False): """ :param name: (string) the feature name :param stage_position: (dict) the stage position of the feature (stage-bare) :param fm_focus_position: (dict) the focus position of the feature :param correlation_data: (Dict[str,FIBFMCorrelationData]) Dictionary mapping the feature status to FIBFMCorrelationData, where feature status like Active, Rough Milled or polished is the key. + :param collect: (bool) Whether this feature is eligible for data collection. + Defaults to False. The GUI sets this based on the per-project sampling + decision made when a project is opened or created. """ self.name = model.StringVA(name) # FIXME: The 'position' parameter should eventually contain the SampleStage coordinates and not stage bare from the stage_position! @@ -205,6 +210,9 @@ def __init__(self, name: str, correlation_data = {} self.correlation_data = correlation_data + # Whether this feature is eligible for data collection + self.collect: bool = collect + # attributes for automated milling self.path: str = None # TODO:support path creation here, rather than on milling data save self.reference_image: model.DataArray = None @@ -292,9 +300,11 @@ def feature_decoder(feature_raw: Dict) -> CryoFeature: fm_focus_position = feature_raw['fm_focus_position'] posture_positions = feature_raw.get('posture_positions', {}) milling_task_json = feature_raw.get('milling_tasks', {}) + collect = feature_raw.get('collect', False) feature = CryoFeature(name=feature_raw['name'], stage_position=stage_position, - fm_focus_position=fm_focus_position + fm_focus_position=fm_focus_position, + collect=collect ) feature.correlation_data = FIBFMCorrelationData.from_dict(correlation_data) if correlation_data else None feature.status.value = feature_raw['status'] @@ -396,6 +406,159 @@ def _create_fibsem_filename(filename: str, acq_type: str) -> str: return create_filename(path, ptn, ext, count="001") +def _stream_overlaps_position(stream: Stream, x: float, y: float) -> bool: + """ + True if the stage position (x, y) falls within the stream's field of view. + + :param stream: A static stream with a getBoundingBox() method. + :param x: Stage x position in metres. + :param y: Stage y position in metres. + :return: True when the position is inside the bounding box, False otherwise. + """ + try: + bbox = stream.getBoundingBox() # (left, top, right, bottom) in metres + except Exception: + return False + left, top, right, bottom = bbox + return left <= x <= right and top <= y <= bottom + + +def collect_feature_data( + feature: CryoFeature, + overview_streams: Optional[List[Stream]] = None, + project_dir: Optional[str] = None) -> None: + """ + Collect anonymized data for a feature and submit it to the data collector. It skips immediately if feature.collect + is False or if data collection consent has not been granted. Never raises — all errors are logged as warnings. + + The payload contains: + - Acquired z-stack per FM channel (or first FM image if no z-stack) before moving to SEM/FIB + - FM and SEM overview images that spatially overlap the feature's position. + - Feature status, stage position, and FM focus position. + + Privacy rules enforced: + - Feature name is never included. + - Image payload keys are generic (channel_0, overview_fm_0, etc.). + - Original filenames are not included in the payload. + + After collection feature.collect is set to False to prevent re-collection. + + :param feature: The feature for which the data is collected. Must have feature.collect == True. + :param overview_streams: Optional list of overview static streams. Used to find FM / SEM overviews + that overlap the feature position. + :param project_dir: Optional project directory path. When provided and + feature.streams is empty, streams are loaded from disk first. + """ + if not feature.collect: + return + + try: + _dc = datacollector.DataCollector() + if not _dc.get_consent(): + return + except Exception: + logging.warning("Failed to access DataCollector; skipping collection", exc_info=True) + return + + try: + # Load feature streams from disk when not yet in memory. + if not feature.streams.value and project_dir: + try: + load_feature_streams_from_disk(feature, project_dir) + except Exception: + logging.warning("Failed to load streams from disk; skipping collection", exc_info=True) + return + + feature_streams = list(feature.streams.value) + + # Prioritize collecting z stack as it is more insightful than a single image + fm_zstacks: List = [] + fm_images: List = [] + for s in feature_streams: + if isinstance(s, StaticFluoStream): + if hasattr(s, "zIndex"): + fm_zstacks.append(s) + else: + fm_images.append(s) + + def _get_raw(stream: Stream) -> Optional[model.DataArray]: + """Return image data for a static stream from raw or image VA.""" + try: + if stream.raw: + return stream.raw[0] + except Exception: + pass + try: + image_va = getattr(stream, "image", None) + if image_va is not None: + return image_va.value + except Exception: + pass + return None + + # Per channel: prefer z-stack, then single FM image. + # Channels are delineated by MD_OUT_WL; use index as fallback. + selected_fm: List = [] + seen_channels: set = set() + for s in fm_zstacks + fm_images: + da = _get_raw(s) + channel_key = da.metadata.get(model.MD_OUT_WL) if da is not None else None + if channel_key not in seen_channels: + seen_channels.add(channel_key) + selected_fm.append(s) + + # Collect spatially overlapping overview streams. + stage_pos = feature.stage_position.value + feat_x = stage_pos.get("x", 0.0) + feat_y = stage_pos.get("y", 0.0) + + overview_fm: List = [] + overview_sem: List = [] + for s in (overview_streams or []): + if not _stream_overlaps_position(s, feat_x, feat_y): + continue + if isinstance(s, StaticFluoStream): + overview_fm.append(s) + elif isinstance(s, (StaticSEMStream, StaticFIBStream)): + overview_sem.append(s) + + # Build privacy-preserving payload — generic keys, no names or filenames. + payload: dict = { + "status": feature.status.value, + "stage_position": dict(stage_pos), + "fm_focus_position": dict(feature.fm_focus_position.value), + } + + + for idx, s in enumerate(selected_fm): + da = _get_raw(s) + if da is not None: + payload[f"channel_{idx}"] = da + + for idx, s in enumerate(overview_fm): + da = _get_raw(s) + if da is not None: + payload[f"overview_fm_{idx}"] = da + + for idx, s in enumerate(overview_sem): + da = _get_raw(s) + if da is not None: + payload[f"overview_sem_{idx}"] = da + + image_keys = [k for k in payload if k.startswith(("channel_", "overview_fm_", "overview_sem_"))] + if not image_keys: + logging.debug( "No images found for the given feature; skipping collection") + return + + _dc.record("feature_collected", "1.0", payload) + + feature.collect = False + logging.debug("Data collected successfully") + + except Exception: + logging.warning("Failed to collect data; skipping collection", exc_info=True) + + # To handle the timeout error when the stage is not able to move to the desired position # It logs the message and raises the MoveError exception class MoveError(Exception): diff --git a/src/odemis/acq/test/feature_test.py b/src/odemis/acq/test/feature_test.py index 164ac65a84..91c9b39968 100644 --- a/src/odemis/acq/test/feature_test.py +++ b/src/odemis/acq/test/feature_test.py @@ -22,18 +22,21 @@ import os import random import unittest - +from unittest.mock import patch import numpy from odemis import model from odemis.acq.feature import ( CryoFeature, + _stream_overlaps_position, + collect_feature_data, load_milling_tasks, FEATURE_READY_TO_MILL, REFERENCE_IMAGE_FILENAME, ) from odemis.acq.move import Posture from odemis.acq.milling import DEFAULT_MILLING_TASKS_PATH +from odemis.acq.stream import StaticFluoStream logging.getLogger().setLevel(logging.DEBUG) @@ -90,5 +93,204 @@ def test_feature_milling_tasks(self): filename = os.path.join(feature.path, f"{feature.name.value}-{REFERENCE_IMAGE_FILENAME}") self.assertTrue(os.path.exists(filename)) + +class TestCollectFlag(unittest.TestCase): + """Tests for the CryoFeature.collect flag and its persistence.""" + + def test_collect_flag_is_bool(self): + """CryoFeature.collect must be False by default.""" + f = CryoFeature("F", {"x": 0, "y": 0, "z": 0}, {"z": 0}) + self.assertFalse(f.collect) + + +class TestCollectFeatureData(unittest.TestCase): + """Tests for collect_feature_data().""" + + def _make_feature(self, collect: bool = True, pos=None) -> CryoFeature: + if pos is None: + pos = {"x": 0.0, "y": 0.0, "z": 0.0} + return CryoFeature("TestFeature", pos, {"z": 0.0}, collect=collect) + + def _make_fluo_stream(self): + """Return a minimal StaticFluoStream with a 2-D DataArray.""" + arr = numpy.zeros((64, 64), dtype=numpy.uint16) + da = model.DataArray(arr, metadata={ + model.MD_POS: (0.0, 0.0), + model.MD_PIXEL_SIZE: (1e-6, 1e-6), + }) + return StaticFluoStream("ch0", da) + + def _make_feature_with_stream(self, collect: bool = True) -> CryoFeature: + """Return a feature with one FM stream attached.""" + f = self._make_feature(collect=collect) + f.streams.value.append(self._make_fluo_stream()) + return f + + def test_skips_when_collect_false(self): + """collect_feature_data must not call record() when feature.collect is False.""" + f = self._make_feature(collect=False) + with patch("odemis.util.datacollector.DataCollector") as MockDC: + collect_feature_data(f) + MockDC.return_value.get_consent.assert_not_called() + + def test_skips_when_no_consent(self): + """collect_feature_data must not call record() when consent is not granted.""" + f = self._make_feature_with_stream(collect=True) + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = False + collect_feature_data(f) + MockDC.return_value.record.assert_not_called() + + def test_no_record_without_images(self): + """record() must NOT be called when the feature has no image streams.""" + f = self._make_feature(collect=True) # no streams attached + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + collect_feature_data(f) + MockDC.return_value.record.assert_not_called() + + def test_sets_collect_false_after_collection(self): + """feature.collect must be False after successful collection with images.""" + f = self._make_feature_with_stream(collect=True) + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + collect_feature_data(f) + self.assertFalse(f.collect) + + def test_payload_contains_status_positions_and_image(self): + """Payload must contain status, stage_position, fm_focus_position, and at least one image.""" + f = self._make_feature_with_stream(collect=True) + f.status.value = "Active" + captured = {} + + def fake_record(event_name, schema_version, payload, **kwargs): + captured.update(payload) + + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + MockDC.return_value.record.side_effect = fake_record + collect_feature_data(f) + + self.assertIn("status", captured) + self.assertIn("stage_position", captured) + self.assertIn("fm_focus_position", captured) + image_keys = [k for k in captured if k.startswith(("channel_", "overview_fm_", "overview_sem_"))] + self.assertTrue(len(image_keys) >= 1, "Payload must contain at least one image") + + def test_payload_has_no_feature_name(self): + """Payload must not contain the feature name string as a key or value.""" + f = self._make_feature_with_stream(collect=True) + f.name.value = "my_secret_feature_name" + captured = {} + + def fake_record(event_name, schema_version, payload, **kwargs): + captured.update(payload) + + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + MockDC.return_value.record.side_effect = fake_record + collect_feature_data(f) + + self.assertNotIn("my_secret_feature_name", captured) + self.assertNotIn("my_secret_feature_name", str(captured.keys())) + + def test_payload_channel_keys_are_generic(self): + """Image payload keys must be generic (channel_N), not derived from feature or stream name. + + A StaticFluoStream named 'test_stream' is attached to the feature. + After collection the payload key for the image must be 'channel_0', + not 'test_stream' or the feature name — ensuring data privacy. + """ + f = self._make_feature_with_stream(collect=True) + captured = {} + + def fake_record(event_name, schema_version, payload, **kwargs): + captured.update(payload) + + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + MockDC.return_value.record.side_effect = fake_record + collect_feature_data(f) + + image_keys = [k for k in captured if k.startswith("channel_")] + self.assertTrue(len(image_keys) >= 1, "Expected at least one channel_N key in payload") + for k in image_keys: + self.assertRegex(k, r"^channel_\d+$") + + def test_collects_on_status_change(self): + """Subscribing to feature.status and calling collect_feature_data on change must call record(). + + This simulates the controller's _on_status_for_collection subscriber: when + the feature status VA changes, collect_feature_data is invoked and record() + is called exactly once (consent granted, images present, collect=True). + """ + f = self._make_feature_with_stream(collect=True) + record_calls = [] + + def fake_record(event_name, schema_version, payload, **kwargs): + record_calls.append((event_name, schema_version)) + + def _on_status_changed(_status): + if f.collect: + collect_feature_data(f) + + f.status.subscribe(_on_status_changed, init=False) + try: + with patch("odemis.util.datacollector.DataCollector") as MockDC: + MockDC.return_value.get_consent.return_value = True + MockDC.return_value.record.side_effect = fake_record + f.status.value = FEATURE_READY_TO_MILL + finally: + f.status.unsubscribe(_on_status_changed) + + self.assertEqual(len(record_calls), 1) + self.assertEqual(record_calls[0][0], "feature_collected") + + +class TestStreamHelpers(unittest.TestCase): + """Tests for stream_overlaps_position.""" + + def _make_static_fluo_stream(self, shape=(64, 64), pos=(0.0, 0.0), pixel_size=(1e-6, 1e-6)): + """Return a minimal StaticFluoStream.""" + from odemis.acq.stream import StaticFluoStream + arr = numpy.zeros(shape, dtype=numpy.uint16) + da = model.DataArray(arr, metadata={ + model.MD_POS: pos, + model.MD_PIXEL_SIZE: pixel_size, + }) + return StaticFluoStream("test_stream", da) + + def _make_zstack_stream(self, pos=(0.0, 0.0), pixel_size=(1e-6, 1e-6)): + """Return a minimal StaticFluoStream that looks like a z-stack (has zIndex).""" + s = self._make_static_fluo_stream(pos=pos, pixel_size=pixel_size) + s.zIndex = model.IntContinuous(0, (0, 3)) + return s + + def test_overlaps_centre(self): + """Position at the stream centre must overlap.""" + # 64 x 64 pixels at 1 µm/pixel centred at (0, 0) → bbox ±32 µm. + s = self._make_static_fluo_stream() + self.assertTrue(_stream_overlaps_position(s, 0.0, 0.0)) + + def test_overlaps_edge(self): + """Position exactly on the bounding-box edge must still overlap.""" + s = self._make_static_fluo_stream(pos=(0.0, 0.0), pixel_size=(2e-6, 2e-6)) + # half-width = 64/2 * 2e-6 = 64e-6 m → right edge at +64e-6 + self.assertTrue(_stream_overlaps_position(s, 64e-6, 0.0)) + + def test_no_overlap_outside(self): + """Position clearly outside the bounding box must not overlap.""" + s = self._make_static_fluo_stream() + # bbox is ±32 µm; 100 µm is well outside. + self.assertFalse(_stream_overlaps_position(s, 100e-6, 0.0)) + + def test_no_overlap_bad_stream(self): + """_stream_overlaps_position returns False when getBoundingBox() raises.""" + from unittest.mock import MagicMock + bad_stream = MagicMock() + bad_stream.getBoundingBox.side_effect = AttributeError("no bbox") + self.assertFalse(_stream_overlaps_position(bad_stream, 0.0, 0.0)) + + if __name__ == "__main__": unittest.main() diff --git a/src/odemis/acq/test/test-features.json b/src/odemis/acq/test/test-features.json index 87828d2736..e1433d3864 100644 --- a/src/odemis/acq/test/test-features.json +++ b/src/odemis/acq/test/test-features.json @@ -3,6 +3,7 @@ { "name": "Feature-1", "status": "Active", + "collect": false, "stage_position": { "x": 0, "y": 0, @@ -42,6 +43,7 @@ { "name": "Feature-2", "status": "Active", + "collect": false, "stage_position": { "x": 0.001, "y": 0.001, diff --git a/src/odemis/gui/cont/features.py b/src/odemis/gui/cont/features.py index 11c2bad79f..71fad8c552 100644 --- a/src/odemis/gui/cont/features.py +++ b/src/odemis/gui/cont/features.py @@ -22,7 +22,10 @@ import itertools import logging +import math import os +import threading +from typing import Optional import wx @@ -33,6 +36,7 @@ FEATURE_READY_TO_MILL, FEATURE_ROUGH_MILLED, CryoFeature, + collect_feature_data, get_feature_position_at_posture, FIBFMCorrelationData, Target, @@ -49,6 +53,10 @@ SUPPORTED_POSTURES = [Posture.SEM_IMAGING, Posture.FM_IMAGING, Posture.MILLING, Posture.FIB_IMAGING, Posture.FIB_VIEW_FM] +# Maximum distance (in metres) within which another feature is considered "nearby" +# for the feature-deletion data-collection trigger. +_NEARBY_FEATURE_DISTANCE_M = 100e-6 + class CryoFeatureController(object): """ controller to handle the cryo feature panel elements It requires features list VA & currentFeature VA on the tab data to function properly @@ -78,6 +86,9 @@ def __init__(self, tab_data, panel, tab, mode: guimod.AcquiMode): self._feature_status_va_connector = None self._feature_z_va_connector = None + # Feature whose status VA we are subscribed to for data-collection triggering. + self._status_collect_feature: Optional[CryoFeature] = None + self._tab_data_model.main.features.subscribe(self._on_features_changes, init=True) self._tab_data_model.main.currentFeature.subscribe(self._on_current_feature_changes, init=True) @@ -104,6 +115,9 @@ def __init__(self, tab_data, panel, tab, mode: guimod.AcquiMode): self._panel.btn_feature_save_position.Show(LICENCE_MILLING_ENABLED) self.pm.current_posture.subscribe(self._on_posture_change) + # Track previous posture so we can detect FM → SEM/FIB transitions. + self._prev_posture = self.pm.getCurrentPostureLabel() if self.pm else None + def _on_btn_create_move_feature(self, _): # As this button is identical to clicking the feature tool, # directly change the tool to feature tool @@ -121,6 +135,7 @@ def _on_btn_delete_feature(self, _): style=wx.YES_NO | wx.ICON_QUESTION | wx.CENTER) ans = box.ShowModal() if ans == wx.ID_YES: + self._maybe_collect_on_delete(current_feature) self._tab_data_model.main.features.value.remove(current_feature) self._tab_data_model.main.currentFeature.value = None if self.acqui_mode is guimod.AcquiMode.FIBSEM: @@ -244,6 +259,11 @@ def _on_posture_change(self, posture: int): return self._enable_feature_ctrls(True) + prev = self._prev_posture + self._prev_posture = posture + if prev == FM_IMAGING and posture in (SEM_IMAGING, FIB_IMAGING): + self._collect_eligible_features_in_thread() + def _enable_feature_ctrls(self, enable: bool): """ Enables/disables the feature controls @@ -326,6 +346,11 @@ def _on_current_feature_changes(self, feature): if self._feature_z_va_connector: self._feature_z_va_connector.disconnect() + # Unsubscribe status-change data-collection trigger from the previous feature. + if self._status_collect_feature is not None: + self._status_collect_feature.status.unsubscribe(self._on_status_for_collection) + self._status_collect_feature = None + self._update_feature_cmb_list() if feature is None: @@ -392,6 +417,10 @@ def _on_current_feature_changes(self, feature): ctrl_2_va=self._on_ctrl_feature_z_change, va_2_ctrl=self._on_feature_focus_pos) + # Subscribe to status changes to trigger data collection (init=False: skip current value). + feature.status.subscribe(self._on_status_for_collection, init=False) + self._status_collect_feature = feature + def _on_feature_focus_pos(self, fm_focus_position: dict): # Set the feature Z ctrl with the focus position self._panel.ctrl_feature_z.SetValue(fm_focus_position["z"]) @@ -449,3 +478,92 @@ def _on_ctrl_feature_z_change(self): zpos = self._panel.ctrl_feature_z.GetValue() return {"z": zpos} + + def _on_status_for_collection(self, _status: str) -> None: + """ + Trigger data collection when the current feature's status changes. + + :param _status: The new feature status value (unused; feature is read + from the stored reference to avoid a race with currentFeature). + """ + feature = self._status_collect_feature + if feature is not None and feature.collect: + self._collect_feature_in_thread(feature) + + def _collect_feature_in_thread(self, feature: CryoFeature) -> None: + """ + Launch collect_feature_data for a single feature in a background thread. + + :param feature: The feature to collect data for. + """ + overview_streams = self._tab_data_model.overviewStreams.value + project_dir = self._tab_data_model.conf.pj_last_path + + def _run(): + collect_feature_data(feature, overview_streams=overview_streams, project_dir=project_dir) + + t = threading.Thread(target=_run, name="FeatureDataCollection", daemon=True) + t.start() + + def _collect_eligible_features_in_thread(self) -> None: + """Launch collect_feature_data for all features with collect=True in a background thread.""" + features = self._tab_data_model.main.features.value + overview_streams = self._tab_data_model.overviewStreams.value + project_dir = self._tab_data_model.conf.pj_last_path + + def _run(): + for feature in features: + if feature.collect: + collect_feature_data( + feature, + overview_streams=overview_streams, + project_dir=project_dir, + ) + + t = threading.Thread(target=_run, name="FeatureDataCollectionBulk", daemon=True) + t.start() + + def _has_zstack_stream(self, feature: CryoFeature) -> bool: + """Return True if the feature has at least one z-stack stream. + + :param feature: The feature to check. + :returns: True when a z-stack stream is present, False otherwise. + """ + return any(hasattr(s, "zIndex") for s in feature.streams.value) + + def _has_nearby_feature(self, feature: CryoFeature, distance_m: float = _NEARBY_FEATURE_DISTANCE_M) -> bool: + """Return True if any other feature is within distance_m of the given feature. + + :param feature: The feature to check proximity for. + :param distance_m: Maximum distance in metres to be considered nearby. + :returns: True when another feature is within the threshold distance. + """ + pos = feature.stage_position.value + fx, fy = pos.get("x", 0.0), pos.get("y", 0.0) + for other in self._tab_data_model.main.features.value: + if other is feature: + continue + other_pos = other.stage_position.value + ox, oy = other_pos.get("x", 0.0), other_pos.get("y", 0.0) + dist = math.sqrt((fx - ox) ** 2 + (fy - oy) ** 2) + if dist <= distance_m: + return True + return False + + def _maybe_collect_on_delete(self, feature: CryoFeature) -> None: + """Trigger data collection before a feature is deleted if eligible. + + Collection is triggered when all three conditions are met: + - feature.collect is True + - The feature has at least one z-stack stream + - No other feature is within 100 µm + + :param feature: The feature about to be deleted. + """ + if not feature.collect: + return + if not self._has_zstack_stream(feature): + return + if self._has_nearby_feature(feature): + return + self._collect_feature_in_thread(feature) diff --git a/src/odemis/gui/cont/tabs/cryo_chamber_tab.py b/src/odemis/gui/cont/tabs/cryo_chamber_tab.py index 3c736191f2..245c4aac36 100644 --- a/src/odemis/gui/cont/tabs/cryo_chamber_tab.py +++ b/src/odemis/gui/cont/tabs/cryo_chamber_tab.py @@ -29,6 +29,7 @@ import math import os.path from concurrent.futures import CancelledError +import random import wx @@ -56,6 +57,7 @@ from odemis.gui.win.acquisition import LoadProjectFileDialog, ShowChamberFileDialog from odemis.model import InstantaneousFuture from odemis.util import almost_equal +from odemis.util.datacollector import DataCollector from odemis.util.dataio import data_to_static_streams, open_acquisition from odemis.util.filename import create_projectname, guess_pattern from odemis.util.units import readable_str @@ -373,6 +375,18 @@ def _change_project_conf(self, new_dir): self.conf.pj_ptn, self.conf.pj_count = guess_pattern(new_dir) self.txt_projectpath.Value = os.path.basename(self.conf.pj_last_path) self.tab_data_model.main.project_path.value = new_dir + # Decide once per project whether features created during this session are + # eligible for data collection. Stored as a dynamic attribute — not part + # of the formal model — and read by add_new_feature via getattr. + probability = DataCollector().probability + self.tab_data_model.main.features_collectable = ( + random.random() < probability + ) + logging.debug( + "Project '%s': features_collectable=%s", + os.path.basename(new_dir), + self.tab_data_model.main.features_collectable, + ) logging.debug("Generated project folder name pattern '%s'", self.conf.pj_ptn) def _create_new_dir(self): @@ -492,7 +506,13 @@ def _load_project_data(self, evt: wx.Event) -> bool: logging.info("Fibsem tab does not exists.") # Load features + # Immediately mark all as not collectable. + # Loaded features were either already collected in a previous session or + # were never selected; the new per-project sampling decision (set in + # _change_project_conf above) applies only to features created after this. decoded_features = [feature_decoder(f) for f in proj_data["features"]] + for f in decoded_features: + f.collect = False self.tab_data_model.main.features.value = [df for df in decoded_features if df is not None] # Load overview streams in the Localization and Fibsem tabs diff --git a/src/odemis/gui/model/tab_gui_data.py b/src/odemis/gui/model/tab_gui_data.py index f5402d045f..1324a0e4bb 100644 --- a/src/odemis/gui/model/tab_gui_data.py +++ b/src/odemis/gui/model/tab_gui_data.py @@ -341,7 +341,8 @@ def add_new_feature(self, stage_position: Dict[str, float], else: md = self.main.focus.getMetadata() fm_focus_position = md[model.MD_FAV_POS_ACTIVE] - feature = CryoFeature(f_name, stage_position, fm_focus_position) + features_collectable = getattr(self.main, "features_collectable", False) + feature = CryoFeature(f_name, stage_position, fm_focus_position, collect=features_collectable) for p in pm.postures: # calculate the position at all postures get_feature_position_at_posture(pm, feature, p) diff --git a/src/odemis/util/datacollector.py b/src/odemis/util/datacollector.py index df00016df0..cf8ded2379 100644 --- a/src/odemis/util/datacollector.py +++ b/src/odemis/util/datacollector.py @@ -32,7 +32,6 @@ import logging import os import queue -import random import re import shutil import socket @@ -657,6 +656,9 @@ def __init__(self) -> None: self._worker: Optional[_BackgroundWorker] = None self._init_ok: bool = False self._init_lock = threading.Lock() + # Collect percentage of the acquired data based on the selected probability + # in order to keep the growth of the collected data in control. + self.probability = _DEFAULT_COLLECTION_PROBABILITY def _lazy_init(self) -> None: """Initialise configuration and worker on first use.""" @@ -788,16 +790,9 @@ def record( days_left = None if days_left is not None and days_left <= 1: - probability = _FULL_COLLECTION_PROBABILITY + self.probability = _FULL_COLLECTION_PROBABILITY else: - probability = _DEFAULT_COLLECTION_PROBABILITY - - if random.random() >= probability: - logging.debug( - "DataCollector: event '%s' not sampled (%.0f%% collection probability).", - event_name, probability * 100, - ) - return + self.probability = _DEFAULT_COLLECTION_PROBABILITY item = _WorkItem( event_name=event_name,