Add microsaccades to Brain-Score core

brainscore_vision/model_helpers/brain_transformation/__init__.py

@@ -24,6 +24,10 @@ def __init__(self, identifier,
                  visual_degrees=8):
         self.layers = layers
         self.activations_model = activations_model
+        # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature.
+        #  The ideal solution would be to not expose the _extractor of the activations_model here, but to change
+        #  the signature of the ActivationsExtractorHelper. See https://github.com/brain-score/vision/issues/554
+        self.activations_model._extractor.set_visual_degrees(visual_degrees)  # for microsaccades
         self._visual_degrees = visual_degrees
         # region-layer mapping
         if region_layer_map is None:

brainscore_vision/model_helpers/brain_transformation/neural.py

@@ -23,21 +23,30 @@ def __init__(self, identifier, activations_model, region_layer_map, visual_degre
     def identifier(self):
         return self._identifier
 
-    def look_at(self, stimuli, number_of_trials=1):
+    def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False):
+        """
+        :param number_of_trials: An integer that determines how many repetitions of the same model performs.
+        :param require_variance: Whether to require models to return different activations for the same stimuli or not.
+                                  For detailed information, see
+                                  :meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.__call__`,
+        """
         layer_regions = {}
         for region in self.recorded_regions:
             layers = self.region_layer_map[region]
             layers = make_list(layers)
             for layer in layers:
                 assert layer not in layer_regions, f"layer {layer} has already been assigned for {layer_regions[layer]}"
                 layer_regions[layer] = region
-        activations = self.run_activations(
-            stimuli, layers=list(layer_regions.keys()), number_of_trials=number_of_trials)
+        activations = self.run_activations(stimuli,
+                                           layers=list(layer_regions.keys()),
+                                           number_of_trials=number_of_trials,
+                                           require_variance=require_variance)
         activations['region'] = 'neuroid', [layer_regions[layer] for layer in activations['layer'].values]
         return activations
 
-    def run_activations(self, stimuli, layers, number_of_trials=1):
-        activations = self.activations_model(stimuli, layers=layers)
+    def run_activations(self, stimuli, layers, number_of_trials=1, require_variance=None):
+        activations = self.activations_model(stimuli, layers=layers, number_of_trials=number_of_trials,
+                                             require_variance=require_variance)
         return activations
 
     def start_task(self, task):

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "importlib-metadata<5", # workaround to https://github.com/brain-score/brainio/issues/28
     "scikit-learn", # for metric_helpers/transformations.py cross-validation
     "scipy", # for benchmark_helpers/properties_common.py
+    "opencv-python",  # for microsaccades
     "h5py",
     "tqdm",
     "gitpython",

tests/test_model_helpers/activations/test___init__.py

@@ -189,6 +189,68 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits):
     return activations
 
 
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("number_of_trials", [1, 5, 25])
+def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=number_of_trials,
+                                        require_variance=True)
+
+    assert activations is not None
+    assert len(activations['microsaccade_shift_x_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_x_degrees']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_degrees']) == number_of_trials * len(stimulus_paths)
+
+
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("require_variance", [False, True])
+@pytest.mark.parametrize("number_of_trials", [1, 2, 10])
+def test_model_requirements(model_ctr, layers, image_name, require_variance, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers,
+                                        number_of_trials=number_of_trials, require_variance=require_variance)
+
+    assert activations is not None
+    if require_variance:
+        assert len(activations['presentation']) == number_of_trials
+    else:
+        assert len(activations['presentation']) == 1
+
+
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_temporary_file_handling(model_ctr, layers, image_name):
+    import tempfile
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=2,
+                                        require_variance=True)
+    temp_files = [f for f in os.listdir(tempfile.gettempdir()) if f.startswith('temp') and f.endswith('.png')]
+
+    assert activations is not None
+    assert len(temp_files) == 0
+
+
 def _build_stimulus_set(image_names):
     stimulus_set = StimulusSet([{'stimulus_id': image_name, 'some_meta': image_name[::-1]}
                                 for image_name in image_names])
@@ -223,6 +285,13 @@ def test_exact_activations(pca_components):
                                        image_name='rgb.jpg', pca_components=pca_components, logits=False)
     path_to_expected = Path(__file__).parent / f'alexnet-rgb-{pca_components}.nc'
     expected = xr.load_dataarray(path_to_expected)
+
+    # Originally, the `stimulus_path` Index was used to index into xarrays in Brain-Score, but this was changed
+    #  as a part of PR #492 to a MultiIndex to allow metadata to be attached to multiple repetitions of the same
+    #  `stimulus_path`. Old .nc files need to be updated to use the `presentation` index instead of `stimulus_path`,
+    #  and instead of changing the extant activations, this test was simply modified to simulate that.
+    expected = expected.rename({'stimulus_path': 'presentation'})
+
     assert (activations == expected).all()
 
 

tests/test_model_helpers/brain_transformation/test___init__.py

@@ -1,9 +1,18 @@
+from unittest.mock import Mock
+
 from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
 from brainscore_vision.model_helpers.utils import fullname
 
 
 class TestVisualDegrees:
     def test_standard_commitment(self):
-        brain_model = ModelCommitment(identifier=fullname(self), activations_model=None,
+        # create mock ActivationsExtractorHelper with a mock set_visual_degrees to avoid failing set_visual_degrees()
+        mock_extractor = Mock()
+        mock_extractor.set_visual_degrees = Mock()
+        mock_activations_model = Mock()
+        mock_activations_model._extractor = mock_extractor
+
+        # Initialize ModelCommitment with the mock activations_model
+        brain_model = ModelCommitment(identifier=fullname(self), activations_model=mock_activations_model,
                                       layers=['dummy'])
         assert brain_model.visual_degrees() == 8