Add eva.multimodal

.gitattributes

@@ -3,6 +3,8 @@ tests/eva/assets/**/*.png filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.jpg filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.tif filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.tiff filter=lfs diff=lfs merge=lfs -text
+tests/eva/assets/**/*.jsonl filter=lfs diff=lfs merge=lfs -text
+tests/eva/assets/**/*.parquet filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.csv filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.pt filter=lfs diff=lfs merge=lfs -text
 tests/eva/assets/**/*.npy filter=lfs diff=lfs merge=lfs -text

README.md

@@ -34,7 +34,7 @@ Check out the [documentation](https://kaiko-ai.github.io/eva/) for more informat
 
 ### Highlights:
 - Easy and reliable benchmark of Oncology FMs
-- Supports patch-level classification, slide-level classification, semantic segmentation, and text classification downstream tasks
+- Supports patch-level classification, slide-level classification, semantic segmentation, and (visual) question answering tasks.
 - Automatic embedding inference and evaluation of a downstream task
 - Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
 - Produce statistics over multiple evaluation fits and multiple metrics
@@ -52,6 +52,9 @@ pip install 'kaiko-eva[vision]'
 # to install the expanded `language` version
 pip install 'kaiko-eva[language]'
 
+# to install the expanded `multimodal` version
+pip install 'kaiko-eva[multimodal]'
+
 # to install everything
 pip install 'kaiko-eva[all]'
 ```

configs/language/pathology/offline/multiple_choice/pubmedqa.yaml

@@ -0,0 +1,74 @@
+---
+trainer:
+  class_path: eva.Trainer
+  init_args:
+    n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
+    default_root_dir: ${oc.env:OUTPUT_ROOT, logs/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/language/pubmedqa}
+    checkpoint_type: null
+    callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
+      - class_path: eva.language.callbacks.writers.TextPredictionWriter
+        init_args:
+          output_dir: &PREDICTIONS_OUTPUT_DIR ${oc.env:PREDICTIONS_OUTPUT_DIR, ./predictions/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/pubmedqa}
+          dataloader_idx_map:
+            0: val
+          save_format: &PREDICTIONS_SAVE_FORMAT ${oc.env:PREDICTIONS_SAVE_FORMAT, jsonl}
+          model:
+            class_path: eva.language.models.wrappers.ModelFromRegistry
+            init_args:
+              model_name: ${oc.env:MODEL_NAME, anthropic/claude-3-7-sonnet-20250219}
+              model_extra_kwargs: ${oc.env:MODEL_EXTRA_KWARGS, null}
+          overwrite: false
+model:
+  class_path: eva.language.models.OfflineLanguageModule
+  init_args:
+    metrics:
+      common:
+        - class_path: eva.metrics.MulticlassClassificationMetrics
+          init_args:
+            num_classes: 3
+            input_type: "discrete"
+    postprocess:
+      predictions_transforms:
+        - class_path: eva.language.utils.str_to_int_tensor.CastStrToIntTensor
+          init_args:
+            mapping: {"no": 0, "yes": 1, "maybe": 2}
+            case_sensitive: false
+data:
+  class_path: eva.DataModule
+  init_args:
+    datasets:
+      val:
+        class_path: eva.language.datasets.TextPredictionDataset
+        init_args: &DATASET_ARGS
+          path: ${oc.env:PREDICTIONS_OUTPUT_DIR, ./predictions/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/pubmedqa}/manifest.${oc.env:PREDICTIONS_SAVE_FORMAT, jsonl}
+          split: val
+      test:
+        class_path: eva.language.datasets.TextPredictionDataset
+        init_args:
+          <<: *DATASET_ARGS
+      predict:
+        - class_path: eva.language.datasets.PubMedQA
+          init_args: &PREDICT_DATASET_ARGS
+            root: ${oc.env:DATA_ROOT, ./data/pubmedqa}
+            split: val
+            download: ${oc.env:DOWNLOAD_DATA, false}
+            # Set `download: true` to download the dataset from https://huggingface.co/datasets/bigbio/pubmed_qa
+            # The PubMedQA dataset is distributed under the following license: MIT License
+            # See (https://github.com/pubmedqa/pubmedqa/blob/master/LICENSE)
+        - class_path: eva.language.datasets.PubMedQA
+          init_args:
+            <<: *PREDICT_DATASET_ARGS
+            split: test
+    dataloaders:
+      val: &DATALOADER_ARGS
+        batch_size: &BATCH_SIZE ${oc.env:BATCH_SIZE, 256}
+        num_workers: &N_DATA_WORKERS ${oc.env:N_DATA_WORKERS, 1}
+        shuffle: false
+        collate_fn: eva.language.data.dataloaders.prediction_collate
+      test:
+        <<: *DATALOADER_ARGS
+      predict:
+        batch_size: &PREDICT_BATCH_SIZE ${oc.env:PREDICT_BATCH_SIZE, 16}
+        num_workers: *N_DATA_WORKERS
+        collate_fn: eva.language.data.dataloaders.text_collate

configs/language/pubmedqa.yaml → configs/language/pathology/online/multiple_choice/pubmedqa.yaml

@@ -5,17 +5,16 @@ trainer:
     n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
     default_root_dir: ${oc.env:OUTPUT_ROOT, logs/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/pubmedqa}
     checkpoint_type: null
+    callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
 model:
-  class_path: eva.language.models.TextModule
+  class_path: eva.language.models.LanguageModule
   init_args:
-    prompt: "Instruction: Carefully read the question and the provided context. Answer with one word: 'yes', 'no', or 'maybe'. Answer: "
     model:
-      class_path: eva.language.models.LiteLLMTextModel
+      class_path: eva.language.models.wrappers.ModelFromRegistry
       init_args:
-        model_name_or_path: ${oc.env:MODEL_NAME, anthropic/claude-3-7-sonnet-latest}
-        model_kwargs:
-          temperature: 0.0 # should be strictly positive for HF models
-          # max_new_tokens: 1 # used for HF
+        model_name: ${oc.env:MODEL_NAME, anthropic/claude-3-7-sonnet-20250219}
+        model_extra_kwargs: ${oc.env:MODEL_EXTRA_KWARGS, null}
     metrics:
       common:
         - class_path: eva.metrics.MulticlassClassificationMetrics
@@ -25,6 +24,9 @@ model:
     postprocess:
       predictions_transforms:
         - class_path: eva.language.utils.str_to_int_tensor.CastStrToIntTensor
+          init_args:
+            mapping: {"no": 0, "yes": 1, "maybe": 2}
+            case_sensitive: false
 data:
   class_path: eva.DataModule
   init_args:
@@ -33,15 +35,23 @@ data:
         class_path: eva.language.datasets.PubMedQA
         init_args: &DATASET_ARGS
           root: ${oc.env:DATA_ROOT, ./data/pubmedqa}
-          split: null
+          split: val
           download: ${oc.env:DOWNLOAD_DATA, false}
           # Set `download: true` to download the dataset from https://huggingface.co/datasets/bigbio/pubmed_qa
           # The PubMedQA dataset is distributed under the following license: MIT License
           # See (https://github.com/pubmedqa/pubmedqa/blob/master/LICENSE)
-          max_samples: 500
+      test:
+        class_path: eva.language.datasets.PubMedQA
+        init_args:
+          <<: *DATASET_ARGS
+          split: test
     dataloaders:
-      val:
+      val: &DATALOADER_ARGS
         batch_size: &BATCH_SIZE ${oc.env:BATCH_SIZE, 16}
         num_workers: &N_DATA_WORKERS ${oc.env:N_DATA_WORKERS, 1}
         shuffle: false
-        collate_fn: eva.core.data.dataloaders.text_collate_fn
+        collate_fn: eva.language.data.dataloaders.text_collate
+      test:
+        <<: *DATALOADER_ARGS
+
+

configs/multimodal/pathology/offline/multiple_choice/patch_camelyon.yaml

@@ -0,0 +1,84 @@
+trainer:
+  class_path: eva.Trainer
+  init_args:
+    accelerator: ${oc.env:ACCELERATOR, auto}
+    n_runs: &N_RUNS ${oc.env:N_RUNS, 2}
+    default_root_dir: ${oc.env:OUTPUT_ROOT, logs/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/multimodal/patch_camelyon}
+    precision: bf16
+    checkpoint_type: null
+    callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
+      - class_path: eva.multimodal.callbacks.writers.TextPredictionWriter
+        init_args:
+          output_dir: &PREDICTIONS_OUTPUT_DIR ${oc.env:PREDICTIONS_OUTPUT_DIR, ./predictions/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/multimodal/patch_camelyon}
+          dataloader_idx_map:
+            0: val
+            1: test
+          save_format: &PREDICTIONS_SAVE_FORMAT ${oc.env:PREDICTIONS_SAVE_FORMAT, jsonl}
+          model:
+            class_path: eva.multimodal.models.wrappers.ModelFromRegistry
+            init_args:
+              model_name: ${oc.env:MODEL_NAME, anthropic/claude-3-7-sonnet-20250219}
+              model_extra_kwargs: ${oc.env:MODEL_EXTRA_KWARGS, null}
+          overwrite: false
+model:
+  class_path: eva.language.models.OfflineLanguageModule
+  init_args:
+    metrics:
+      common:
+        - class_path: eva.metrics.MulticlassClassificationMetrics
+          init_args:
+            num_classes: 2
+            input_type: "discrete"
+    postprocess:
+      predictions_transforms:
+        - class_path: eva.language.utils.str_to_int_tensor.CastStrToIntTensor
+          init_args:
+            mapping: {"A": 0, "B": 1}
+            case_sensitive: false
+data:
+  class_path: eva.DataModule
+  init_args:
+    datasets:
+      val:
+        class_path: eva.language.datasets.TextPredictionDataset
+        init_args: &DATASET_ARGS
+          path: ${oc.env:PREDICTIONS_OUTPUT_DIR, ./predictions/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/multimodal/patch_camelyon}/manifest.${oc.env:PREDICTIONS_SAVE_FORMAT, jsonl}
+          split: val
+      test:
+        class_path: eva.language.datasets.TextPredictionDataset
+        init_args:
+          <<: *DATASET_ARGS
+          split: test
+      predict:
+        - class_path: eva.multimodal.data.datasets.PatchCamelyon
+          init_args: &PREDICTION_DATASET_ARGS
+            root: ${oc.env:DATA_ROOT, /mnt/localdisk/data/patch_camelyon}
+            split: val
+            download: ${oc.env:DOWNLOAD_DATA, false}
+            # Set `download: true` to download the dataset from https://zenodo.org/records/1494286
+            # The PatchCamelyon dataset is distributed under the following license: 
+            # "Creative Commons Zero v1.0 Universal"
+            # (see: https://choosealicense.com/licenses/cc0-1.0/)
+            transforms:
+              image:
+                class_path: eva.vision.data.transforms.Resize
+                init_args:
+                  size: ${oc.env:RESIZE_DIM, null}
+                  max_bytes: ${oc.env:IMAGE_MAX_BYTES, null}
+            max_samples: 500
+        - class_path: eva.multimodal.data.datasets.PatchCamelyon
+          init_args:
+            <<: *PREDICTION_DATASET_ARGS
+            split: test
+    dataloaders:
+      val: &DATALOADER_ARGS
+        batch_size: &BATCH_SIZE ${oc.env:BATCH_SIZE, 256}
+        num_workers: &N_DATA_WORKERS ${oc.env:N_DATA_WORKERS, 1}
+        collate_fn: eva.language.data.dataloaders.prediction_collate
+      test:
+        <<: *DATALOADER_ARGS
+      predict:
+        batch_size: &PREDICT_BATCH_SIZE ${oc.env:PREDICT_BATCH_SIZE, 16}
+        num_workers: *N_DATA_WORKERS
+        collate_fn: eva.multimodal.data.dataloaders.text_image_collate

configs/multimodal/pathology/online/multiple_choice/patch_camelyon.yaml

@@ -0,0 +1,63 @@
+trainer:
+  class_path: eva.Trainer
+  init_args:
+    accelerator: ${oc.env:ACCELERATOR, auto}
+    n_runs: &N_RUNS ${oc.env:N_RUNS, 2}
+    default_root_dir: ${oc.env:OUTPUT_ROOT, logs/${oc.env:MODEL_NAME, anthropic-claude-3-7-sonnet-latest}/pubmedqa}
+    precision: bf16
+    checkpoint_type: null
+    callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
+model:
+  class_path: eva.multimodal.models.modules.VisionLanguageModule
+  init_args:
+    model:
+      class_path: eva.multimodal.models.wrappers.ModelFromRegistry
+      init_args:
+        model_name: ${oc.env:MODEL_NAME, anthropic/claude-3-7-sonnet-20250219}
+        model_extra_kwargs: ${oc.env:MODEL_EXTRA_KWARGS, null}
+    metrics:
+      common:
+        - class_path: eva.metrics.MulticlassClassificationMetrics
+          init_args:
+            num_classes: 2
+            input_type: "discrete"
+    postprocess:
+      predictions_transforms:
+        - class_path: eva.language.utils.str_to_int_tensor.CastStrToIntTensor
+          init_args:
+            mapping: {"A": 0, "B": 1}
+            case_sensitive: false
+data:
+  class_path: eva.DataModule
+  init_args:
+    datasets:
+      val:
+        class_path: eva.multimodal.data.datasets.PatchCamelyon
+        init_args: &DATASET_ARGS
+          root: ${oc.env:DATA_ROOT, /mnt/localdisk/data/patch_camelyon}
+          split: val
+          download: ${oc.env:DOWNLOAD_DATA, false}
+          # Set `download: true` to download the dataset from https://zenodo.org/records/1494286
+          # The PatchCamelyon dataset is distributed under the following license: 
+          # "Creative Commons Zero v1.0 Universal"
+          # (see: https://choosealicense.com/licenses/cc0-1.0/)
+          transforms:
+            image:
+              class_path: eva.vision.data.transforms.Resize
+              init_args:
+                size: ${oc.env:RESIZE_DIM, null}
+                max_bytes: ${oc.env:IMAGE_MAX_BYTES, null}
+          max_samples: 500
+      test:
+        class_path: eva.multimodal.data.datasets.PatchCamelyon
+        init_args:
+          <<: *DATASET_ARGS
+          split: test
+    dataloaders:
+      val: &DATALOADER_ARGS
+        batch_size: &BATCH_SIZE ${oc.env:BATCH_SIZE, 16}
+        num_workers: &N_DATA_WORKERS ${oc.env:N_DATA_WORKERS, 1}
+        collate_fn: eva.multimodal.data.dataloaders.text_image_collate
+      test:
+        <<: *DATALOADER_ARGS

docs/index.md

@@ -31,7 +31,7 @@ hide:
 
 _Oncology FM Evaluation Framework by [kaiko.ai](https://www.kaiko.ai/)_
 
-*eva* currently supports performance evaluation for vision Foundation Models ("FMs") and supervised machine learning models on WSI (patch- and slide-level), radiology image segmentation, and text classification tasks.
+*eva* currently supports performance evaluation for Foundation Models ("FMs") accross multiple oncology domains and data modalities.
 
 With *eva* we provide the open-source community with an easy-to-use framework that follows industry best practices to deliver a robust, reproducible and fair evaluation benchmark across FMs of different sizes and architectures.
 
@@ -69,6 +69,11 @@ Supported datasets & tasks include:
 
 -	**[PubMedQA](datasets/pubmedqa.md)**: Medical question answering classification
 
+*Multimodal datasets*
+
+-	**[PatchCamelyon (image-text)](datasets/patch_camelyon.md)**: Vision-language benchmark variation for the popular vision Patch Camelyon task, where the goal is to classify breast cancer patches, using both the image and a text prompt.
+
+
 To evaluate FMs, *eva* provides support for different model-formats, including models trained with PyTorch, models available on HuggingFace and ONNX-models. For other formats custom wrappers can be implemented.
 
 

docs/reference/language/models/networks.md

@@ -2,4 +2,4 @@
 
 Reference information for the language model `Networks` API.
 
-::: eva.language.models.modules.TextModule
+::: eva.language.models.modules.LanguageModule

docs/reference/language/models/wrappers.md

@@ -2,6 +2,6 @@
 
 Reference information for the language model `Wrappers` API.
 
-::: eva.language.models.wrappers.HuggingFaceTextModel
-::: eva.language.models.wrappers.LiteLLMTextModel
-::: eva.language.models.wrappers.VLLMTextModel
+::: eva.language.models.wrappers.HuggingFaceModel
+::: eva.language.models.wrappers.LiteLLMModel
+::: eva.language.models.wrappers.VllmModel

docs/reference/multimodal/data/datasets.md

@@ -0,0 +1,6 @@
+# Datasets
+
+Reference information for the multimodal data `Datasets` API.
+
+::: eva.multimodal.data.datasets.TextImageDataset
+::: eva.multimodal.data.datasets.PatchCamelyon

docs/reference/multimodal/data/index.md

@@ -0,0 +1,3 @@
+# Data
+
+Multimodal data utilities for working with datasets that combine multiple modalities.

docs/reference/multimodal/index.md

@@ -0,0 +1,8 @@
+# Multimodal
+
+Reference information for the `multimodal` API.
+
+If you have not already installed the `multimodal`-package, install it with:
+```
+pip install 'kaiko-eva[multimodal]'
+```

docs/reference/multimodal/models/modules.md

@@ -0,0 +1,5 @@
+# Modules
+
+Reference information for the multimodal `Modules` API.
+
+::: eva.multimodal.models.modules.VisionLanguageModule