FEAT: Backfill class-level metadata for all remote seed datasets (#1780)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: romanlutz

pyrit/datasets/seed_datasets/remote/aegis_ai_content_safety_dataset.py

@@ -9,7 +9,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +60,11 @@ class _AegisContentSafetyDataset(_RemoteDatasetLoader):
         "Violence",
     ]
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 19093 annotated human-LLM interactions
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/aya_redteaming_dataset.py

@@ -8,7 +8,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +39,11 @@ class _AyaRedteamingDataset(_RemoteDatasetLoader):
         "Tagalog": "tgl",
     }
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "medium"  # 987 prompts across multiple languages
+    tags: frozenset[str] = frozenset({"safety", "multilingual"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/babelscape_alert_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +23,11 @@ class _BabelscapeAlertDataset(_RemoteDatasetLoader):
     Reference: [@tedeschi2024alert]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 30968 prompts (default config)
+    tags: frozenset[str] = frozenset({"default", "safety", "jailbreak"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/beaver_tails_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +31,11 @@ class _BeaverTailsDataset(_RemoteDatasetLoader):
 
     HF_DATASET_NAME: str = "PKU-Alignment/BeaverTails"
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 166382 annotated prompt-response entries (default config)
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/cbt_bench_dataset.py

@@ -7,7 +7,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +28,11 @@ class _CBTBenchDataset(_RemoteDatasetLoader):
         - [@zhang2024cbtbench]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "small"  # 20 core_fine_seed therapy seeds (default config)
+    tags: frozenset[str] = frozenset({"safety", "medical"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/ccp_sensitive_prompts_dataset.py

@@ -6,7 +6,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +21,11 @@ class _CCPSensitivePromptsDataset(_RemoteDatasetLoader):
     Reference: [@promptfoo2025ccp]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 1360 censorship-sensitive prompts (single-language Mandarin)
+    tags: frozenset[str] = frozenset({"safety", "multilingual"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/coconot_dataset.py

@@ -285,7 +285,7 @@ class _CoCoNotContrastDataset(_CoCoNotBaseDataset):
     CONFIG: str = "contrast"
     SPLITS: tuple[str, ...] = ("test",)
     size: str = "medium"
-    tags: set[str] = set()
+    tags: set[str] = {"safety", "refusal"}
     DEFAULT_DESCRIPTION: str = (
         "CoCoNot contrast set — 379 benign prompts that look superficially similar to "
         "refusal-target prompts but should be complied with. Used to measure "

pyrit/datasets/seed_datasets/remote/comic_jailbreak_dataset.py

@@ -12,7 +12,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import Seed, SeedDataset, SeedObjective, SeedPrompt
+from pyrit.models import Modality, Seed, SeedDataset, SeedObjective, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -92,8 +92,8 @@ class _ComicJailbreakDataset(_RemoteDatasetLoader):
         "sexual",
         "privacy",
     )
-    modalities: tuple[str, ...] = ("text", "image")
-    size: str = "large"  # 300 goals × 5 templates
+    modalities: tuple[Modality, ...] = (Modality.TEXT, Modality.IMAGE)
+    size: str = "large"  # 3501 image-text jailbreak prompts
     tags: frozenset[str] = frozenset({"safety", "multimodal"})
 
     def __init__(

pyrit/datasets/seed_datasets/remote/darkbench_dataset.py

@@ -4,7 +4,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 
 class _DarkBenchDataset(_RemoteDatasetLoader):
@@ -23,6 +23,11 @@ class _DarkBenchDataset(_RemoteDatasetLoader):
         - https://openreview.net/forum?id=odjMSBSWRt
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "large"  # 660 prompts across 6 dark-pattern categories
+    tags: frozenset[str] = frozenset({"default", "safety"})
+
     def __init__(
         self,
         *,

pyrit/datasets/seed_datasets/remote/equitymedqa_dataset.py

@@ -8,7 +8,7 @@
 from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
     _RemoteDatasetLoader,
 )
-from pyrit.models import SeedDataset, SeedPrompt
+from pyrit.models import Modality, SeedDataset, SeedPrompt
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +24,11 @@ class _EquityMedQADataset(_RemoteDatasetLoader):
     Reference: [@pfohl2024equitymedqa]
     """
 
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "huge"  # 5565 prompts across 11 medical-bias subsets
+    tags: frozenset[str] = frozenset({"safety", "bias", "medical"})
+
     DATA_SUBSETS: list[str] = [
         "cc_llm",
         "cc_manual",