FEAT: Adding Garak Figsafe Scenario

doc/code/datasets/1_loading_datasets.ipynb

@@ -64,8 +64,8 @@
     "(`garak_pypi_packages`, `garak_npm_packages`, `garak_crates_packages`,\n",
     "`garak_rubygems_packages`, `garak_dart_packages`, `garak_perl_packages`,\n",
     "`garak_raku_packages`), system-prompt libraries (`garak_drh_system_prompts`,\n",
-    "`garak_tm_system_prompts`), and an audio jailbreak set\n",
-    "(`garak_audio_achilles_heel`)."
+    "`garak_tm_system_prompts`), an audio jailbreak set\n",
+    "(`garak_audio_achilles_heel`), and visual jailbreak sets (`figstep`, `figstep_pro`)."
    ]
   },
   {
@@ -115,6 +115,7 @@
        " 'decoding_trust_toxicity',\n",
        " 'equitymedqa',\n",
        " 'figstep',\n",
+       " 'figstep_pro',\n",
        " 'forbidden_questions',\n",
        " 'garak_access_shell_commands',\n",
        " 'garak_audio_achilles_heel',\n",

doc/code/datasets/1_loading_datasets.py

@@ -68,8 +68,8 @@
 # (`garak_pypi_packages`, `garak_npm_packages`, `garak_crates_packages`,
 # `garak_rubygems_packages`, `garak_dart_packages`, `garak_perl_packages`,
 # `garak_raku_packages`), system-prompt libraries (`garak_drh_system_prompts`,
-# `garak_tm_system_prompts`), and an audio jailbreak set
-# (`garak_audio_achilles_heel`).
+# `garak_tm_system_prompts`), an audio jailbreak set
+# (`garak_audio_achilles_heel`), and visual jailbreak sets (`figstep`, `figstep_pro`).
 
 # %%
 from pyrit.datasets import SeedDatasetProvider

doc/scanner/0_scanner.md

@@ -35,7 +35,7 @@ PyRIT ships with scenarios organized into the following families:
 | **AIRT** | RapidResponse, Psychosocial, Cyber, Jailbreak, Leakage, Scam | [AIRT Scenarios](airt.ipynb) |
 | **Benchmark** | AdversarialBenchmark | [Benchmark Scenarios](benchmark.ipynb) |
 | **Foundry** | RedTeamAgent | [Foundry Scenarios](foundry.ipynb) |
-| **Garak** | Encoding | [Garak Scenarios](garak.ipynb) |
+| **Garak** | Encoding, FigStep | [Garak Scenarios](garak.ipynb) |
 
 Each scenario page shows how to run it with minimal configuration.
 

doc/scanner/1_pyrit_scan.ipynb

@@ -830,6 +830,12 @@
     "- `--max-retries <int>`: Maximum number of automatic retries if the scenario raises an exception\n",
     "- `--memory-labels <json>`: Additional labels to apply to all attack runs (must be a JSON string with string keys and values)\n",
     "\n",
+    "Dataset-backed scenarios can also select a supported dataset:\n",
+    "\n",
+    "```shell\n",
+    "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n",
+    "```\n",
+    "\n",
     "You can also use custom initialization scripts by passing file paths. It is relative to your current working directory, but to avoid confusion, full paths are always better:\n",
     "\n",
     "```shell\n",

doc/scanner/1_pyrit_scan.py

@@ -119,6 +119,12 @@
 # - `--max-retries <int>`: Maximum number of automatic retries if the scenario raises an exception
 # - `--memory-labels <json>`: Additional labels to apply to all attack runs (must be a JSON string with string keys and values)
 #
+# Dataset-backed scenarios can also select a supported dataset:
+#
+# ```shell
+# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1
+# ```
+#
 # You can also use custom initialization scripts by passing file paths. It is relative to your current working directory, but to avoid confusion, full paths are always better:
 #
 # ```shell

doc/scanner/garak.ipynb

@@ -14,8 +14,9 @@
     "data-exfiltration or cross-site-scripting payloads), a doctor probe (which applies the Policy\n",
     "Puppetry universal bypass), system-prompt-extraction probes (which test whether a target can be\n",
     "coaxed into revealing its own system prompt), package-hallucination probes (which test whether a\n",
-    "target recommends non-existent packages that an attacker could squat), and an audio probe (which\n",
-    "delivers spoken jailbreaks to multimodal targets).\n",
+    "target recommends non-existent packages that an attacker could squat), an audio probe (which\n",
+    "delivers spoken jailbreaks to multimodal targets), and FigStep visual jailbreaks (which place\n",
+    "harmful instructions in images).\n",
     "\n",
     "For full programming details, see the\n",
     "[Scenarios Programming Guide](../code/scenarios/0_scenarios.ipynb)."
@@ -35,9 +36,11 @@
     "from pyrit.output import output_scenario_async\n",
     "from pyrit.prompt_target import RealtimeTarget\n",
     "from pyrit.registry import TargetRegistry\n",
+    "from pyrit.scenario import DatasetAttackConfiguration\n",
     "from pyrit.scenario.garak import (\n",
     "    Encoding,\n",
     "    EncodingTechnique,\n",
+    "    FigStep,\n",
     "    SystemPromptExtraction,\n",
     "    SystemPromptExtractionTechnique,\n",
     ")\n",
@@ -119,6 +122,59 @@
    "cell_type": "markdown",
    "id": "5",
    "metadata": {},
+   "source": [
+    "## FigStep\n",
+    "\n",
+    "Tests whether a vision-language target follows harmful instructions that appear in an image.\n",
+    "`FigStep` sends one typographic image and carrier text. `FigStep-Pro` splits the visual prompt\n",
+    "across several images. Both variants reuse the built-in SafeBench-Tiny groups, images, and carrier\n",
+    "text. PyRIT scores whether the response completes the harmful objective. It does not only check\n",
+    "whether the response contains numbered steps.\n",
+    "\n",
+    "**CLI examples:**\n",
+    "\n",
+    "```bash\n",
+    "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1\n",
+    "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n",
+    "```\n",
+    "\n",
+    "> **Note:** The objective target must natively support multi-piece user messages and accept text\n",
+    "> and image input in the same message. Select exactly one of the `figstep` or `figstep_pro`\n",
+    "> datasets; unrelated named datasets are rejected because they do not contain the required visual\n",
+    "> payload. By default, PyRIT also sends each sampled objective as direct text. Use\n",
+    "> `--include-baseline False` to omit this comparison."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "figstep_dataset_config = DatasetAttackConfiguration(dataset_names=[\"figstep\"], max_dataset_size=1)\n",
+    "\n",
+    "figstep_scenario = FigStep()\n",
+    "figstep_scenario.set_params_from_args(  # type: ignore\n",
+    "    args={\n",
+    "        \"objective_target\": objective_target,\n",
+    "        \"dataset_config\": figstep_dataset_config,\n",
+    "    }\n",
+    ")\n",
+    "await figstep_scenario.initialize_async()  # type: ignore\n",
+    "\n",
+    "print(f\"Scenario: {figstep_scenario.name}\")\n",
+    "print(f\"Atomic attacks: {figstep_scenario.atomic_attack_count}\")\n",
+    "\n",
+    "figstep_result = await figstep_scenario.run_async()  # type: ignore\n",
+    "\n",
+    "await output_scenario_async(figstep_result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7",
+   "metadata": {},
    "source": [
     "## WebInjection\n",
     "\n",
@@ -143,7 +199,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6",
+   "id": "8",
    "metadata": {},
    "source": [
     "## Doctor\n",
@@ -167,7 +223,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "7",
+   "id": "9",
    "metadata": {},
    "source": [
     "## SystemPromptExtraction\n",
@@ -200,7 +256,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8",
+   "id": "10",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -222,7 +278,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9",
+   "id": "11",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,7 +287,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "10",
+   "id": "12",
    "metadata": {},
    "source": [
     "## PackageHallucination\n",
@@ -262,7 +318,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "11",
+   "id": "13",
    "metadata": {},
    "source": [
     "## AudioAchillesHeel\n",
@@ -290,7 +346,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "12",
+   "id": "14",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -318,7 +374,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "13",
+   "id": "15",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -327,7 +383,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "14",
+   "id": "16",
    "metadata": {},
    "source": [
     "For more details, see the [Scenarios Programming Guide](../code/scenarios/0_scenarios.ipynb) and\n",

doc/scanner/garak.py

@@ -18,8 +18,9 @@
 # data-exfiltration or cross-site-scripting payloads), a doctor probe (which applies the Policy
 # Puppetry universal bypass), system-prompt-extraction probes (which test whether a target can be
 # coaxed into revealing its own system prompt), package-hallucination probes (which test whether a
-# target recommends non-existent packages that an attacker could squat), and an audio probe (which
-# delivers spoken jailbreaks to multimodal targets).
+# target recommends non-existent packages that an attacker could squat), an audio probe (which
+# delivers spoken jailbreaks to multimodal targets), and FigStep visual jailbreaks (which place
+# harmful instructions in images).
 #
 # For full programming details, see the
 # [Scenarios Programming Guide](../code/scenarios/0_scenarios.ipynb).
@@ -30,9 +31,11 @@
 from pyrit.output import output_scenario_async
 from pyrit.prompt_target import RealtimeTarget
 from pyrit.registry import TargetRegistry
+from pyrit.scenario import DatasetAttackConfiguration
 from pyrit.scenario.garak import (
     Encoding,
     EncodingTechnique,
+    FigStep,
     SystemPromptExtraction,
     SystemPromptExtractionTechnique,
 )
@@ -89,6 +92,47 @@
 # %%
 await output_scenario_async(scenario_result)
 
+# %% [markdown]
+# ## FigStep
+#
+# Tests whether a vision-language target follows harmful instructions that appear in an image.
+# `FigStep` sends one typographic image and carrier text. `FigStep-Pro` splits the visual prompt
+# across several images. Both variants reuse the built-in SafeBench-Tiny groups, images, and carrier
+# text. PyRIT scores whether the response completes the harmful objective. It does not only check
+# whether the response contains numbered steps.
+#
+# **CLI examples:**
+#
+# ```bash
+# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1
+# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1
+# ```
+#
+# > **Note:** The objective target must natively support multi-piece user messages and accept text
+# > and image input in the same message. Select exactly one of the `figstep` or `figstep_pro`
+# > datasets; unrelated named datasets are rejected because they do not contain the required visual
+# > payload. By default, PyRIT also sends each sampled objective as direct text. Use
+# > `--include-baseline False` to omit this comparison.
+
+# %%
+figstep_dataset_config = DatasetAttackConfiguration(dataset_names=["figstep"], max_dataset_size=1)
+
+figstep_scenario = FigStep()
+figstep_scenario.set_params_from_args(  # type: ignore
+    args={
+        "objective_target": objective_target,
+        "dataset_config": figstep_dataset_config,
+    }
+)
+await figstep_scenario.initialize_async()  # type: ignore
+
+print(f"Scenario: {figstep_scenario.name}")
+print(f"Atomic attacks: {figstep_scenario.atomic_attack_count}")
+
+figstep_result = await figstep_scenario.run_async()  # type: ignore
+
+await output_scenario_async(figstep_result)
+
 # %% [markdown]
 # ## WebInjection
 #

pyrit/datasets/seed_datasets/remote/__init__.py

@@ -41,7 +41,12 @@
     _DecodingTrustToxicityDataset,
 )
 from pyrit.datasets.seed_datasets.remote.equitymedqa_dataset import _EquityMedQADataset
-from pyrit.datasets.seed_datasets.remote.figstep_dataset import FigStepCategory, FigStepVariant, _FigStepDataset
+from pyrit.datasets.seed_datasets.remote.figstep_dataset import (
+    FigStepCategory,
+    FigStepVariant,
+    _FigStepDataset,
+    _FigStepProDataset,
+)
 from pyrit.datasets.seed_datasets.remote.forbidden_questions_dataset import _ForbiddenQuestionsDataset
 from pyrit.datasets.seed_datasets.remote.garak_audio_dataset import _GarakAudioAchillesHeelDataset
 from pyrit.datasets.seed_datasets.remote.garak_package_hallucination_dataset import (
@@ -182,6 +187,7 @@
     "_DecodingTrustToxicityDataset",
     "_EquityMedQADataset",
     "_FigStepDataset",
+    "_FigStepProDataset",
     "_ForbiddenQuestionsDataset",
     "_GarakAudioAchillesHeelDataset",
     "_GarakCratesDataset",

pyrit/datasets/seed_datasets/remote/figstep_dataset.py

@@ -240,7 +240,7 @@ def __init__(
     @override
     def dataset_name(self) -> str:
         """The dataset name."""
-        return "figstep"
+        return self.variant.value
 
     @override
     async def fetch_dataset_async(self, *, cache: bool = True) -> SeedDataset:
@@ -660,3 +660,30 @@ async def _fetch_figstep_pro_sub_images_async(
 
         indexed_paths.sort(key=lambda item: item[0])
         return [path for _, path in indexed_paths]
+
+
+class _FigStepProDataset(_FigStepDataset):
+    """Provider entry for the FigStep-Pro SafeBench-Tiny variant."""
+
+    def __init__(
+        self,
+        *,
+        categories: list[FigStepCategory] | None = None,
+        source: str | None = None,
+        source_type: Literal["public_url", "file"] = "public_url",
+    ) -> None:
+        """
+        Initialize the FigStep-Pro provider.
+
+        Args:
+            categories (list[FigStepCategory] | None): Optional harmful-topic filter.
+            source (str | None): Optional question CSV URL or local path override.
+            source_type (Literal["public_url", "file"]): How to interpret ``source``.
+        """
+        super().__init__(
+            use_tiny=True,
+            variant=FigStepVariant.FIGSTEP_PRO,
+            categories=categories,
+            source=source,
+            source_type=source_type,
+        )

pyrit/scenario/scenarios/garak/__init__.py

@@ -8,6 +8,7 @@
 from pyrit.scenario.scenarios.garak.audio_achilles_heel import AudioAchillesHeel, AudioAchillesHeelTechnique
 from pyrit.scenario.scenarios.garak.doctor import Doctor, _build_doctor_technique
 from pyrit.scenario.scenarios.garak.encoding import Encoding, EncodingTechnique
+from pyrit.scenario.scenarios.garak.figstep import FigStep, FigStepTechnique
 from pyrit.scenario.scenarios.garak.package_hallucination import (
     PackageHallucination,
     PackageHallucinationTechnique,
@@ -41,6 +42,8 @@ def __getattr__(name: str) -> Any:
     "DoctorTechnique",
     "Encoding",
     "EncodingTechnique",
+    "FigStep",
+    "FigStepTechnique",
     "PackageHallucination",
     "PackageHallucinationTechnique",
     "SystemPromptExtraction",