Merge branch 'main' into bjagdagdorj/converter_mixed_media_type

Author: jbolor21

.env_example

@@ -79,6 +79,11 @@ ADVERSARIAL_CHAT_ENDPOINT="https://xxxxx.openai.azure.com/openai/v1"
 ADVERSARIAL_CHAT_KEY="xxxxx"
 ADVERSARIAL_CHAT_MODEL="deployment-name"
 
+# Objective Scorer chat target (used in scorers in scenarios)
+OBJECTIVE_SCORER_CHAT_ENDPOINT="https://xxxxx.openai.azure.com/openai/v1"
+OBJECTIVE_SCORER_CHAT_KEY="xxxxx"
+OBJECTIVE_SCORER_CHAT_MODEL="deployment-name"
+
 AZURE_FOUNDRY_DEEPSEEK_ENDPOINT="https://xxxxx.eastus2.models.ai.azure.com"
 AZURE_FOUNDRY_DEEPSEEK_KEY="xxxxx"
 AZURE_FOUNDRY_DEEPSEEK_MODEL=""

.github/instructions/scenarios.instructions.md

@@ -11,11 +11,15 @@ Scenarios orchestrate multi-attack security testing campaigns. Each scenario gro
 All scenarios inherit from `Scenario` (ABC) and must:
 
 1. **Define `VERSION`** as a class constant (increment on breaking changes)
-2. **Implement three abstract methods:**
+2. **Optionally declare `BASELINE_POLICY`** (defaults to `BaselinePolicy.Enabled` — a baseline `PromptSendingAttack` is prepended and callers can opt out per run via `initialize_async(include_baseline=False)`):
+   - `BaselinePolicy.Disabled` — baseline supported but off by default (e.g. `Jailbreak`, where templates dominate the run).
+   - `BaselinePolicy.Forbidden` — baseline is meaningless for this scenario's comparison axis (e.g. `AdversarialBenchmark`, which compares against gold-standard answers). Explicit `include_baseline=True` raises `ValueError`.
+3. **Implement three abstract methods:**
 
 ```python
 class MyScenario(Scenario):
     VERSION: int = 1
+    BASELINE_POLICY: ClassVar[BaselinePolicy] = BaselinePolicy.Enabled
 
     @classmethod
     def get_strategy_class(cls) -> type[ScenarioStrategy]:
@@ -30,7 +34,7 @@ class MyScenario(Scenario):
         return DatasetConfiguration(dataset_names=["my_dataset"])
 ```
 
-3. **Optionally override `_get_atomic_attacks_async()`** — the base class provides a default
+4. **Optionally override `_get_atomic_attacks_async()`** — the base class provides a default
    that uses the factory/registry pattern (see "AtomicAttack Construction" below).
    Only override if your scenario needs custom attack construction logic.
 
@@ -154,6 +158,8 @@ The default implementation:
 Only override when the scenario **cannot** use the factory/registry pattern — e.g., scenarios
 with custom composite logic, per-strategy converter stacks, or non-standard attack construction.
 
+Overrides that want baseline support must emit it themselves by calling `self._build_baseline_atomic_attack(seed_groups=...)` with the same seeds used for the strategy attacks and prepending the result. The base implementation emits baseline automatically; passing freshly resolved seeds reintroduces ADO 9012 (baseline-vs-strategy population divergence under `max_dataset_size`).
+
 ### Manual AtomicAttack construction (for overrides):
 
 ```python

.github/workflows/build_and_test.yml

@@ -33,6 +33,11 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v5
+        with:
+          # Full history is required so pre-commit hooks (notably
+          # enforce_alembic_revision_immutability) can compute merge-bases and
+          # diff ranges against origin/main.
+          fetch-depth: 0
 
       - uses: actions/setup-python@v6
         with:

.pyrit_conf_example

@@ -111,6 +111,26 @@ operation: op_trash_panda
 #   - /path/to/.env
 #   - /path/to/.env.local
 
+# Max Concurrent Scenario Runs
+# ----------------------------
+# Maximum number of scenario runs that can execute concurrently in the backend.
+# Applies only to the pyrit_backend server.
+max_concurrent_scenario_runs: 3
+
+# Custom Initializer Registration (REST API)
+# -------------------------------------------
+# When true, the REST API accepts POST /api/initializers to register custom
+# initializer scripts and DELETE /api/initializers/{name} to remove custom
+# initializers.
+#
+# ⚠️  WARNING: Enabling this allows arbitrary Python code execution on the
+# server via the REST API. Only enable on trusted networks.
+# The pyrit_backend default host is localhost, which limits exposure.
+# If you bind to 0.0.0.0, ensure you are on a trusted network.
+#
+# Default: false
+allow_custom_initializers: false
+
 # Silent Mode
 # -----------
 # If true, suppresses print statements during initialization.

build_scripts/enforce_alembic_revision_immutability.py

@@ -4,36 +4,86 @@
 """
 Migration history must be immutable. This hook enforces that by preventing deletion or updates to migration scripts.
 
-Checks both staged changes (local pre-commit) and the full branch diff against origin/main (CI).
+Checks staged changes (local pre-commit), the full branch diff against origin/main (CI PRs),
+and the previous commit (CI merge-queue / push-to-main).
 """
 
+import os
 import subprocess
 import sys
 
 _VERSIONS_PATH = "pyrit/memory/alembic/versions/"
 
 
-def _git(*args: str) -> str:
-    result = subprocess.run(["git", *args], capture_output=True, text=True)
-    return result.stdout.strip()
+def _git(*args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(["git", *args], capture_output=True, text=True)
 
 
-def _has_non_add_changes(diff_spec: list[str]) -> bool:
-    output = _git("diff", "--name-status", *diff_spec, "--", _VERSIONS_PATH)
-    return any(line and not line.startswith("A") for line in output.splitlines())
+def _git_stdout(*args: str) -> str:
+    return _git(*args).stdout.strip()
+
+
+def _get_violations(diff_spec: list[str]) -> list[str]:
+    """Return lines from ``git diff --name-status`` that are not pure additions."""
+    output = _git_stdout("diff", "--name-status", *diff_spec, "--", _VERSIONS_PATH)
+    return [line for line in output.splitlines() if line and not line.startswith("A")]
+
+
+def _in_ci() -> bool:
+    return os.environ.get("CI", "").lower() in {"1", "true"} or "GITHUB_ACTIONS" in os.environ
+
+
+def _fail_ci(reason: str) -> bool:
+    """Fail closed in CI when we can't perform the check, pass through locally."""
+    if _in_ci():
+        print(f"[ERROR] Cannot verify alembic revision immutability: {reason}")
+        print("        Ensure the CI checkout has full history (fetch-depth: 0).")
+        return True
+    return False
 
 
 def has_revision_violations() -> bool:
     # Local pre-commit: check staged changes
-    if _has_non_add_changes(["--cached"]):
+    violations = _get_violations(["--cached"])
+    if violations:
+        _report(violations)
         return True
 
-    # CI: check full branch diff against origin/main
-    merge_base = _git("merge-base", "origin/main", "HEAD")
-    return bool(merge_base and _has_non_add_changes([f"{merge_base}...HEAD"]))
+    # CI (PR): diff branch against its merge-base with origin/main.
+    # The three-dot syntax (A...B) resolves to ``git diff $(merge-base A B) B``
+    # automatically, so we don't need a separate merge-base call.  When
+    # origin/main is missing (shallow clone) git exits non-zero.
+    pr_diff = _git("diff", "--name-status", "origin/main...HEAD", "--", _VERSIONS_PATH)
+    if pr_diff.returncode == 0:
+        violations = [line for line in pr_diff.stdout.strip().splitlines() if line and not line.startswith("A")]
+        if violations:
+            _report(violations)
+            return True
+    elif _fail_ci("origin/main is not available (shallow clone?)"):
+        return True
+
+    # CI (merge-queue / push-to-main): on main the branch *is* origin/main, so
+    # the diff above is empty.  Compare HEAD against its first parent to catch
+    # deletions or modifications introduced by the merge commit itself.
+    head_parent = _git("rev-parse", "--verify", "HEAD~1")
+    if head_parent.returncode == 0:
+        violations = _get_violations(["HEAD~1..HEAD"])
+        if violations:
+            _report(violations)
+            return True
+    elif _fail_ci("HEAD~1 is not available (shallow clone?)"):
+        return True
+
+    return False
+
+
+def _report(violations: list[str]) -> None:
+    print("[ERROR] Migration scripts can only be added, not modified or deleted.")
+    print("The following disallowed changes were detected:")
+    for v in violations:
+        print(f"  {v}")
 
 
 if __name__ == "__main__":
     if has_revision_violations():
-        print("[ERROR] Migration scripts can only be added, not modified or deleted.")
         sys.exit(1)