diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 0f78764f5a..e52a794dfc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -47,6 +47,7 @@ Please provide the following information: - version of Python packages: please run the following snippet and paste the output: ```python import pyrit + pyrit.show_versions() ``` --> diff --git a/.github/instructions/converters.instructions.md b/.github/instructions/converters.instructions.md index d2b1030eb0..0cda8f453d 100644 --- a/.github/instructions/converters.instructions.md +++ b/.github/instructions/converters.instructions.md @@ -14,11 +14,10 @@ All converters MUST inherit from `Converter` and implement: ```python class MyConverter(Converter): - SUPPORTED_INPUT_TYPES = ("text",) # Required — non-empty tuple of PromptDataType values - SUPPORTED_OUTPUT_TYPES = ("text",) # Required — non-empty tuple of PromptDataType values + SUPPORTED_INPUT_TYPES = ("text",) # Required — non-empty tuple of PromptDataType values + SUPPORTED_OUTPUT_TYPES = ("text",) # Required — non-empty tuple of PromptDataType values - async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: - ... + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: ... ``` Missing or empty `SUPPORTED_INPUT_TYPES` / `SUPPORTED_OUTPUT_TYPES` raises `TypeError` at class definition time via `__init_subclass__`. @@ -51,8 +50,8 @@ All converters inherit `Identifiable`. Override `_build_identifier()` to include ```python def _build_identifier(self) -> ComponentIdentifier: return self._create_identifier( - params={"encoding": self._encoding}, # Behavioral params only - children={"target": self._target.get_identifier()} # If converter wraps a target + params={"encoding": self._encoding}, # Behavioral params only + children={"target": self._target.get_identifier()}, # If converter wraps a target ) ``` @@ -78,10 +77,10 @@ Use keyword-only arguments. Use `@apply_defaults` if the converter accepts targe ```python from pyrit.common.apply_defaults import apply_defaults + class MyConverter(Converter): @apply_defaults - def __init__(self, *, target: PromptTarget, template: str = "default") -> None: - ... + def __init__(self, *, target: PromptTarget, template: str = "default") -> None: ... ``` ### Keyword-only ``__init__`` is enforced @@ -97,13 +96,14 @@ The check is satisfied by either of: ```python def __init__(self, *, foo: str, bar: int = 0) -> None: ... + def __init__(self, *args: str, foo: str = "") -> None: ... # *args after self ``` It rejects: ```python -def __init__(self, foo: str, bar: int = 0) -> None: ... # missing * +def __init__(self, foo: str, bar: int = 0) -> None: ... # missing * ``` ## Exports and External Updates diff --git a/.github/instructions/datasets.instructions.md b/.github/instructions/datasets.instructions.md index cfd7d433db..2aa7849be7 100644 --- a/.github/instructions/datasets.instructions.md +++ b/.github/instructions/datasets.instructions.md @@ -73,7 +73,7 @@ class _MyDataset(_RemoteDatasetLoader): HF_DATASET_NAME: str = "owner/my-dataset" harm_categories: list[str] = ["harassment", "violence"] modalities: list[str] = ["text"] - size: str = "medium" # tiny <10, small 10-99, medium 100-499, large 500-4999, huge 5000+ + size: str = "medium" # tiny <10, small 10-99, medium 100-499, large 500-4999, huge 5000+ tags: set[str] = {"default", "safety"} ``` diff --git a/.github/instructions/output.instructions.md b/.github/instructions/output.instructions.md index 05f88fddec..27f187d8fc 100644 --- a/.github/instructions/output.instructions.md +++ b/.github/instructions/output.instructions.md @@ -61,6 +61,7 @@ Every new domain printer **must** have a corresponding convenience function adde ```python from pyrit.output.helpers import output_attack_async + await output_attack_async(result, format="pretty") ``` diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index 9089b3f6ad..e9e18680df 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -144,8 +144,8 @@ Technique members should represent **attack techniques** — the *how* of an att ```python class MyTechnique(ScenarioTechnique): - ALL = ("all", {"all"}) # Required aggregate - DEFAULT = ("default", {"default"}) # Recommended default aggregate + ALL = ("all", {"all"}) # Required aggregate + DEFAULT = ("default", {"default"}) # Recommended default aggregate SINGLE_TURN = ("single_turn", {"single_turn"}) # Category aggregate PromptSending = ("prompt_sending", {"single_turn", "default"}) @@ -208,8 +208,7 @@ Note: `atomic_attack_name` must remain unique per `AtomicAttack` for correct res Every scenario implements the single abstract extension point: ```python -async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: - ... +async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: ... ``` `initialize_async` resolves the run's inputs once (objective target, techniques, dataset @@ -226,6 +225,7 @@ Scenarios whose construction is the plain technique × dataset cross-product del ```python from pyrit.scenario.core.matrix_atomic_attack_builder import build_matrix_atomic_attacks + async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: return build_matrix_atomic_attacks( context=context, @@ -260,13 +260,13 @@ and is loaded into the registry by `TechniqueInitializer`. from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory AttackTechniqueFactory( - name="prompt_sending", # REQUIRED — must match the technique enum value + name="prompt_sending", # REQUIRED — must match the technique enum value attack_class=PromptSendingAttack, technique_tags=["core", "single_turn", "default"], attack_kwargs={"max_turns": 5}, - adversarial_chat=None, # None = resolve adversarial target lazily at create() + adversarial_chat=None, # None = resolve adversarial target lazily at create() seed_technique=None, - uses_adversarial=None, # None = auto-derive from attack signature/seeds + uses_adversarial=None, # None = auto-derive from attack signature/seeds scorer_override_policy=ScorerOverridePolicy.WARN, ) ``` @@ -318,10 +318,10 @@ population — that reintroduces baseline-vs-technique population divergence und ```python AtomicAttack( - atomic_attack_name=technique_name, # groups related attacks + atomic_attack_name=technique_name, # groups related attacks attack_technique=AttackTechnique(attack=attack_instance), # bundles the AttackStrategy - seed_groups=list(seed_groups), # must be non-empty - memory_labels=context.memory_labels, # from the context snapshot + seed_groups=list(seed_groups), # must be non-empty + memory_labels=context.memory_labels, # from the context snapshot ) ``` diff --git a/.github/instructions/style-guide.instructions.md b/.github/instructions/style-guide.instructions.md index 3ca0c7626c..1bba6d4113 100644 --- a/.github/instructions/style-guide.instructions.md +++ b/.github/instructions/style-guide.instructions.md @@ -19,21 +19,25 @@ async def _send_async(self): with open(self.file_path, "rb") as fp: return fp.read() + # CORRECT — async file read async def _send_async(self): async with aiofiles.open(self.file_path, "rb") as fp: return await fp.read() + # WRONG — sync-only library called directly async def _read_audio_async(self, path): with wave.open(path, "rb") as wav: return wav.readframes(wav.getnframes()) + # CORRECT — wrap blocking lib in to_thread def _read_wav_sync(path): with wave.open(path, "rb") as wav: return wav.readframes(wav.getnframes()) + async def _read_audio_async(self, path): return await asyncio.to_thread(_read_wav_sync, path) ``` @@ -47,8 +51,8 @@ async def _read_audio_async(self, path): ```python # CORRECT -async def send_prompt_async(self, prompt: str) -> Message: - ... +async def send_prompt_async(self, prompt: str) -> Message: ... + # INCORRECT async def send_prompt(self, prompt: str) -> Message: # Missing _async suffix @@ -68,8 +72,8 @@ async def send_prompt(self, prompt: str) -> Message: # Missing _async suffix ```python # CORRECT -def _validate_input(self, data: dict) -> None: - ... +def _validate_input(self, data: dict) -> None: ... + # INCORRECT def validate_input(self, data: dict) -> None: # Should be private @@ -94,11 +98,11 @@ def validate_input(self, data: dict) -> None: # Should be private ```python # CORRECT -def process_data(self, *, data: list[str], threshold: float = 0.5) -> dict[str, Any]: - ... +def process_data(self, *, data: list[str], threshold: float = 0.5) -> dict[str, Any]: ... + + +def get_name(self) -> str | None: ... -def get_name(self) -> str | None: - ... # INCORRECT def process_data(self, data, threshold=0.5): # Missing all type annotations @@ -113,18 +117,11 @@ def process_data(self, data, threshold=0.5): # Missing all type annotations ```python # CORRECT -def __init__( - self, - *, - target: PromptTarget, - scorer: Scorer | None = None, - max_retries: int = 3 -) -> None: - ... +def __init__(self, *, target: PromptTarget, scorer: Scorer | None = None, max_retries: int = 3) -> None: ... + # INCORRECT -def __init__(self, target: PromptTarget, scorer: Scorer | None = None, max_retries: int = 3): - ... +def __init__(self, target: PromptTarget, scorer: Scorer | None = None, max_retries: int = 3): ... ``` ### Forwarded Constructor Parameters @@ -140,8 +137,7 @@ def __init__(self, target: PromptTarget, scorer: Scorer | None = None, max_retri ```python # CORRECT -def process(self, data: str) -> str: - ... +def process(self, data: str) -> str: ... ``` ## Imports @@ -163,10 +159,13 @@ third-party packages (`transformers`, `azure.storage.blob`, `alembic`, `openai`, def main() -> int: parsed_args = parse_args() from pyrit.cli import frontend_core # deferred: heavy + ... + async def _create_container_client_async(self): from azure.storage.blob.aio import ContainerClient # deferred: heavy + ... ``` @@ -237,12 +236,7 @@ from typing import Self, override ```python def calculate_score( - self, - *, - response: str, - objective: str, - threshold: float = 0.8, - max_attempts: int | None = None + self, *, response: str, objective: str, threshold: float = 0.8, max_attempts: int | None = None ) -> Score: """ Calculate the score for a response against an objective. @@ -281,6 +275,7 @@ navigation in the rendered docs without any extra markup. # WRONG — reST roles render as literal `:class:\`SeedPrompt\`` under MyST, # and the pre-commit guard will reject them """Returns a :class:`SeedPrompt` instance.""" + """Delegate to :func:`download_files_async` (deprecated alias).""" """See :meth:`PromptTarget.apply_capabilities` for details.""" @@ -318,6 +313,7 @@ class TreeOfAttacksAttack(AttackStrategy): DEFAULT_TREE_DEPTH: int = 5 MIN_CONFIDENCE_THRESHOLD: float = 0.7 + # INCORRECT DEFAULT_TREE_WIDTH = 3 # Should be inside class DEFAULT_TREE_DEPTH = 5 @@ -343,11 +339,13 @@ async def execute_attack_async(self, *, context: AttackContext) -> AttackResult: return result + def _validate_context(self, context: AttackContext) -> None: """Validate the attack context.""" if not context.objective: raise ValueError("Context must have an objective") + # INCORRECT - Too long and doing too many things async def execute_attack_async(self, *, context: AttackContext) -> AttackResult: # 50+ lines of mixed validation, preparation, sending, and evaluation logic @@ -372,9 +370,7 @@ async def execute_attack_async(self, *, context: AttackContext) -> AttackResult: ```python # CORRECT if not self._model: - raise ValueError( - "Model not initialized. Call initialize_model() before executing attack." - ) + raise ValueError("Model not initialized. Call initialize_model() before executing attack.") # INCORRECT if not self._model: @@ -397,6 +393,7 @@ def process_items(self, *, items: list[str]) -> list[str]: # Main logic for multiple items return [self._process_single(item) for item in items] + # INCORRECT - Excessive nesting def process_items(self, *, items: list[str]) -> list[str]: if items: @@ -417,6 +414,7 @@ Set `removed_in` to **current version + 2 minor versions** (e.g. `0.14.x` → `r ```python from pyrit.common.deprecation import print_deprecation_message + def old_method(self, *, foo: str) -> None: print_deprecation_message( old_item="MyClass.old_method", @@ -431,6 +429,7 @@ def old_method(self, *, foo: str) -> None: ```python # INCORRECT - bypasses the helper, breaks consistent formatting and filtering import warnings + warnings.warn("foo is deprecated, use bar", DeprecationWarning, stacklevel=2) ``` @@ -484,6 +483,7 @@ async with self._get_client() as client: # For custom resources from contextlib import asynccontextmanager + @asynccontextmanager async def temporary_config(self, **kwargs): old_config = self._config.copy() @@ -508,12 +508,14 @@ def is_complete(self) -> bool: """Whether the attack is complete.""" return self._status == AttackStatus.COMPLETE + # INCORRECT - verb-phrase docstring, flagged by Ruff D421 @property def is_complete(self) -> bool: """Check if the attack is complete.""" return self._status == AttackStatus.COMPLETE + # INCORRECT - Too complex for property @property def analysis_report(self) -> str: @@ -531,17 +533,12 @@ def analysis_report(self) -> str: ```python # CORRECT class AttackExecutor: - def __init__( - self, - *, - target: PromptTarget, - scorer: Scorer, - logger: logging.Logger | None = None - ) -> None: + def __init__(self, *, target: PromptTarget, scorer: Scorer, logger: logging.Logger | None = None) -> None: self._target = target self._scorer = scorer self._logger = logger or logging.getLogger(__name__) + # INCORRECT class AttackExecutor: def __init__(self): @@ -560,6 +557,7 @@ def calculate_score(response: str, objective: str) -> float: # Logic without side effects return score + async def evaluate_response_async(self, *, response: str) -> Score: """I/O function that uses the pure function.""" score_value = calculate_score(response, self._objective) @@ -580,6 +578,7 @@ def process_large_dataset(self, *, file_path: Path) -> Generator[Result, None, N for line in f: yield self._process_line(line) + # INCORRECT def process_large_dataset(self, *, file_path: Path) -> list[Result]: with open(file_path) as f: diff --git a/.github/instructions/targets.instructions.md b/.github/instructions/targets.instructions.md index 36b985e736..f25d11c60b 100644 --- a/.github/instructions/targets.instructions.md +++ b/.github/instructions/targets.instructions.md @@ -34,10 +34,7 @@ class MyTarget(PromptTarget): ) self._api_key = api_key - async def _send_prompt_to_target_async( - self, *, normalized_conversation: list[Message] - ) -> list[Message]: - ... + async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]: ... ``` ``send_prompt_async`` (the public entry point) is ``@final`` and MUST NOT @@ -61,13 +58,14 @@ The check is satisfied by either of: ```python def __init__(self, *, endpoint: str, api_key: str) -> None: ... + def __init__(self, *args: Any, **kwargs: Any) -> None: ... # *args after self ``` It rejects: ```python -def __init__(self, endpoint: str, api_key: str) -> None: ... # missing * +def __init__(self, endpoint: str, api_key: str) -> None: ... # missing * ``` > [!NOTE] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75876d6b70..b84b4147b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: - id: detect-private-key - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.4 + rev: v0.16.5 hooks: - id: ruff-format - id: ruff-check diff --git a/doc/blog/2025_03_03.md b/doc/blog/2025_03_03.md index f184b1264a..879d1404a7 100644 --- a/doc/blog/2025_03_03.md +++ b/doc/blog/2025_03_03.md @@ -39,14 +39,10 @@ There are a couple of ways OpenAI serializes messages. One of the first ways was ```python from openai import OpenAI + client = OpenAI() -completion = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "user", "content": "Hello!"} - ] -) +completion = client.chat.completions.create(model="gpt-4o", messages=[{"role": "user", "content": "Hello!"}]) print(completion.choices[0].message) ``` diff --git a/doc/code/registry/0_registry.md b/doc/code/registry/0_registry.md index 58998a56aa..84d87ad28e 100644 --- a/doc/code/registry/0_registry.md +++ b/doc/code/registry/0_registry.md @@ -34,6 +34,7 @@ This makes it easy to write code that inspects any registry: ```python from pyrit.registry import ScenarioRegistry + def show_registry_contents(registry) -> None: for name in registry.get_names(): print(name) diff --git a/doc/code/setup/default_values.md b/doc/code/setup/default_values.md index cec61ce4f4..6db6c72b6b 100644 --- a/doc/code/setup/default_values.md +++ b/doc/code/setup/default_values.md @@ -21,6 +21,7 @@ Classes that want to participate in the default value system use the `@apply_def ```python from pyrit.common.apply_defaults import apply_defaults + class MyConverter(Converter): @apply_defaults def __init__(self, *, converter_target: PromptTarget | None = None, temperature: float | None = None): diff --git a/doc/getting_started/install_docker.md b/doc/getting_started/install_docker.md index 51aa2e5c7e..ba8103bf2c 100644 --- a/doc/getting_started/install_docker.md +++ b/doc/getting_started/install_docker.md @@ -86,6 +86,7 @@ Once JupyterLab is open: ```python import pyrit + print(pyrit.__version__) ``` @@ -188,6 +189,7 @@ To use NVIDIA GPUs with PyRIT: ```python import torch + print(f"CUDA available: {torch.cuda.is_available()}") print(f"GPU count: {torch.cuda.device_count()}") ``` diff --git a/doc/getting_started/install_local.md b/doc/getting_started/install_local.md index eef454f844..62f0d8c37d 100644 --- a/doc/getting_started/install_local.md +++ b/doc/getting_started/install_local.md @@ -32,6 +32,7 @@ Notebooks and your PyRIT installation must be on the same version. This pip inst Or in Python: ```python import pyrit + print(pyrit.__version__) ``` diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index fb498a9bec..54c530537a 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -380,8 +380,8 @@ from pyrit.setup import ConfigurationLoader # Layer 2 and 3 overrides are optional keyword arguments: config = ConfigurationLoader.load_with_overrides( config_file=Path("./my_project.yaml"), # Layer 2: explicit config file (omit to skip) - memory_db_type="in_memory", # Layer 3: override database type - initializers=["target", "scorer"], # Layer 3: override initializers + memory_db_type="in_memory", # Layer 3: override database type + initializers=["target", "scorer"], # Layer 3: override initializers ) await config.initialize_pyrit_async() diff --git a/docker/README.md b/docker/README.md index efdb2cb7a6..ab53cca9ae 100644 --- a/docker/README.md +++ b/docker/README.md @@ -127,6 +127,7 @@ Start a new notebook in JupyterLab and try the following: ```python import pyrit + print(pyrit.__version__) # Example PyRIT usage: