FEAT: Adding Scorables as an entry point

doc/code/memory/5_advanced_memory.ipynb

@@ -602,7 +602,7 @@
     }
    ],
    "source": [
-    "from pyrit.models import Message\n",
+    "from pyrit.models import Message, MessageScorable\n",
     "from pyrit.score import SubStringScorer\n",
     "\n",
     "# Create three scorers with different substrings\n",
@@ -623,9 +623,10 @@
     "\n",
     "# Score every response with both scorers — scores are automatically persisted in memory\n",
     "for msg in assistant_messages:\n",
-    "    await scorer_molotov.score_async(msg)  # type: ignore\n",
-    "    await scorer_launder.score_async(msg)  # type: ignore\n",
-    "    await scorer_assist.score_async(msg)  # type: ignore\n",
+    "    scorable = MessageScorable.from_message(msg)\n",
+    "    await scorer_molotov.score_async(scorable=scorable)  # type: ignore\n",
+    "    await scorer_launder.score_async(scorable=scorable)  # type: ignore\n",
+    "    await scorer_assist.score_async(scorable=scorable)  # type: ignore\n",
     "\n",
     "print(f\"Scored {len(assistant_messages)} messages with all three scorers.\")"
    ]

doc/code/memory/5_advanced_memory.py

@@ -246,7 +246,7 @@
 # retrieve only the scores produced by a specific scorer.
 
 # %%
-from pyrit.models import Message
+from pyrit.models import Message, MessageScorable
 from pyrit.score import SubStringScorer
 
 # Create three scorers with different substrings
@@ -267,9 +267,10 @@
 
 # Score every response with both scorers — scores are automatically persisted in memory
 for msg in assistant_messages:
-    await scorer_molotov.score_async(msg)  # type: ignore
-    await scorer_launder.score_async(msg)  # type: ignore
-    await scorer_assist.score_async(msg)  # type: ignore
+    scorable = MessageScorable.from_message(msg)
+    await scorer_molotov.score_async(scorable=scorable)  # type: ignore
+    await scorer_launder.score_async(scorable=scorable)  # type: ignore
+    await scorer_assist.score_async(scorable=scorable)  # type: ignore
 
 print(f"Scored {len(assistant_messages)} messages with all three scorers.")
 

doc/code/scoring/1_true_false_scorers.ipynb

@@ -365,7 +365,7 @@
     }
    ],
    "source": [
-    "from pyrit.models import MessagePiece\n",
+    "from pyrit.models import MessagePiece, ScoringExpectation\n",
     "from pyrit.score import SelfAskRefusalScorer\n",
     "\n",
     "refusal_scorer = SelfAskRefusalScorer(chat_target=OpenAIChatTarget())\n",
@@ -382,8 +382,12 @@
     "refusal = MessagePiece(role=\"assistant\", original_value=\"Sorry, but I can't help with that.\").to_message()\n",
     "refusal.set_response_not_in_memory()\n",
     "\n",
-    "blocked_score = (await refusal_scorer.score_async(message=blocked))[0]  # type: ignore\n",
-    "refusal_score = (await refusal_scorer.score_async(message=refusal, objective=\"Explain how to pick a lock\"))[0]  # type: ignore\n",
+    "blocked_score = (await refusal_scorer.score_message_async(message=blocked))[0]  # type: ignore\n",
+    "refusal_score = (\n",
+    "    await refusal_scorer.score_message_async(\n",
+    "        message=refusal, expectation=ScoringExpectation(objective=\"Explain how to pick a lock\")\n",
+    "    )\n",
+    ")[0]  # type: ignore\n",
     "\n",
     "print(f\"[refusal] blocked response -> {blocked_score.get_value()}\")\n",
     "print(f\"[refusal] explicit refusal -> {refusal_score.get_value()}\")"

doc/code/scoring/1_true_false_scorers.py

@@ -176,7 +176,7 @@
 # which is common with content filters. A partially blocked response still carries content, so
 # it is scored normally by the LLM rather than short-circuited.
 # %%
-from pyrit.models import MessagePiece
+from pyrit.models import MessagePiece, ScoringExpectation
 from pyrit.score import SelfAskRefusalScorer
 
 refusal_scorer = SelfAskRefusalScorer(chat_target=OpenAIChatTarget())
@@ -193,8 +193,12 @@
 refusal = MessagePiece(role="assistant", original_value="Sorry, but I can't help with that.").to_message()
 refusal.set_response_not_in_memory()
 
-blocked_score = (await refusal_scorer.score_async(message=blocked))[0]  # type: ignore
-refusal_score = (await refusal_scorer.score_async(message=refusal, objective="Explain how to pick a lock"))[0]  # type: ignore
+blocked_score = (await refusal_scorer.score_message_async(message=blocked))[0]  # type: ignore
+refusal_score = (
+    await refusal_scorer.score_message_async(
+        message=refusal, expectation=ScoringExpectation(objective="Explain how to pick a lock")
+    )
+)[0]  # type: ignore
 
 print(f"[refusal] blocked response -> {blocked_score.get_value()}")
 print(f"[refusal] explicit refusal -> {refusal_score.get_value()}")

doc/code/scoring/2_float_scale_scorers.ipynb

@@ -99,7 +99,7 @@
     "\n",
     "from pyrit.auth import get_azure_token_provider\n",
     "from pyrit.memory import CentralMemory\n",
-    "from pyrit.models import Message, MessagePiece\n",
+    "from pyrit.models import Message, MessagePiece, MessageScorable\n",
     "from pyrit.score import AzureContentFilterScorer\n",
     "\n",
     "azure_content_filter = AzureContentFilterScorer(\n",
@@ -120,7 +120,7 @@
     "# The score table has a foreign key on the message, so write it to memory first.\n",
     "CentralMemory.get_memory_instance().add_message_to_memory(request=response)\n",
     "\n",
-    "scores = await azure_content_filter.score_async(response)  # type: ignore\n",
+    "scores = await azure_content_filter.score_async(scorable=MessageScorable.from_message(response))  # type: ignore\n",
     "for score in scores:\n",
     "    # One score per harm category; score_metadata holds the original 0-7 severity.\n",
     "    print(f\"{score.score_category}: value={score.get_value()} metadata={score.score_metadata}\")"
@@ -289,7 +289,7 @@
     }
    ],
    "source": [
-    "from pyrit.models import MessagePiece\n",
+    "from pyrit.models import MessagePiece, MessageScorable\n",
     "from pyrit.score import InsecureCodeScorer\n",
     "\n",
     "insecure_code_scorer = InsecureCodeScorer.from_harm_categories(chat_target=OpenAIChatTarget())\n",
@@ -302,7 +302,7 @@
     "request = MessagePiece(role=\"assistant\", original_value=snippet, conversation_id=str(uuid4())).to_message()\n",
     "insecure_code_scorer._memory.add_message_to_memory(request=request)\n",
     "\n",
-    "scored = (await insecure_code_scorer.score_async(request))[0]  # type: ignore\n",
+    "scored = (await insecure_code_scorer.score_async(scorable=MessageScorable.from_message(request)))[0]  # type: ignore\n",
     "print(f\"[insecure code] risk={scored.get_value()}\")\n",
     "print(f\"rationale: {scored.score_rationale}\")"
    ]

doc/code/scoring/2_float_scale_scorers.py

@@ -43,7 +43,7 @@
 
 from pyrit.auth import get_azure_token_provider
 from pyrit.memory import CentralMemory
-from pyrit.models import Message, MessagePiece
+from pyrit.models import Message, MessagePiece, MessageScorable
 from pyrit.score import AzureContentFilterScorer
 
 azure_content_filter = AzureContentFilterScorer(
@@ -64,7 +64,7 @@
 # The score table has a foreign key on the message, so write it to memory first.
 CentralMemory.get_memory_instance().add_message_to_memory(request=response)
 
-scores = await azure_content_filter.score_async(response)  # type: ignore
+scores = await azure_content_filter.score_async(scorable=MessageScorable.from_message(response))  # type: ignore
 for score in scores:
     # One score per harm category; score_metadata holds the original 0-7 severity.
     print(f"{score.score_category}: value={score.get_value()} metadata={score.score_metadata}")
@@ -146,7 +146,7 @@
 #
 # Rates how risky a code snippet is, flagging vulnerabilities like injection or weak auth.
 # %%
-from pyrit.models import MessagePiece
+from pyrit.models import MessagePiece, MessageScorable
 from pyrit.score import InsecureCodeScorer
 
 insecure_code_scorer = InsecureCodeScorer.from_harm_categories(chat_target=OpenAIChatTarget())
@@ -159,7 +159,7 @@ def authenticate_user(username, password):
 request = MessagePiece(role="assistant", original_value=snippet, conversation_id=str(uuid4())).to_message()
 insecure_code_scorer._memory.add_message_to_memory(request=request)
 
-scored = (await insecure_code_scorer.score_async(request))[0]  # type: ignore
+scored = (await insecure_code_scorer.score_async(scorable=MessageScorable.from_message(request)))[0]  # type: ignore
 print(f"[insecure code] risk={scored.get_value()}")
 print(f"rationale: {scored.score_rationale}")
 

doc/code/scoring/3_combining_scorers.ipynb

@@ -296,7 +296,7 @@
     "import uuid\n",
     "\n",
     "from pyrit.memory import CentralMemory\n",
-    "from pyrit.models import MessagePiece\n",
+    "from pyrit.models import MessagePiece, MessageScorable\n",
     "from pyrit.score import create_conversation_scorer\n",
     "\n",
     "memory = CentralMemory.get_memory_instance()\n",
@@ -317,7 +317,7 @@
     "conversation_scorer = create_conversation_scorer(scorer=persona_breach_scorer)\n",
     "\n",
     "# Any message from the conversation works as the trigger.\n",
-    "score = (await conversation_scorer.score_async(turns[0]))[0]  # type: ignore\n",
+    "score = (await conversation_scorer.score_async(scorable=MessageScorable.from_message(turns[0])))[0]  # type: ignore\n",
     "print(f\"[conversation] persona breach across turns -> {score.get_value()}\")"
    ]
   },

doc/code/scoring/3_combining_scorers.py

@@ -160,7 +160,7 @@
 import uuid
 
 from pyrit.memory import CentralMemory
-from pyrit.models import MessagePiece
+from pyrit.models import MessagePiece, MessageScorable
 from pyrit.score import create_conversation_scorer
 
 memory = CentralMemory.get_memory_instance()
@@ -181,7 +181,7 @@
 conversation_scorer = create_conversation_scorer(scorer=persona_breach_scorer)
 
 # Any message from the conversation works as the trigger.
-score = (await conversation_scorer.score_async(turns[0]))[0]  # type: ignore
+score = (await conversation_scorer.score_async(scorable=MessageScorable.from_message(turns[0])))[0]  # type: ignore
 print(f"[conversation] persona breach across turns -> {score.get_value()}")
 
 # %% [markdown]

doc/code/targets/round_robin_target.ipynb

@@ -99,7 +99,7 @@
     "import os\n",
     "\n",
     "from pyrit.auth import get_azure_openai_auth\n",
-    "from pyrit.models import Message\n",
+    "from pyrit.models import Message, MessageScorable\n",
     "from pyrit.prompt_normalizer import PromptNormalizer\n",
     "from pyrit.prompt_target import OpenAIChatTarget, RoundRobinTarget\n",
     "from pyrit.setup import IN_MEMORY, initialize_pyrit_async\n",
@@ -788,7 +788,7 @@
     "# You may want to use `score_prompts_batch_async` like below in practice for efficiency\n",
     "# await scorer.score_prompts_batch_async(messages=response_messages)  # type: ignore\n",
     "for i, response_message in enumerate(response_messages):\n",
-    "    scores = await scorer.score_async(message=response_message)  # type: ignore\n",
+    "    scores = await scorer.score_async(scorable=MessageScorable.from_message(response_message))  # type: ignore\n",
     "\n",
     "    # The scorer's internal LLM response has inner_target_identifier in metadata.\n",
     "    # We can check the round-robin counter to determine which target was used.\n",

doc/code/targets/round_robin_target.py

@@ -38,7 +38,7 @@
 import os
 
 from pyrit.auth import get_azure_openai_auth
-from pyrit.models import Message
+from pyrit.models import Message, MessageScorable
 from pyrit.prompt_normalizer import PromptNormalizer
 from pyrit.prompt_target import OpenAIChatTarget, RoundRobinTarget
 from pyrit.setup import IN_MEMORY, initialize_pyrit_async
@@ -252,7 +252,7 @@
 # You may want to use `score_prompts_batch_async` like below in practice for efficiency
 # await scorer.score_prompts_batch_async(messages=response_messages)  # type: ignore
 for i, response_message in enumerate(response_messages):
-    scores = await scorer.score_async(message=response_message)  # type: ignore
+    scores = await scorer.score_async(scorable=MessageScorable.from_message(response_message))  # type: ignore
 
     # The scorer's internal LLM response has inner_target_identifier in metadata.
     # We can check the round-robin counter to determine which target was used.

pyproject.toml

@@ -174,6 +174,11 @@ addopts = [
 pythonpath = ["."]
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
+filterwarnings = [
+    # Keep the in-repo suite on the scorable/expectation contract. Tests that cover the
+    # shim itself opt back in with pytest.warns.
+    "error:Scorer\\.score_async:DeprecationWarning",
+]
 
 [tool.ty]
 [tool.ty.rules]

pyrit/executor/attack/multi_turn/crescendo.py

@@ -10,21 +10,11 @@
 
 from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults
 from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH
-from pyrit.exceptions import (
-    ComponentRole,
-    execution_context,
-)
-from pyrit.executor.attack.component import (
-    ConversationManager,
-    PrependedConversationConfig,
-)
+from pyrit.exceptions import ComponentRole, execution_context
+from pyrit.executor.attack.component import ConversationManager, PrependedConversationConfig
 from pyrit.executor.attack.component.adversarial_conversation_manager import _AdversarialConversationManager
 from pyrit.executor.attack.component.modality_router import _ModalityFeedbackRouter
-from pyrit.executor.attack.core import (
-    AttackAdversarialConfig,
-    AttackConverterConfig,
-    AttackScoringConfig,
-)
+from pyrit.executor.attack.core import AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig
 from pyrit.executor.attack.multi_turn.multi_turn_attack_strategy import (
     ConversationSession,
     MultiTurnAttackContext,
@@ -42,12 +32,15 @@
     Message,
     MessagePiece,
     Score,
+    ScoringExpectation,
     SeedPrompt,
 )
 from pyrit.prompt_normalizer import PromptNormalizer
 from pyrit.prompt_target import CapabilityName, TargetRequirements
 from pyrit.score import (
     FloatScaleThresholdScorer,
+    MessageScorable,
+    MessageScoringOptions,
     NumericRubric,
     Scorer,
     SelfAskRefusalScorer,
@@ -681,9 +674,9 @@ async def _check_refusal_async(self, context: CrescendoAttackContext, objective:
             objective=context.objective,
         ):
             scores = await self._refusal_scorer.score_async(
-                message=context.last_response,
-                objective=objective,
-                skip_on_error_result=False,
+                scorable=MessageScorable.from_message(context.last_response),
+                expectation=ScoringExpectation(objective=objective),
+                message_options=MessageScoringOptions(skip_on_error_result=False),
             )
         return scores[0]
 

pyrit/executor/attack/multi_turn/red_teaming.py

@@ -18,11 +18,7 @@
     get_adversarial_chat_messages,
 )
 from pyrit.executor.attack.component.modality_router import _ModalityFeedbackRouter
-from pyrit.executor.attack.core.attack_config import (
-    AttackAdversarialConfig,
-    AttackConverterConfig,
-    AttackScoringConfig,
-)
+from pyrit.executor.attack.core.attack_config import AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig
 from pyrit.executor.attack.multi_turn.multi_turn_attack_strategy import (
     ConversationSession,
     MultiTurnAttackContext,
@@ -38,10 +34,12 @@
     ConversationType,
     Message,
     Score,
+    ScoringExpectation,
 )
 from pyrit.prompt_normalizer import PromptNormalizer
 from pyrit.prompt_target import CapabilityName
 from pyrit.prompt_target.common.target_requirements import TargetRequirements
+from pyrit.score import MessageScorable, MessageScoringOptions
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -527,9 +525,9 @@ async def _score_response_async(self, *, context: MultiTurnAttackContext[Any]) -
         ):
             # score_async handles blocked, filtered, other errors
             scoring_results = await self._objective_scorer.score_async(
-                message=context.last_response,
-                role_filter="assistant",
-                objective=context.objective,
+                scorable=MessageScorable.from_message(context.last_response),
+                expectation=ScoringExpectation(objective=context.objective),
+                message_options=MessageScoringOptions(role_filter="assistant"),
             )
 
         objective_scores = scoring_results