diff --git a/pyrit/score/float_scale/float_scale_scorer.py b/pyrit/score/float_scale/float_scale_scorer.py index e0501b0c12..273ae2641f 100644 --- a/pyrit/score/float_scale/float_scale_scorer.py +++ b/pyrit/score/float_scale/float_scale_scorer.py @@ -141,11 +141,11 @@ async def _score_value_with_llm_async( prepended_text_message_piece: str | None = None, category: str | UUID | None = None, objective: str | None = None, - score_value_output_key: str = "score_value", - rationale_output_key: str = "rationale", - description_output_key: str = "description", - metadata_output_key: str = "metadata", - category_output_key: str = "category", + score_value_output_key: str | None = None, + rationale_output_key: str | None = None, + description_output_key: str | None = None, + metadata_output_key: str | None = None, + category_output_key: str | None = None, attack_identifier: ComponentIdentifier | None = None, ) -> UnvalidatedScore: score: UnvalidatedScore | None = None diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index 17105defb9..d122e266d5 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -36,11 +36,11 @@ def __init__( min_value: int = 0, max_value: int = 100, validator: ScorerPromptValidator | None = None, - score_value_output_key: str = "score_value", - rationale_output_key: str = "rationale", - description_output_key: str = "description", - metadata_output_key: str = "metadata", - category_output_key: str = "category", + score_value_output_key: str | None = None, + rationale_output_key: str | None = None, + description_output_key: str | None = None, + metadata_output_key: str | None = None, + category_output_key: str | None = None, ) -> None: """ Initialize the SelfAskGeneralFloatScaleScorer. @@ -64,11 +64,16 @@ def __init__( max_value (int): Maximum of the model's native scale. Defaults to 100. validator (ScorerPromptValidator | None): Custom validator. If omitted, a default validator will be used requiring text input and an objective. - score_value_output_key (str): JSON key for the score value. Defaults to "score_value". - rationale_output_key (str): JSON key for the rationale. Defaults to "rationale". - description_output_key (str): JSON key for the description. Defaults to "description". - metadata_output_key (str): JSON key for the metadata. Defaults to "metadata". - category_output_key (str): JSON key for the category. Defaults to "category". + score_value_output_key (str | None): JSON key for the score value. Defaults to + ``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value"). + rationale_output_key (str | None): JSON key for the rationale. Defaults to + ``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale"). + description_output_key (str | None): JSON key for the description. Defaults to + ``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description"). + metadata_output_key (str | None): JSON key for the metadata. Defaults to + ``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata"). + category_output_key (str | None): JSON key for the category. Defaults to + ``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category"). Raises: ValueError: If system_prompt_format_string is not provided or empty. @@ -87,11 +92,21 @@ def __init__( self._score_category = category self._min_value = min_value self._max_value = max_value - self._score_value_output_key = score_value_output_key - self._rationale_output_key = rationale_output_key - self._description_output_key = description_output_key - self._metadata_output_key = metadata_output_key - self._category_output_key = category_output_key + self._score_value_output_key = ( + score_value_output_key if score_value_output_key is not None else self.DEFAULT_SCORE_VALUE_OUTPUT_KEY + ) + self._rationale_output_key = ( + rationale_output_key if rationale_output_key is not None else self.DEFAULT_RATIONALE_OUTPUT_KEY + ) + self._description_output_key = ( + description_output_key if description_output_key is not None else self.DEFAULT_DESCRIPTION_OUTPUT_KEY + ) + self._metadata_output_key = ( + metadata_output_key if metadata_output_key is not None else self.DEFAULT_METADATA_OUTPUT_KEY + ) + self._category_output_key = ( + category_output_key if category_output_key is not None else self.DEFAULT_CATEGORY_OUTPUT_KEY + ) def _build_identifier(self) -> ComponentIdentifier: """ diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index f3cda9923b..48e25829e7 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -72,6 +72,15 @@ class Scorer(Identifiable, abc.ABC): #: validate it. TARGET_REQUIREMENTS: ClassVar[TargetRequirements] = TargetRequirements() + #: Default JSON keys parsed from the LLM scoring response in + #: ``_score_value_with_llm_async``. Subclasses and callers may pass + #: overrides through that method or their own constructors. + DEFAULT_SCORE_VALUE_OUTPUT_KEY: ClassVar[str] = "score_value" + DEFAULT_RATIONALE_OUTPUT_KEY: ClassVar[str] = "rationale" + DEFAULT_DESCRIPTION_OUTPUT_KEY: ClassVar[str] = "description" + DEFAULT_METADATA_OUTPUT_KEY: ClassVar[str] = "metadata" + DEFAULT_CATEGORY_OUTPUT_KEY: ClassVar[str] = "category" + _identifier: ComponentIdentifier | None = None #: When True, blocked responses that contain partial content @@ -671,11 +680,11 @@ async def _score_value_with_llm_async( prepended_text_message_piece: str | None = None, category: Sequence[str] | str | None = None, objective: str | None = None, - score_value_output_key: str = "score_value", - rationale_output_key: str = "rationale", - description_output_key: str = "description", - metadata_output_key: str = "metadata", - category_output_key: str = "category", + score_value_output_key: str | None = None, + rationale_output_key: str | None = None, + description_output_key: str | None = None, + metadata_output_key: str | None = None, + category_output_key: str | None = None, attack_identifier: ComponentIdentifier | None = None, ) -> UnvalidatedScore: """ @@ -700,16 +709,16 @@ async def _score_value_with_llm_async( the JSON response if not provided. Defaults to None. objective (str | None): A description of the objective that is associated with the score, used for contextualizing the result. Defaults to None. - score_value_output_key (str): The key in the JSON response that contains the score value. - Defaults to "score_value". - rationale_output_key (str): The key in the JSON response that contains the rationale. - Defaults to "rationale". - description_output_key (str): The key in the JSON response that contains the description. - Defaults to "description". - metadata_output_key (str): The key in the JSON response that contains the metadata. - Defaults to "metadata". - category_output_key (str): The key in the JSON response that contains the category. - Defaults to "category". + score_value_output_key (str | None): The key in the JSON response that contains the score value. + Defaults to ``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value"). + rationale_output_key (str | None): The key in the JSON response that contains the rationale. + Defaults to ``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale"). + description_output_key (str | None): The key in the JSON response that contains the description. + Defaults to ``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description"). + metadata_output_key (str | None): The key in the JSON response that contains the metadata. + Defaults to ``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata"). + category_output_key (str | None): The key in the JSON response that contains the category. + Defaults to ``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category"). attack_identifier (ComponentIdentifier | None): The attack identifier. Defaults to None. @@ -722,6 +731,17 @@ async def _score_value_with_llm_async( InvalidJsonException: If the response is not valid JSON. Exception: For other unexpected errors during scoring. """ + if score_value_output_key is None: + score_value_output_key = self.DEFAULT_SCORE_VALUE_OUTPUT_KEY + if rationale_output_key is None: + rationale_output_key = self.DEFAULT_RATIONALE_OUTPUT_KEY + if description_output_key is None: + description_output_key = self.DEFAULT_DESCRIPTION_OUTPUT_KEY + if metadata_output_key is None: + metadata_output_key = self.DEFAULT_METADATA_OUTPUT_KEY + if category_output_key is None: + category_output_key = self.DEFAULT_CATEGORY_OUTPUT_KEY + conversation_id = str(uuid.uuid4()) prompt_target.set_system_prompt( diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index 71acd45a56..9a7b7362d1 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -39,11 +39,11 @@ def __init__( category: str | None = None, validator: ScorerPromptValidator | None = None, score_aggregator: TrueFalseAggregatorFunc = TrueFalseScoreAggregator.OR, - score_value_output_key: str = "score_value", - rationale_output_key: str = "rationale", - description_output_key: str = "description", - metadata_output_key: str = "metadata", - category_output_key: str = "category", + score_value_output_key: str | None = None, + rationale_output_key: str | None = None, + description_output_key: str | None = None, + metadata_output_key: str | None = None, + category_output_key: str | None = None, ) -> None: """ Initialize the SelfAskGeneralTrueFalseScorer. @@ -67,11 +67,16 @@ def __init__( validator will be used requiring text input and an objective. score_aggregator (TrueFalseAggregatorFunc): Aggregator for combining scores. Defaults to TrueFalseScoreAggregator.OR. - score_value_output_key (str): JSON key for the score value. Defaults to "score_value". - rationale_output_key (str): JSON key for the rationale. Defaults to "rationale". - description_output_key (str): JSON key for the description. Defaults to "description". - metadata_output_key (str): JSON key for the metadata. Defaults to "metadata". - category_output_key (str): JSON key for the category. Defaults to "category". + score_value_output_key (str | None): JSON key for the score value. Defaults to + ``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value"). + rationale_output_key (str | None): JSON key for the rationale. Defaults to + ``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale"). + description_output_key (str | None): JSON key for the description. Defaults to + ``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description"). + metadata_output_key (str | None): JSON key for the metadata. Defaults to + ``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata"). + category_output_key (str | None): JSON key for the category. Defaults to + ``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category"). Raises: ValueError: If system_prompt_format_string is not provided or empty. @@ -88,11 +93,21 @@ def __init__( self._prompt_format_string = prompt_format_string self._score_category = category - self._score_value_output_key = score_value_output_key - self._rationale_output_key = rationale_output_key - self._description_output_key = description_output_key - self._metadata_output_key = metadata_output_key - self._category_output_key = category_output_key + self._score_value_output_key = ( + score_value_output_key if score_value_output_key is not None else self.DEFAULT_SCORE_VALUE_OUTPUT_KEY + ) + self._rationale_output_key = ( + rationale_output_key if rationale_output_key is not None else self.DEFAULT_RATIONALE_OUTPUT_KEY + ) + self._description_output_key = ( + description_output_key if description_output_key is not None else self.DEFAULT_DESCRIPTION_OUTPUT_KEY + ) + self._metadata_output_key = ( + metadata_output_key if metadata_output_key is not None else self.DEFAULT_METADATA_OUTPUT_KEY + ) + self._category_output_key = ( + category_output_key if category_output_key is not None else self.DEFAULT_CATEGORY_OUTPUT_KEY + ) def _build_identifier(self) -> ComponentIdentifier: """