Skip to content

REFACTOR: De-dup scorer output_key defaults onto Scorer base as ClassVars #1970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Roman Lutz (romanlutz) wants to merge 1 commit into from
Closed

REFACTOR: De-dup scorer output_key defaults onto Scorer base as ClassVars #1970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions pyrit/score/float_scale/float_scale_scorer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,11 +141,11 @@ async def _score_value_with_llm_async(
prepended_text_message_piece: str | None = None,
category: str | UUID | None = None,
objective: str | None = None,
score_value_output_key: str = "score_value",
rationale_output_key: str = "rationale",
description_output_key: str = "description",
metadata_output_key: str = "metadata",
category_output_key: str = "category",
score_value_output_key: str | None = None,
rationale_output_key: str | None = None,
description_output_key: str | None = None,
metadata_output_key: str | None = None,
category_output_key: str | None = None,
attack_identifier: ComponentIdentifier | None = None,
) -> UnvalidatedScore:
score: UnvalidatedScore | None = None
Expand Down
45 changes: 30 additions & 15 deletions pyrit/score/float_scale/self_ask_general_float_scale_scorer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ def __init__(
min_value: int = 0,
max_value: int = 100,
validator: ScorerPromptValidator | None = None,
score_value_output_key: str = "score_value",
rationale_output_key: str = "rationale",
description_output_key: str = "description",
metadata_output_key: str = "metadata",
category_output_key: str = "category",
score_value_output_key: str | None = None,
rationale_output_key: str | None = None,
description_output_key: str | None = None,
metadata_output_key: str | None = None,
category_output_key: str | None = None,
) -> None:
"""
Initialize the SelfAskGeneralFloatScaleScorer.
Expand All @@ -64,11 +64,16 @@ def __init__(
max_value (int): Maximum of the model's native scale. Defaults to 100.
validator (ScorerPromptValidator | None): Custom validator. If omitted, a default
validator will be used requiring text input and an objective.
score_value_output_key (str): JSON key for the score value. Defaults to "score_value".
rationale_output_key (str): JSON key for the rationale. Defaults to "rationale".
description_output_key (str): JSON key for the description. Defaults to "description".
metadata_output_key (str): JSON key for the metadata. Defaults to "metadata".
category_output_key (str): JSON key for the category. Defaults to "category".
score_value_output_key (str | None): JSON key for the score value. Defaults to
``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value").
rationale_output_key (str | None): JSON key for the rationale. Defaults to
``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale").
description_output_key (str | None): JSON key for the description. Defaults to
``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description").
metadata_output_key (str | None): JSON key for the metadata. Defaults to
``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata").
category_output_key (str | None): JSON key for the category. Defaults to
``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category").

Raises:
ValueError: If system_prompt_format_string is not provided or empty.
Expand All @@ -87,11 +92,21 @@ def __init__(
self._score_category = category
self._min_value = min_value
self._max_value = max_value
self._score_value_output_key = score_value_output_key
self._rationale_output_key = rationale_output_key
self._description_output_key = description_output_key
self._metadata_output_key = metadata_output_key
self._category_output_key = category_output_key
self._score_value_output_key = (
score_value_output_key if score_value_output_key is not None else self.DEFAULT_SCORE_VALUE_OUTPUT_KEY
)
self._rationale_output_key = (
rationale_output_key if rationale_output_key is not None else self.DEFAULT_RATIONALE_OUTPUT_KEY
)
self._description_output_key = (
description_output_key if description_output_key is not None else self.DEFAULT_DESCRIPTION_OUTPUT_KEY
)
self._metadata_output_key = (
metadata_output_key if metadata_output_key is not None else self.DEFAULT_METADATA_OUTPUT_KEY
)
self._category_output_key = (
category_output_key if category_output_key is not None else self.DEFAULT_CATEGORY_OUTPUT_KEY
)

def _build_identifier(self) -> ComponentIdentifier:
"""
Expand Down
50 changes: 35 additions & 15 deletions pyrit/score/scorer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ class Scorer(Identifiable, abc.ABC):
#: validate it.
TARGET_REQUIREMENTS: ClassVar[TargetRequirements] = TargetRequirements()

#: Default JSON keys parsed from the LLM scoring response in
#: ``_score_value_with_llm_async``. Subclasses and callers may pass
#: overrides through that method or their own constructors.
DEFAULT_SCORE_VALUE_OUTPUT_KEY: ClassVar[str] = "score_value"
DEFAULT_RATIONALE_OUTPUT_KEY: ClassVar[str] = "rationale"
DEFAULT_DESCRIPTION_OUTPUT_KEY: ClassVar[str] = "description"
DEFAULT_METADATA_OUTPUT_KEY: ClassVar[str] = "metadata"
DEFAULT_CATEGORY_OUTPUT_KEY: ClassVar[str] = "category"

_identifier: ComponentIdentifier | None = None

#: When True, blocked responses that contain partial content
Expand Down Expand Up @@ -671,11 +680,11 @@ async def _score_value_with_llm_async(
prepended_text_message_piece: str | None = None,
category: Sequence[str] | str | None = None,
objective: str | None = None,
score_value_output_key: str = "score_value",
rationale_output_key: str = "rationale",
description_output_key: str = "description",
metadata_output_key: str = "metadata",
category_output_key: str = "category",
score_value_output_key: str | None = None,
rationale_output_key: str | None = None,
description_output_key: str | None = None,
metadata_output_key: str | None = None,
category_output_key: str | None = None,
attack_identifier: ComponentIdentifier | None = None,
) -> UnvalidatedScore:
"""
Expand All @@ -700,16 +709,16 @@ async def _score_value_with_llm_async(
the JSON response if not provided. Defaults to None.
objective (str | None): A description of the objective that is associated with the score,
used for contextualizing the result. Defaults to None.
score_value_output_key (str): The key in the JSON response that contains the score value.
Defaults to "score_value".
rationale_output_key (str): The key in the JSON response that contains the rationale.
Defaults to "rationale".
description_output_key (str): The key in the JSON response that contains the description.
Defaults to "description".
metadata_output_key (str): The key in the JSON response that contains the metadata.
Defaults to "metadata".
category_output_key (str): The key in the JSON response that contains the category.
Defaults to "category".
score_value_output_key (str | None): The key in the JSON response that contains the score value.
Defaults to ``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value").
rationale_output_key (str | None): The key in the JSON response that contains the rationale.
Defaults to ``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale").
description_output_key (str | None): The key in the JSON response that contains the description.
Defaults to ``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description").
metadata_output_key (str | None): The key in the JSON response that contains the metadata.
Defaults to ``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata").
category_output_key (str | None): The key in the JSON response that contains the category.
Defaults to ``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category").
attack_identifier (ComponentIdentifier | None): The attack identifier.
Defaults to None.

Expand All @@ -722,6 +731,17 @@ async def _score_value_with_llm_async(
InvalidJsonException: If the response is not valid JSON.
Exception: For other unexpected errors during scoring.
"""
if score_value_output_key is None:
score_value_output_key = self.DEFAULT_SCORE_VALUE_OUTPUT_KEY
if rationale_output_key is None:
rationale_output_key = self.DEFAULT_RATIONALE_OUTPUT_KEY
if description_output_key is None:
description_output_key = self.DEFAULT_DESCRIPTION_OUTPUT_KEY
if metadata_output_key is None:
metadata_output_key = self.DEFAULT_METADATA_OUTPUT_KEY
if category_output_key is None:
category_output_key = self.DEFAULT_CATEGORY_OUTPUT_KEY

conversation_id = str(uuid.uuid4())

prompt_target.set_system_prompt(
Expand Down
45 changes: 30 additions & 15 deletions pyrit/score/true_false/self_ask_general_true_false_scorer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ def __init__(
category: str | None = None,
validator: ScorerPromptValidator | None = None,
score_aggregator: TrueFalseAggregatorFunc = TrueFalseScoreAggregator.OR,
score_value_output_key: str = "score_value",
rationale_output_key: str = "rationale",
description_output_key: str = "description",
metadata_output_key: str = "metadata",
category_output_key: str = "category",
score_value_output_key: str | None = None,
rationale_output_key: str | None = None,
description_output_key: str | None = None,
metadata_output_key: str | None = None,
category_output_key: str | None = None,
) -> None:
"""
Initialize the SelfAskGeneralTrueFalseScorer.
Expand All @@ -67,11 +67,16 @@ def __init__(
validator will be used requiring text input and an objective.
score_aggregator (TrueFalseAggregatorFunc): Aggregator for combining scores. Defaults to
TrueFalseScoreAggregator.OR.
score_value_output_key (str): JSON key for the score value. Defaults to "score_value".
rationale_output_key (str): JSON key for the rationale. Defaults to "rationale".
description_output_key (str): JSON key for the description. Defaults to "description".
metadata_output_key (str): JSON key for the metadata. Defaults to "metadata".
category_output_key (str): JSON key for the category. Defaults to "category".
score_value_output_key (str | None): JSON key for the score value. Defaults to
``DEFAULT_SCORE_VALUE_OUTPUT_KEY`` ("score_value").
rationale_output_key (str | None): JSON key for the rationale. Defaults to
``DEFAULT_RATIONALE_OUTPUT_KEY`` ("rationale").
description_output_key (str | None): JSON key for the description. Defaults to
``DEFAULT_DESCRIPTION_OUTPUT_KEY`` ("description").
metadata_output_key (str | None): JSON key for the metadata. Defaults to
``DEFAULT_METADATA_OUTPUT_KEY`` ("metadata").
category_output_key (str | None): JSON key for the category. Defaults to
``DEFAULT_CATEGORY_OUTPUT_KEY`` ("category").

Raises:
ValueError: If system_prompt_format_string is not provided or empty.
Expand All @@ -88,11 +93,21 @@ def __init__(
self._prompt_format_string = prompt_format_string

self._score_category = category
self._score_value_output_key = score_value_output_key
self._rationale_output_key = rationale_output_key
self._description_output_key = description_output_key
self._metadata_output_key = metadata_output_key
self._category_output_key = category_output_key
self._score_value_output_key = (
score_value_output_key if score_value_output_key is not None else self.DEFAULT_SCORE_VALUE_OUTPUT_KEY
)
self._rationale_output_key = (
rationale_output_key if rationale_output_key is not None else self.DEFAULT_RATIONALE_OUTPUT_KEY
)
self._description_output_key = (
description_output_key if description_output_key is not None else self.DEFAULT_DESCRIPTION_OUTPUT_KEY
)
self._metadata_output_key = (
metadata_output_key if metadata_output_key is not None else self.DEFAULT_METADATA_OUTPUT_KEY
)
self._category_output_key = (
category_output_key if category_output_key is not None else self.DEFAULT_CATEGORY_OUTPUT_KEY
)

def _build_identifier(self) -> ComponentIdentifier:
"""
Expand Down
Loading