diff --git a/project/paperbench/paperbench/judge/base.py b/project/paperbench/paperbench/judge/base.py index e6bbc6e4..bf0c8fe4 100644 --- a/project/paperbench/paperbench/judge/base.py +++ b/project/paperbench/paperbench/judge/base.py @@ -163,7 +163,7 @@ async def grade( weight=task.weight, sub_tasks=graded_sub_tasks, score=weighted_score, - valid_score=True, + valid_score=all(child.valid_score for child in graded_sub_tasks), explanation="Aggregated score from sub-tasks.", judge_metadata=None, ) @@ -193,4 +193,4 @@ def get_logger(self, task: TaskNode) -> BoundLogger: run_logger.addHandler(log_file_handler) run_logger.propagate = False - return wrap_logger(run_logger) + return wrap_logger(run_logger) \ No newline at end of file diff --git a/project/paperbench/tests/unit/test_judge_validity.py b/project/paperbench/tests/unit/test_judge_validity.py new file mode 100644 index 00000000..4dc0bf37 --- /dev/null +++ b/project/paperbench/tests/unit/test_judge_validity.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import pytest + +from paperbench.judge.base import Judge +from paperbench.judge.graded_task_node import GradedTaskNode +from paperbench.rubric.tasks import TaskNode + + +class ValidityJudge(Judge): + @property + def judge_type(self) -> str: + return "validity-test" + + async def grade_leaf(self, task: TaskNode) -> GradedTaskNode: + raise NotImplementedError + + async def grade_subtree(self, task: TaskNode) -> GradedTaskNode: + raise NotImplementedError + + +def _leaf(node_id: str) -> TaskNode: + return TaskNode( + id=node_id, + requirements=node_id, + weight=1, + task_category="Code Development", + ) + + +@pytest.mark.asyncio +async def test_aggregate_validity_propagates_child_grading_failure(tmp_path: Path) -> None: + good = _leaf("good") + bad = _leaf("bad") + root = TaskNode( + id="root", + requirements="root", + weight=1, + sub_tasks=[good, bad], + ) + judge = ValidityJudge( + paper_path=tmp_path / "paper.pdf", + rubric=root, + addendum=None, + judge_addendum=None, + submission_dir=tmp_path, + ) + + async def grade_leaf(task: TaskNode) -> GradedTaskNode: + if task.id == "bad": + raise RuntimeError("judge failed") + return GradedTaskNode.from_task( + task, + score=1.0, + valid_score=True, + explanation="graded", + ) + + graded = await judge.grade(root, grade_leaf) + + assert graded.score == 0.5 + assert graded.valid_score is False + assert graded.sub_tasks[0].valid_score is True + assert graded.sub_tasks[1].valid_score is False + + +@pytest.mark.asyncio +async def test_aggregate_validity_remains_true_when_all_children_are_valid(tmp_path: Path) -> None: + children = [_leaf("a"), _leaf("b")] + root = TaskNode( + id="root", + requirements="root", + weight=1, + sub_tasks=children, + ) + judge = ValidityJudge( + paper_path=tmp_path / "paper.pdf", + rubric=root, + addendum=None, + judge_addendum=None, + submission_dir=tmp_path, + ) + + async def grade_leaf(task: TaskNode) -> GradedTaskNode: + return GradedTaskNode.from_task( + task, + score=1.0, + valid_score=True, + explanation="graded", + ) + + graded = await judge.grade(root, grade_leaf) + + assert graded.score == 1.0 + assert graded.valid_score is True