diff --git a/src/imagent_bench/scoring.py b/src/imagent_bench/scoring.py index c5a9203..5c685dc 100644 --- a/src/imagent_bench/scoring.py +++ b/src/imagent_bench/scoring.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from __future__ import annotations import base64 import html @@ -125,8 +125,6 @@ def evaluate_openrouter_vision(image_path: Path, *, prompt: str, config: dict[st parsed = _parse_judge_json(content) dimension_scores = _normalize_dimension_scores(parsed.get("scores", {}), dimensions) overall_score = _weighted_score(dimension_scores, dimensions) - if isinstance(parsed.get("overall_score"), int | float): - overall_score = _clamp_score(float(parsed["overall_score"])) usage = response_payload.get("usage") usage = usage if isinstance(usage, dict) else {} diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 6c03f38..8651f29 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -59,3 +59,56 @@ def read(self) -> bytes: assert result["dimensions"] == {"prompt_alignment": 90.0, "visual_quality": 80.0} assert result["cost_usd"] == 0.002 assert result["judge"]["model"] == "judge/model" + + +def test_openrouter_vision_judge_ignores_self_reported_overall_score( + monkeypatch, tmp_path: Path +) -> None: # noqa: ANN001 + image_path = tmp_path / "image.png" + image_path.write_bytes(b"fake-image") + + class FakeResponse: + def __enter__(self): # noqa: ANN001 + return self + + def __exit__(self, *args): # noqa: ANN002 + return None + + def read(self) -> bytes: + return json.dumps( + { + "model": "judge/model", + "choices": [ + { + "message": { + "content": json.dumps( + { + "scores": { + "prompt_alignment": 40, + "visual_quality": 40, + }, + "overall_score": 99, + "rationale": "inflated self-report", + } + ) + } + } + ], + } + ).encode("utf-8") + + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key") + monkeypatch.setattr("urllib.request.urlopen", lambda *args, **kwargs: FakeResponse()) + + result = evaluate_openrouter_vision( + image_path, + prompt="Create a clean product image.", + config={ + "provider": "openrouter_vision", + "model": "judge/model", + "dimensions": {"prompt_alignment": 0.75, "visual_quality": 0.25}, + }, + ) + + assert result["overall_score"] == 40.0 + assert result["judge"]["raw_response"]["overall_score"] == 99