Skip to content

Commit 0ade1ea

Browse files
feat(eval): log agent response, judge scores, dimensions, and feedback
Add visibility into the eval loop: agent response preview (first 300 chars), dimension scores, judge feedback, and raw judge JSON response. Essential for understanding why scenarios pass/fail and what to change. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent cb246d4 commit 0ade1ea

2 files changed

Lines changed: 11 additions & 1 deletion

File tree

‎packages/studyctl/src/studyctl/eval/judge/llm.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,13 @@ def score(self, scenario: Scenario, response: str) -> JudgeResult:
5050
# 2. Build prompt from template
5151
prompt = self._build_prompt(scenario, response)
5252

53-
# 3. Call LLM
53+
# 3. Call LLM judge
5454
try:
5555
raw = self.client.chat(
5656
[{"role": "user", "content": prompt}],
5757
temperature=0.1,
5858
)
59+
logger.info("Judge raw response: %s", raw[:500])
5960
except LLMClientError as exc:
6061
logger.warning("LLM call failed for scenario %s: %s", scenario.id, exc)
6162
return JudgeResult(

‎packages/studyctl/src/studyctl/eval/orchestrator.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ def run_evaluation(
4242
try:
4343
target.setup(scenario)
4444
response = target.run(scenario)
45+
if response:
46+
logger.info(" Agent response (%d chars): %s", len(response), response[:300])
4547
score = judge.score(scenario, response)
4648
except EvalTimeout:
4749
logger.warning("Scenario %s timed out", scenario.id)
@@ -62,6 +64,13 @@ def run_evaluation(
6264
score.weighted_score,
6365
"PASS" if score.passed else "FAIL",
6466
)
67+
# Log dimensions and feedback for visibility
68+
if score.dimensions:
69+
dims = " ".join(f"{k}={v}" for k, v in score.dimensions.items())
70+
logger.info(" Dimensions: %s", dims)
71+
if score.feedback:
72+
for fb in score.feedback:
73+
logger.info(" Feedback: %s", fb)
6574

6675
# Capture persona_hash from target if available (set during setup)
6776
if not persona_hash:

0 commit comments

Comments
 (0)