From e7dc9a71a73d063b7271570d3da3a4ed6b858ef6 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 5 Jun 2026 09:52:24 +0800 Subject: [PATCH 1/6] feat(agent-eval): add LLMAgent* evaluators for agent trace monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 8 LLM evaluators + 3 rule evaluators for AI agent trace quality assessment, aligned with DeepEval 3-layer taxonomy (Execution/Action/Reasoning) + Recovery. LLM evaluators (dingo/model/llm/agent_eval/): - LLMAgentTaskCompletion, LLMAgentStepEfficiency (Execution) - LLMAgentToolCorrectness, LLMAgentArgumentCorrectness (Action) - LLMAgentPlanQuality, LLMAgentPlanAdherence (Reasoning) - LLMAgentErrorRecovery (Recovery), LLMAgentTraceConclusion (synthesis) Rule evaluators: RuleAgentTraceLoopDetection, RuleAgentTraceTokenBudget, RuleAgentTraceLatencyAnomaly Shared BaseLLMAgentEval with 0-10→0.0-1.0 normalization, CJK language detection, and configurable threshold. --- .pre-commit-config.yaml | 46 ++-- dingo/model/llm/agent_eval/__init__.py | 10 + .../llm/agent_eval/base_llm_agent_eval.py | 112 +++++++++ .../llm_agent_argument_correctness.py | 82 +++++++ .../agent_eval/llm_agent_error_recovery.py | 150 ++++++++++++ .../agent_eval/llm_agent_plan_adherence.py | 85 +++++++ .../llm/agent_eval/llm_agent_plan_quality.py | 160 ++++++++++++ .../agent_eval/llm_agent_step_efficiency.py | 75 ++++++ .../agent_eval/llm_agent_task_completion.py | 72 ++++++ .../agent_eval/llm_agent_tool_correctness.py | 82 +++++++ .../agent_eval/llm_agent_trace_conclusion.py | 119 +++++++++ dingo/model/rule/rule_agent.py | 230 ++++++++++++++++++ 12 files changed, 1200 insertions(+), 23 deletions(-) create mode 100644 dingo/model/llm/agent_eval/__init__.py create mode 100644 dingo/model/llm/agent_eval/base_llm_agent_eval.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_argument_correctness.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_error_recovery.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_plan_adherence.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_plan_quality.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_step_efficiency.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_task_completion.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_tool_correctness.py create mode 100644 dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py create mode 100644 dingo/model/rule/rule_agent.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c1fd47c1..6b50e489 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,23 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: trailing-whitespace - exclude: '^README.*\.md$' - - id: end-of-file-fixer - exclude: 'docs/metrics\.md' - - id: check-yaml - - id: check-added-large-files -- repo: https://github.com/PyCQA/isort - rev: 6.0.1 - hooks: - - id: isort - args: [ "-l", "200", "-m", "0", "-p", "dingo" ] -- repo: https://github.com/PyCQA/flake8 - rev: 7.2.0 - hooks: - - id: flake8 - args: [ "--max-line-length=2200", "--ignore=E121,E131,E125,W503,W504,W604,E203,E231,E702,E128,F541,F401,E266" ] - exclude: 'app/' +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + exclude: '^README.*\.md$' + - id: end-of-file-fixer + exclude: 'docs/metrics\.md' + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/PyCQA/isort + rev: 6.0.1 + hooks: + - id: isort + args: [ "-l", "200", "-m", "0", "-p", "dingo" ] +- repo: https://github.com/PyCQA/flake8 + rev: 7.2.0 + hooks: + - id: flake8 + args: [ "--max-line-length=2200", "--ignore=E121,E131,E125,W503,W504,W604,E203,E231,E702,E128,F541,F401,E266" ] + exclude: 'app/' diff --git a/dingo/model/llm/agent_eval/__init__.py b/dingo/model/llm/agent_eval/__init__.py new file mode 100644 index 00000000..a6e8dbbb --- /dev/null +++ b/dingo/model/llm/agent_eval/__init__.py @@ -0,0 +1,10 @@ +""" +Agent Trace Evaluation metrics (LLMAgent* series). + +These evaluators assess agent trace quality using LLM-as-Judge methodology. +They are distinct from the agent/ directory which contains agent-framework-based +evaluators (AgentFactCheck, AgentHallucination) that USE agent frameworks to DO evaluation. + +Evaluators in this package EVALUATE agent traces — task completion, plan quality, +tool correctness, error recovery, etc. +""" diff --git a/dingo/model/llm/agent_eval/base_llm_agent_eval.py b/dingo/model/llm/agent_eval/base_llm_agent_eval.py new file mode 100644 index 00000000..5669cf1e --- /dev/null +++ b/dingo/model/llm/agent_eval/base_llm_agent_eval.py @@ -0,0 +1,112 @@ +""" +Base class for all Agent evaluation metrics. + +Provides standardized: +- 0~10 score → 0.0~1.0 normalization +- Configurable threshold via dynamic_config.model_extra +- Unified JSON response parsing with {"score": 0-10, "reason": "...", ...} +- Error fallback handling +- `eval_layer` and `input_data_type` declarations for orchestrator integration + +Subclasses only need to define: +- `prompt`: the evaluation prompt template +- `build_messages()`: how to format input data into LLM messages +- Optionally override `process_response()` for custom parsing +""" + +import json +from typing import Optional + +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.llm.base_openai import BaseOpenAI +from dingo.utils import log +from dingo.utils.exception import ConvertJsonError + + +class BaseLLMAgentEval(BaseOpenAI): + """Shared base class for all Agent evaluation metrics.""" + + eval_layer: str = "" + input_data_type: str = "trace_summary" + default_threshold: float = 0.6 + + @classmethod + def _detect_language_hint(cls, text: str) -> str: + """Detect if text contains CJK characters and return a language instruction.""" + if not text: + return "" + import re + cjk_count = len(re.findall(r'[一-鿿㐀-䶿]', text[:500])) + if cjk_count > 5: + return '\n\n注意：请用中文回答 "reason" 字段。' + return "" + + @classmethod + def _get_threshold(cls) -> float: + if cls.dynamic_config and cls.dynamic_config.model_extra: + return float(cls.dynamic_config.model_extra.get( + "threshold", cls.default_threshold + )) + return cls.default_threshold + + @classmethod + def _strip_json_fences(cls, response: str) -> str: + response = response.strip() + if response.startswith("```json"): + response = response[7:] + if response.startswith("```"): + response = response[3:] + if response.endswith("```"): + response = response[:-3] + return response.strip() + + @classmethod + def _parse_json_response(cls, response: str) -> dict: + cleaned = cls._strip_json_fences(response) + try: + return json.loads(cleaned) + except json.JSONDecodeError: + raise ConvertJsonError( + f"Failed to parse agent eval JSON: {cleaned[:500]}" + ) + + @classmethod + def process_response(cls, response: str) -> EvalDetail: + """Standardized response processing for agent evaluators. + + Expected LLM output: {"score": 0-10, "reason": "...", ...extra fields...} + Score is normalized to 0.0~1.0 and compared against threshold. + Extra fields are preserved in EvalDetail.reason as JSON. + """ + log.info(response) + data = cls._parse_json_response(response) + + raw_score = data.get("score", data.get("overall_score", 0)) + try: + raw_score = float(raw_score) + except (TypeError, ValueError): + raw_score = 0.0 + + normalized_score = max(0.0, min(1.0, raw_score / 10.0)) + threshold = cls._get_threshold() + + reason_text = data.get("reason", "") + details = {k: v for k, v in data.items() if k not in ("score", "reason")} + + result = EvalDetail(metric=cls.__name__) + result.score = normalized_score + + if normalized_score >= threshold: + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + else: + result.status = True + result.label = [f"AGENT_QUALITY.{cls.__name__}"] + + reason_parts = [reason_text] if reason_text else [] + if details: + reason_parts.append(json.dumps(details, ensure_ascii=False, default=str)) + result.reason = reason_parts if reason_parts else None + + return result diff --git a/dingo/model/llm/agent_eval/llm_agent_argument_correctness.py b/dingo/model/llm/agent_eval/llm_agent_argument_correctness.py new file mode 100644 index 00000000..3ef3b9ca --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_argument_correctness.py @@ -0,0 +1,82 @@ +""" +LLMAgentArgumentCorrectness: Evaluates whether the agent passed correct arguments to each tool call. + +Performs referenceless LLM-judge evaluation — no ground-truth arguments are required. +The judge assesses argument quality based on the task objective and the expected +semantics of each tool. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval + + +@Model.llm_register("LLMAgentArgumentCorrectness") +class LLMAgentArgumentCorrectness(BaseLLMAgentEval): + """ + Evaluates the correctness of tool arguments in an agent's execution. + + Input: + prompt - The task objective or user request + content - JSON-formatted sequence of tool calls with their arguments + + Performs referenceless evaluation: the LLM judge determines whether + each tool received correct, well-formed, and contextually appropriate + arguments, without requiring ground-truth argument values. + """ + + eval_layer = "action" + input_data_type = "tool_calls" + default_threshold = 0.6 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + prompt = """You are an expert evaluator assessing whether an AI agent passed correct arguments to its tool calls. + +For each tool call in the sequence, evaluate the arguments: +- Are the argument values correct and appropriate for the task context? +- Are required arguments present and non-null? +- Are argument types and formats valid? +- Do the arguments make semantic sense given what the tool does? + +Count: +- **correct_args**: Tool calls where arguments were fully correct +- **total_calls**: Total number of tool calls evaluated + +List specific argument issues found (wrong value, missing required argument, type mismatch, semantically incorrect argument, etc.). + +A score of 10 means all tool calls had perfectly correct arguments. +A score of 0 means all tool calls had wrong or missing arguments. + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "correct_args": , + "total_calls": , + "issues": ["", ...], + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for argument correctness evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Tool Call Sequence with Arguments +{input_data.content} + +Evaluate the tool arguments and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] diff --git a/dingo/model/llm/agent_eval/llm_agent_error_recovery.py b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py new file mode 100644 index 00000000..b7f260ab --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py @@ -0,0 +1,150 @@ +""" +LLMAgentErrorRecovery: Evaluates the agent's ability to recover from errors encountered during execution. + +If no error events are present in the input, returns score=1.0 (pass) immediately, +since perfect execution with no errors requires no recovery. +""" + +import time +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval +from dingo.utils import log +from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens + +try: + from pydantic import ValidationError +except ImportError: + ValidationError = Exception + + +@Model.llm_register("LLMAgentErrorRecovery") +class LLMAgentErrorRecovery(BaseLLMAgentEval): + """ + Evaluates the error recovery capability of an agent. + + Input: + prompt - The task objective or user request + content - The error events or failure log from the agent execution + + If no errors are found in the content, the evaluator short-circuits + and returns score=1.0 (pass) without calling the LLM. + """ + + eval_layer = "recovery" + input_data_type = "error_events" + default_threshold = 0.5 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + _NO_ERROR_INDICATORS = [ + "no error", "no errors", "no failures", "no exception", + "0 errors", "zero errors", "none", "n/a", "[]", "{}", + ] + + prompt = """You are an expert evaluator assessing how well an AI agent recovered from errors during task execution. + +For each error event, evaluate: +- Did the agent detect the error? +- Did the agent attempt recovery? +- Was the recovery successful? +- Was the recovery strategy appropriate? + +Count: +- **errors_encountered**: Total number of distinct errors or failures +- **recovered_count**: Number of errors from which the agent successfully recovered + +Assess overall **recovery_quality** from 0 to 10: +- 10: Agent recovered from all errors with optimal strategies +- 7-9: Agent recovered from most errors with reasonable strategies +- 4-6: Agent recovered from some errors but used suboptimal approaches +- 1-3: Agent attempted recovery but largely failed +- 0: Agent did not attempt recovery or made errors worse + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "errors_encountered": , + "recovered_count": , + "recovery_quality": , + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def _has_error_events(cls, content: str) -> bool: + """Check if the content contains actual error events.""" + if not content or not content.strip(): + return False + stripped = content.strip().lower() + for indicator in cls._NO_ERROR_INDICATORS: + if stripped == indicator: + return False + return True + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for error recovery evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Error Events / Failure Log +{input_data.content} + +Evaluate the agent's error recovery and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Override eval() to handle the no-error special case.""" + content = getattr(input_data, "content", "") or "" + + if not cls._has_error_events(content): + log.info(f"{cls.__name__}: No error events detected, returning pass") + result = EvalDetail(metric=cls.__name__) + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + result.score = 1.0 + result.reason = ["No error events found in execution trace; recovery evaluation skipped."] + return result + + if cls.client is None: + cls.create_client() + + messages = cls.build_messages(input_data) + + attempts = 0 + except_msg = "" + except_name = Exception.__class__.__name__ + while attempts < 3: + try: + response = cls.send_messages(messages) + res: EvalDetail = cls.process_response(response) + return res + except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: + except_msg = str(e) + except_name = e.__class__.__name__ + break + except Exception as e: + attempts += 1 + time.sleep(1) + except_msg = str(e) + except_name = e.__class__.__name__ + + res = EvalDetail(metric=cls.__name__) + res.status = True + res.label = [f"QUALITY_BAD.{except_name}"] + res.reason = [except_msg] + return res diff --git a/dingo/model/llm/agent_eval/llm_agent_plan_adherence.py b/dingo/model/llm/agent_eval/llm_agent_plan_adherence.py new file mode 100644 index 00000000..0755b47b --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_plan_adherence.py @@ -0,0 +1,85 @@ +""" +LLMAgentPlanAdherence: Evaluates how closely the agent followed its stated plan during execution. + +Compares the original plan (prompt) against the actual execution steps (content), +with the task goal available as context. Justified deviations are scored more +leniently than unjustified ones. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval + + +@Model.llm_register("LLMAgentPlanAdherence") +class LLMAgentPlanAdherence(BaseLLMAgentEval): + """ + Evaluates how closely the agent adhered to its original plan. + + Input: + prompt - The agent's original plan (steps or strategy) + content - The actual execution steps taken by the agent + context - The overarching task goal or user objective + + Deviations are classified as justified (e.g., adapting to unexpected + obstacles) or unjustified (e.g., skipping steps without reason). + """ + + eval_layer = "reasoning" + input_data_type = "plan_vs_execution" + default_threshold = 0.5 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT, RequiredField.CONTEXT] + + prompt = """You are an expert evaluator assessing how closely an AI agent followed its original plan during execution. + +Compare the original plan against the actual execution steps and classify any deviations: +- **Justified deviations**: The agent deviated from the plan for a valid reason (e.g., encountered an obstacle, discovered new information, adapted to dynamic conditions). +- **Unjustified deviations**: The agent deviated from the plan without apparent reason (e.g., skipped steps, added unplanned steps unrelated to the goal). + +Count: +- **followed_steps**: Number of planned steps that were executed as intended +- **total_planned**: Total number of steps in the original plan +- **justified_deviations**: Deviations with valid justification +- **unjustified_deviations**: Deviations without justification + +A score of 10 means perfect adherence (or all deviations were justified). +A score of 0 means the agent completely ignored its plan without justification. + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "followed_steps": , + "total_planned": , + "justified_deviations": , + "unjustified_deviations": , + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for plan adherence evaluation.""" + task_goal = getattr(input_data, "context", "") or "" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Goal +{task_goal} + +## Original Plan +{input_data.prompt} + +## Actual Execution Steps +{input_data.content} + +Evaluate how closely the agent followed its plan and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] diff --git a/dingo/model/llm/agent_eval/llm_agent_plan_quality.py b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py new file mode 100644 index 00000000..be4f33da --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py @@ -0,0 +1,160 @@ +""" +LLMAgentPlanQuality: Evaluates the quality of an agent's reasoning plan. + +Assesses coherence, completeness, and feasibility of the agent's plan. +If no planning content is found in the trace, defaults to passing (score=1.0). +""" + +import time +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval +from dingo.utils import log +from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens + +try: + from pydantic import ValidationError +except ImportError: + ValidationError = Exception + + +@Model.llm_register("LLMAgentPlanQuality") +class LLMAgentPlanQuality(BaseLLMAgentEval): + """ + Evaluates the quality of an agent's reasoning plan. + + Input: + prompt - The task objective or user request + content - The agent trace or plan description + + If no planning content is detected in the trace, the evaluator + returns score=1.0 (pass) because absence of planning may be + acceptable for simple tasks. + """ + + eval_layer = "reasoning" + input_data_type = "trace_summary" + default_threshold = 0.6 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + _NO_PLAN_KEYWORDS = [ + "no plan", "no planning", "no explicit plan", + "did not plan", "skipped planning", "planning not found", + ] + + prompt = """You are an expert evaluator assessing the quality of an AI agent's reasoning plan. + +First, determine whether the trace contains any planning content (explicit steps, strategy, or reasoning about how to approach the task). If there is no planning content at all, set score to -1 as a sentinel value. + +If planning content exists, evaluate it on three dimensions: +1. **Coherence** (1-5): Is the plan logically structured and internally consistent? +2. **Completeness** (1-5): Does the plan cover all necessary steps to achieve the goal? +3. **Feasibility** (1-5): Are the planned steps realistic and achievable? + +Compute an overall score from 0 to 10. + +IMPORTANT: The "reason" field MUST be in the same language as the Task Objective. If the task objective is in Chinese, respond in Chinese. If in English, respond in English. + +Return your evaluation as a JSON object with this exact schema: +{ + "coherence": , + "completeness": , + "feasibility": , + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for plan quality evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Agent Trace / Plan +{input_data.content} + +Evaluate the plan quality and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Override eval() to handle the no-planning special case.""" + if cls.client is None: + cls.create_client() + + messages = cls.build_messages(input_data) + + attempts = 0 + except_msg = "" + except_name = Exception.__class__.__name__ + while attempts < 3: + try: + response = cls.send_messages(messages) + + data = cls._parse_json_response(response) + raw_score = data.get("score", 0) + + try: + raw_score = float(raw_score) + except (TypeError, ValueError): + raw_score = 0.0 + + result = EvalDetail(metric=cls.__name__) + + if raw_score < 0: + # Sentinel value: no planning content found, treat as pass + log.info(f"{cls.__name__}: No planning content found in trace, defaulting to pass") + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + result.score = 1.0 + result.reason = [data.get("reason", "No planning content found; evaluation skipped.")] + return result + + normalized_score = max(0.0, min(1.0, raw_score / 10.0)) + threshold = cls._get_threshold() + reason_text = data.get("reason", "") + details = {k: v for k, v in data.items() if k not in ("score", "reason")} + + import json + result.score = normalized_score + if normalized_score >= threshold: + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + else: + result.status = True + result.label = [f"AGENT_QUALITY.{cls.__name__}"] + + reason_parts = [reason_text] if reason_text else [] + if details: + reason_parts.append(json.dumps(details, ensure_ascii=False, default=str)) + result.reason = reason_parts if reason_parts else None + + return result + + except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: + except_msg = str(e) + except_name = e.__class__.__name__ + break + except Exception as e: + attempts += 1 + time.sleep(1) + except_msg = str(e) + except_name = e.__class__.__name__ + + res = EvalDetail(metric=cls.__name__) + res.status = True + res.label = [f"QUALITY_BAD.{except_name}"] + res.reason = [except_msg] + return res diff --git a/dingo/model/llm/agent_eval/llm_agent_step_efficiency.py b/dingo/model/llm/agent_eval/llm_agent_step_efficiency.py new file mode 100644 index 00000000..cb3ab959 --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_step_efficiency.py @@ -0,0 +1,75 @@ +""" +LLMAgentStepEfficiency: Evaluates whether the agent executed its task with minimal redundant steps. + +Detects wasted steps, execution loops, and unnecessary operations in the agent trace, +scoring higher for lean and purposeful execution. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval + + +@Model.llm_register("LLMAgentStepEfficiency") +class LLMAgentStepEfficiency(BaseLLMAgentEval): + """ + Evaluates the step efficiency of an agent's execution trace. + + Input: + prompt - The task objective or user request + content - The agent execution trace or step-by-step summary + + Output score reflects how efficiently the agent reached its goal, + penalizing redundant steps, loops, and unnecessary operations. + """ + + eval_layer = "execution" + input_data_type = "trace_summary" + default_threshold = 0.5 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + prompt = """You are an expert evaluator assessing the step efficiency of an AI agent's execution. + +Analyze the agent's execution trace and identify: +- **Total steps**: Count all steps/actions taken by the agent +- **Necessary steps**: Steps that directly contribute to completing the task +- **Wasted steps**: Redundant, repeated, or unnecessary steps +- **Loops detected**: Whether the agent got stuck in a repetitive pattern + +A score of 10 means perfectly efficient execution with no wasted steps. +A score of 0 means the agent was completely stuck in loops or took entirely unnecessary actions. + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "total_steps": , + "necessary_steps": , + "wasted_steps": , + "loops_detected": , + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for step efficiency evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Agent Execution Trace +{input_data.content} + +Analyze the execution efficiency and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] diff --git a/dingo/model/llm/agent_eval/llm_agent_task_completion.py b/dingo/model/llm/agent_eval/llm_agent_task_completion.py new file mode 100644 index 00000000..ec21019b --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_task_completion.py @@ -0,0 +1,72 @@ +""" +LLMAgentTaskCompletion: Evaluates whether an Agent successfully completed its assigned task. + +Compares the task objective (prompt) against the execution result summary (content) +and scores on goal achievement, accuracy, and completeness. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval + + +@Model.llm_register("LLMAgentTaskCompletion") +class LLMAgentTaskCompletion(BaseLLMAgentEval): + """ + Evaluates whether the agent completed its assigned task. + + Input: + prompt - The task objective or user request + content - The agent execution result summary + + Output score reflects the degree to which the agent achieved the goal, + produced accurate results, and covered all required aspects. + """ + + eval_layer = "execution" + input_data_type = "trace_summary" + default_threshold = 0.6 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + prompt = """You are an expert evaluator assessing whether an AI agent successfully completed its assigned task. + +Evaluate the agent's performance across three dimensions: +1. **Goal Achievement** (1-5): Did the agent accomplish the main objective? +2. **Accuracy** (1-5): Is the result correct and free of errors? +3. **Completeness** (1-5): Did the agent address all aspects of the task? + +Then compute an overall score from 0 to 10 reflecting the combined quality. + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "goal_achievement": , + "accuracy": , + "completeness": , + "score": , + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for task completion evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Agent Execution Result +{input_data.content} + +Evaluate whether the agent completed its task and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] diff --git a/dingo/model/llm/agent_eval/llm_agent_tool_correctness.py b/dingo/model/llm/agent_eval/llm_agent_tool_correctness.py new file mode 100644 index 00000000..2d0f5232 --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_tool_correctness.py @@ -0,0 +1,82 @@ +""" +LLMAgentToolCorrectness: Evaluates whether the agent selected the correct tools for each step. + +Performs referenceless evaluation — no expected tool sequence is required. +The LLM judge assesses tool choices based on the task objective and the +context of each tool invocation. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.model import Model +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval + + +@Model.llm_register("LLMAgentToolCorrectness") +class LLMAgentToolCorrectness(BaseLLMAgentEval): + """ + Evaluates the correctness of tool selections in an agent's execution. + + Input: + prompt - The task objective or user request + content - JSON-formatted sequence of tool calls made by the agent + + Performs referenceless evaluation: the LLM judge determines whether + each tool choice was appropriate given the task and execution context, + without requiring a ground-truth tool sequence. + """ + + eval_layer = "action" + input_data_type = "tool_calls" + default_threshold = 0.6 + + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + prompt = """You are an expert evaluator assessing whether an AI agent selected the correct tools during task execution. + +For each tool call in the sequence, determine: +- Was this the right tool for the situation? +- Was this tool call necessary, or was it redundant? + +Count: +- **correct_calls**: Tool calls that were appropriate and necessary +- **total_calls**: Total number of tool calls made +- **redundant_calls**: Tool calls that were unnecessary or duplicated without reason + +List specific issues (wrong tool chosen, tool used out of order, missing tool that should have been called, etc.). + +A score of 10 means every tool call was correct and necessary. +A score of 0 means all tool calls were wrong or the agent failed to use required tools. + +Respond in the same language as the input content for the "reason" field. + +Return your evaluation as a JSON object with this exact schema: +{ + "correct_calls": , + "total_calls": , + "redundant_calls": , + "score": , + "issues": ["", ...], + "reason": "" +} + +Do not include any text outside the JSON object.""" + + @classmethod + def build_messages(cls, input_data: Data) -> List[dict]: + """Build LLM messages for tool correctness evaluation.""" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + user_content = f"""{cls.prompt} + +## Task Objective +{input_data.prompt} + +## Tool Call Sequence +{input_data.content} + +Evaluate the tool selections and return the JSON evaluation.{lang_hint}""" + + return [{"role": "user", "content": user_content}] diff --git a/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py b/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py new file mode 100644 index 00000000..5cb21869 --- /dev/null +++ b/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py @@ -0,0 +1,119 @@ +""" +LLMAgentTraceConclusion — synthesizes all evaluation results into a structured diagnosis. + +Called after all other evaluators complete for a trace. Takes the full set of +evaluation scores as input and produces: +- Overall severity (critical / warning / good) +- Root cause analysis +- Actionable recommendations +- A single aggregate score (0-10) + +This is NOT an evaluator in the traditional sense — it's a diagnostic synthesizer. +""" + +from typing import List + +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval +from dingo.model.model import Model + +CONCLUSION_PROMPT = """You are an AI agent quality analyst. Given the evaluation results from multiple evaluators that assessed an agent's execution trace, synthesize a comprehensive diagnosis. + +## Task Objective +{objective} + +## Evaluation Results +{eval_results} + +## Trace Summary +{trace_summary} + +## Instructions +Analyze all evaluation scores and produce a structured JSON diagnosis: + +1. **severity**: "critical" (any score < 0.3), "warning" (any score < 0.6), or "good" (all scores >= 0.6) +2. **root_causes**: List the primary reasons for any failures or low scores +3. **recommendations**: Actionable suggestions to improve the agent's performance +4. **highlights**: What the agent did well +5. **score**: Overall quality score from 0 to 10, weighing task completion most heavily + +Output STRICTLY as JSON: +```json +{{ + "severity": "critical|warning|good", + "root_causes": ["cause 1", "cause 2"], + "recommendations": ["rec 1", "rec 2"], + "highlights": ["highlight 1"], + "score": 0-10, + "summary": "One-paragraph overall assessment in the same language as the task objective" +}} +```""" + + +@Model.llm_register("LLMAgentTraceConclusion") +class LLMAgentTraceConclusion(BaseLLMAgentEval): + """Synthesize evaluation results into a structured trace-level diagnosis.""" + + eval_layer = "conclusion" + input_data_type = "eval_synthesis" + default_threshold = 0.5 + _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] + + @classmethod + def build_messages(cls, input_data: Data) -> List: + objective = getattr(input_data, "prompt", "") or "Agent trace" + eval_results = getattr(input_data, "content", "") or "{}" + trace_summary = getattr(input_data, "context", "") or "" + lang_hint = cls._detect_language_hint( + str(input_data.prompt) + str(input_data.content) + ) + + prompt_text = CONCLUSION_PROMPT.format( + objective=objective, + eval_results=eval_results, + trace_summary=trace_summary, + ) + lang_hint + return [{"role": "user", "content": prompt_text}] + + @classmethod + def process_response(cls, response: str) -> EvalDetail: + from dingo.utils import log + log.info(response) + + data = cls._parse_json_response(response) + + raw_score = data.get("score", 5) + try: + raw_score = float(raw_score) + except (TypeError, ValueError): + raw_score = 5.0 + + normalized_score = max(0.0, min(1.0, raw_score / 10.0)) + severity = data.get("severity", "warning") + + result = EvalDetail(metric=cls.__name__) + result.score = normalized_score + + if severity == "good": + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + elif severity == "critical": + result.status = True + result.label = ["AGENT_QUALITY.TraceConclusion.CRITICAL"] + else: + result.status = True + result.label = ["AGENT_QUALITY.TraceConclusion.WARNING"] + + import json + result.reason = [ + data.get("summary", ""), + json.dumps({ + "severity": severity, + "root_causes": data.get("root_causes", []), + "recommendations": data.get("recommendations", []), + "highlights": data.get("highlights", []), + }, ensure_ascii=False), + ] + + return result diff --git a/dingo/model/rule/rule_agent.py b/dingo/model/rule/rule_agent.py new file mode 100644 index 00000000..2606a81c --- /dev/null +++ b/dingo/model/rule/rule_agent.py @@ -0,0 +1,230 @@ +""" +Agent-specific rule evaluators for deterministic quality checks. + +These rules run without LLM calls, checking structural properties +of agent execution traces (loops, token budget, latency anomalies). +""" + +import json +import statistics +from typing import List, Optional + +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.base import BaseRule + + +@Model.rule_register("AGENT_TRACE_QUALITY", ["agent_trace_basic"]) +class RuleAgentTraceLoopDetection(BaseRule): + """Detect repetitive tool call patterns indicating infinite loops. + + Input: content = JSON array of tool call objects with 'tool_name' field. + Detection: n-gram analysis on tool name sequences. + A loop is detected when the same subsequence of 2+ tool names + repeats 3 or more consecutive times. + """ + + _required_fields = [RequiredField.CONTENT] + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + result = EvalDetail(metric=cls.__name__) + + tool_names = cls._extract_tool_names(input_data.content) + if len(tool_names) < 6: + result.label = [QualityLabel.QUALITY_GOOD] + return result + + loop_info = cls._detect_loops(tool_names) + if loop_info: + result.status = True + result.label = [f"{cls.metric_type}.{cls.__name__}"] + result.reason = [ + f"Loop detected: pattern {loop_info['pattern']} " + f"repeats {loop_info['count']} times at position {loop_info['position']}" + ] + else: + result.label = [QualityLabel.QUALITY_GOOD] + + return result + + @classmethod + def _extract_tool_names(cls, content: str) -> List[str]: + try: + data = json.loads(content) if isinstance(content, str) else content + except (json.JSONDecodeError, TypeError): + return [] + + if isinstance(data, dict): + items = data.get("tool_calls", data.get("steps", [])) + elif isinstance(data, list): + items = data + else: + return [] + + return [ + item.get("tool_name", item.get("name", "")) + for item in items + if isinstance(item, dict) and item.get("tool_name") or item.get("name") + ] + + @classmethod + def _detect_loops( + cls, names: List[str], min_pattern_len: int = 2, min_repeats: int = 3 + ) -> Optional[dict]: + for pattern_len in range(min_pattern_len, len(names) // min_repeats + 1): + for start in range(len(names) - pattern_len * min_repeats + 1): + pattern = names[start : start + pattern_len] + count = 1 + pos = start + pattern_len + while pos + pattern_len <= len(names): + if names[pos : pos + pattern_len] == pattern: + count += 1 + pos += pattern_len + else: + break + if count >= min_repeats: + return { + "pattern": pattern, + "count": count, + "position": start, + } + return None + + +@Model.rule_register("AGENT_TRACE_QUALITY", ["agent_trace_basic"]) +class RuleAgentTraceTokenBudget(BaseRule): + """Check if total token usage exceeds a configurable budget. + + Input: content = JSON with 'total_tokens' field, or metadata with token info. + Default budget: 500,000 tokens (configurable via dynamic_config.threshold). + """ + + _required_fields = [RequiredField.CONTENT] + dynamic_config = None + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + result = EvalDetail(metric=cls.__name__) + + budget = 500_000 + if cls.dynamic_config and hasattr(cls.dynamic_config, "threshold"): + try: + budget = int(cls.dynamic_config.threshold) + except (TypeError, ValueError): + pass + + total_tokens = cls._extract_tokens(input_data) + if total_tokens is None: + result.label = [QualityLabel.QUALITY_GOOD] + return result + + if total_tokens > budget: + result.status = True + result.label = [f"{cls.metric_type}.{cls.__name__}"] + result.reason = [ + f"Token usage {total_tokens:,} exceeds budget {budget:,}" + ] + result.score = min(1.0, budget / total_tokens) if total_tokens > 0 else 0.0 + else: + result.label = [QualityLabel.QUALITY_GOOD] + result.score = 1.0 + + return result + + @classmethod + def _extract_tokens(cls, input_data: Data) -> Optional[int]: + for source in [input_data.content, getattr(input_data, "metadata", None)]: + if source is None: + continue + try: + data = json.loads(source) if isinstance(source, str) else source + except (json.JSONDecodeError, TypeError): + continue + if isinstance(data, dict): + val = data.get("total_tokens") + if val is not None: + try: + return int(val) + except (TypeError, ValueError): + pass + return None + + +@Model.rule_register("AGENT_TRACE_QUALITY", ["agent_trace_basic"]) +class RuleAgentTraceLatencyAnomaly(BaseRule): + """Detect abnormally slow steps using statistical outlier analysis. + + Input: content = JSON array of step objects with 'duration' or 'duration_seconds' field. + A step is flagged if its duration exceeds mean + 3*stddev. + """ + + _required_fields = [RequiredField.CONTENT] + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + result = EvalDetail(metric=cls.__name__) + + steps = cls._extract_steps(input_data.content) + durations = [s["duration"] for s in steps if s["duration"] is not None and s["duration"] > 0] + + if len(durations) < 3: + result.label = [QualityLabel.QUALITY_GOOD] + return result + + mean = statistics.mean(durations) + stdev = statistics.stdev(durations) + threshold = mean + 3 * stdev + + anomalies = [ + s for s in steps + if s["duration"] is not None and s["duration"] > threshold + ] + + if anomalies: + result.status = True + result.label = [f"{cls.metric_type}.{cls.__name__}"] + result.reason = [ + f"Step '{a['name']}' took {a['duration']:.2f}s " + f"(threshold: {threshold:.2f}s, mean: {mean:.2f}s)" + for a in anomalies[:5] + ] + else: + result.label = [QualityLabel.QUALITY_GOOD] + + return result + + @classmethod + def _extract_steps(cls, content: str) -> List[dict]: + try: + data = json.loads(content) if isinstance(content, str) else content + except (json.JSONDecodeError, TypeError): + return [] + + if isinstance(data, dict): + items = data.get("steps", data.get("tool_calls", [])) + elif isinstance(data, list): + items = data + else: + return [] + + return [ + { + "name": item.get("name", "unknown"), + "duration": cls._safe_float( + item.get("duration", item.get("duration_seconds")) + ), + } + for item in items + if isinstance(item, dict) + ] + + @classmethod + def _safe_float(cls, val) -> Optional[float]: + if val is None: + return None + try: + return float(val) + except (TypeError, ValueError): + return None From 0965f01cd44d94eeff25aecc8b9db1269ffb6c6f Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 8 Jun 2026 14:12:00 +0800 Subject: [PATCH 2/6] chore: update .gitignore with AI artifacts, benchmarks, and downloads --- .gitignore | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.gitignore b/.gitignore index 91b2065f..e45c88dc 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,20 @@ outputs/ coverage.xml dingo_python.egg-info/* + +# Claude Code / AI assistant artifacts +.claude/ +CLAUDE.md +CLAUDE.local.md +claude_docs/ + +# Benchmarks and experiment data +benchmarks/ +meta_rater/ + +# Downloaded models / large binary files +downloads/ + +# Temporary docs (plans, specs) +docs/plans/ +docs/superpowers/ From 206d449920ab4ce25f162f8212e44ce76bf3d404 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 13 Jun 2026 16:43:08 +0800 Subject: [PATCH 3/6] fix(agent_eval): make scoring consistent and JSON parsing robust Audit fixes to the agent evaluators: - base: _parse_json_response now tolerates prose around the JSON and uppercase/missed code fences by extracting the first balanced object, so a recoverable response is no longer penalized as a hard parse failure (which previously surfaced as a misleading score of 0). - TraceConclusion: derive severity from the overall score (good >= 0.6, warning >= 0.3, else critical) instead of a separate LLM field, so the pass/fail status, severity, and numeric score can never diverge. - ErrorRecovery: drop the redundant `recovery_quality` field (the base reads `score`); make the no-error short-circuit parse the {error_events, steps} payload instead of exact-string matching, so a clean run skips the LLM call. - TaskCompletion / PlanQuality: anchor the overall score to the rated sub-dimensions so they can no longer contradict each other; PlanQuality also drops an unused keyword list. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llm/agent_eval/base_llm_agent_eval.py | 53 ++++++++++++++++--- .../agent_eval/llm_agent_error_recovery.py | 26 ++++++--- .../llm/agent_eval/llm_agent_plan_quality.py | 9 ++-- .../agent_eval/llm_agent_task_completion.py | 6 ++- .../agent_eval/llm_agent_trace_conclusion.py | 27 ++++++---- 5 files changed, 90 insertions(+), 31 deletions(-) diff --git a/dingo/model/llm/agent_eval/base_llm_agent_eval.py b/dingo/model/llm/agent_eval/base_llm_agent_eval.py index 5669cf1e..c2988141 100644 --- a/dingo/model/llm/agent_eval/base_llm_agent_eval.py +++ b/dingo/model/llm/agent_eval/base_llm_agent_eval.py @@ -53,23 +53,64 @@ def _get_threshold(cls) -> float: @classmethod def _strip_json_fences(cls, response: str) -> str: response = response.strip() - if response.startswith("```json"): - response = response[7:] + # Remove a leading code fence (```json / ```JSON / ```) of any case by + # dropping the fence line, then a trailing fence. if response.startswith("```"): - response = response[3:] + newline = response.find("\n") + response = response[newline + 1:] if newline != -1 else response[3:] if response.endswith("```"): response = response[:-3] return response.strip() + @staticmethod + def _extract_json_object(text: str) -> Optional[str]: + """Return the first balanced top-level JSON object substring, or None. + + Tolerates prose before/after the object (a common LLM output pattern) + and brace characters inside strings. + """ + start = text.find("{") + if start == -1: + return None + depth = 0 + in_str = False + esc = False + for i in range(start, len(text)): + ch = text[i] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + elif ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start:i + 1] + return None + @classmethod def _parse_json_response(cls, response: str) -> dict: cleaned = cls._strip_json_fences(response) try: return json.loads(cleaned) except json.JSONDecodeError: - raise ConvertJsonError( - f"Failed to parse agent eval JSON: {cleaned[:500]}" - ) + pass + # Fall back to extracting the first balanced JSON object — handles + # responses with prose around the JSON or a missed/uppercase fence — + # so a recoverable response is not penalized as a parse failure. + candidate = cls._extract_json_object(cleaned) or cls._extract_json_object(response) + if candidate: + try: + return json.loads(candidate) + except json.JSONDecodeError: + pass + raise ConvertJsonError(f"Failed to parse agent eval JSON: {cleaned[:500]}") @classmethod def process_response(cls, response: str) -> EvalDetail: diff --git a/dingo/model/llm/agent_eval/llm_agent_error_recovery.py b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py index b7f260ab..0f9cd38d 100644 --- a/dingo/model/llm/agent_eval/llm_agent_error_recovery.py +++ b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py @@ -5,6 +5,7 @@ since perfect execution with no errors requires no recovery. """ +import json import time from typing import List @@ -57,7 +58,7 @@ class LLMAgentErrorRecovery(BaseLLMAgentEval): - **errors_encountered**: Total number of distinct errors or failures - **recovered_count**: Number of errors from which the agent successfully recovered -Assess overall **recovery_quality** from 0 to 10: +Assess the overall recovery **score** from 0 to 10: - 10: Agent recovered from all errors with optimal strategies - 7-9: Agent recovered from most errors with reasonable strategies - 4-6: Agent recovered from some errors but used suboptimal approaches @@ -70,7 +71,6 @@ class LLMAgentErrorRecovery(BaseLLMAgentEval): { "errors_encountered": , "recovered_count": , - "recovery_quality": , "score": , "reason": "" } @@ -79,14 +79,26 @@ class LLMAgentErrorRecovery(BaseLLMAgentEval): @classmethod def _has_error_events(cls, content: str) -> bool: - """Check if the content contains actual error events.""" + """Check if the content contains actual error events. + + The structured payload is ``{"error_events": [...], "steps": [...]}`` — + whether errors occurred is the ``error_events`` list, not the wrapper + (which is never literally ``{}``/``[]``). Parse it so a clean run + short-circuits to a pass instead of spending an LLM call on an empty + error list. Falls back to phrase matching for free-text inputs. + """ if not content or not content.strip(): return False + try: + parsed = json.loads(content) + if isinstance(parsed, dict) and "error_events" in parsed: + return bool(parsed.get("error_events")) + if isinstance(parsed, list): + return bool(parsed) + except (ValueError, TypeError): + pass stripped = content.strip().lower() - for indicator in cls._NO_ERROR_INDICATORS: - if stripped == indicator: - return False - return True + return stripped not in cls._NO_ERROR_INDICATORS @classmethod def build_messages(cls, input_data: Data) -> List[dict]: diff --git a/dingo/model/llm/agent_eval/llm_agent_plan_quality.py b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py index be4f33da..0d3b85bd 100644 --- a/dingo/model/llm/agent_eval/llm_agent_plan_quality.py +++ b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py @@ -41,11 +41,6 @@ class LLMAgentPlanQuality(BaseLLMAgentEval): _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT] - _NO_PLAN_KEYWORDS = [ - "no plan", "no planning", "no explicit plan", - "did not plan", "skipped planning", "planning not found", - ] - prompt = """You are an expert evaluator assessing the quality of an AI agent's reasoning plan. First, determine whether the trace contains any planning content (explicit steps, strategy, or reasoning about how to approach the task). If there is no planning content at all, set score to -1 as a sentinel value. @@ -55,7 +50,9 @@ class LLMAgentPlanQuality(BaseLLMAgentEval): 2. **Completeness** (1-5): Does the plan cover all necessary steps to achieve the goal? 3. **Feasibility** (1-5): Are the planned steps realistic and achievable? -Compute an overall score from 0 to 10. +Compute an overall **score** from 0 to 10 that is CONSISTENT with the three +dimensions above (e.g. all 5s → 9-10, all 3s → 5-6, all 1s → 0-2). Do not let +the score contradict the dimension ratings. IMPORTANT: The "reason" field MUST be in the same language as the Task Objective. If the task objective is in Chinese, respond in Chinese. If in English, respond in English. diff --git a/dingo/model/llm/agent_eval/llm_agent_task_completion.py b/dingo/model/llm/agent_eval/llm_agent_task_completion.py index ec21019b..12d6b662 100644 --- a/dingo/model/llm/agent_eval/llm_agent_task_completion.py +++ b/dingo/model/llm/agent_eval/llm_agent_task_completion.py @@ -38,7 +38,11 @@ class LLMAgentTaskCompletion(BaseLLMAgentEval): 2. **Accuracy** (1-5): Is the result correct and free of errors? 3. **Completeness** (1-5): Did the agent address all aspects of the task? -Then compute an overall score from 0 to 10 reflecting the combined quality. +Then compute an overall **score** from 0 to 10 that is CONSISTENT with the three +dimensions above — it must reflect their combined level (e.g. all 5s → 9-10, +all 4s → 7-8, all 3s → 5-6, all 2s → 3-4, all 1s → 0-2). Do not let the score +contradict the dimension ratings. If the run was incomplete, credit verifiable +partial progress rather than scoring 0. Respond in the same language as the input content for the "reason" field. diff --git a/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py b/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py index 5cb21869..74a49362 100644 --- a/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py +++ b/dingo/model/llm/agent_eval/llm_agent_trace_conclusion.py @@ -11,6 +11,7 @@ This is NOT an evaluator in the traditional sense — it's a diagnostic synthesizer. """ +import json from typing import List from dingo.io.input import Data, RequiredField @@ -32,11 +33,11 @@ ## Instructions Analyze all evaluation scores and produce a structured JSON diagnosis: -1. **severity**: "critical" (any score < 0.3), "warning" (any score < 0.6), or "good" (all scores >= 0.6) -2. **root_causes**: List the primary reasons for any failures or low scores -3. **recommendations**: Actionable suggestions to improve the agent's performance -4. **highlights**: What the agent did well -5. **score**: Overall quality score from 0 to 10, weighing task completion most heavily +1. **score**: Overall quality score from 0 to 10, weighing task completion most heavily +2. **severity**: the overall tier, and it MUST be consistent with **score** — "critical" if score < 3, "warning" if 3 <= score < 6, "good" if score >= 6 +3. **root_causes**: List the primary reasons for any failures or low scores +4. **recommendations**: Actionable suggestions to improve the agent's performance +5. **highlights**: What the agent did well Output STRICTLY as JSON: ```json @@ -90,22 +91,26 @@ def process_response(cls, response: str) -> EvalDetail: raw_score = 5.0 normalized_score = max(0.0, min(1.0, raw_score / 10.0)) - severity = data.get("severity", "warning") result = EvalDetail(metric=cls.__name__) result.score = normalized_score - if severity == "good": + # Derive severity from the overall score so the pass/fail status, the + # severity badge, and the numeric score are always consistent (the LLM's + # own "severity" field is advisory and can drift from its score). + if normalized_score >= 0.6: + severity = "good" result.status = False result.label = [QualityLabel.QUALITY_GOOD] - elif severity == "critical": + elif normalized_score >= 0.3: + severity = "warning" result.status = True - result.label = ["AGENT_QUALITY.TraceConclusion.CRITICAL"] + result.label = ["AGENT_QUALITY.TraceConclusion.WARNING"] else: + severity = "critical" result.status = True - result.label = ["AGENT_QUALITY.TraceConclusion.WARNING"] + result.label = ["AGENT_QUALITY.TraceConclusion.CRITICAL"] - import json result.reason = [ data.get("summary", ""), json.dumps({ From 75d54ed45ec682c00d39bbc81dd7ba404edf73e0 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 15 Jun 2026 13:52:25 +0800 Subject: [PATCH 4/6] refactor(llm): dedupe agent eval retry loops via super() delegation LLMAgentErrorRecovery and LLMAgentPlanQuality each copied the full retry/parse/error-fallback loop from BaseOpenAI.eval. Replace the duplication with the template-method hooks: - ErrorRecovery: keep the no-error short-circuit, then `return super().eval()` instead of re-implementing the loop. - PlanQuality: override `process_response` to handle the score=-1 "no planning" sentinel and delegate everything else to `super().process_response()`; drop the custom `eval`. Behavior is identical (verified: short-circuit, sentinel, and normal 8/10 -> 0.8 scoring paths). Also removes now-unused imports. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agent_eval/llm_agent_error_recovery.py | 45 ++------ .../llm/agent_eval/llm_agent_plan_quality.py | 104 +++++------------- 2 files changed, 37 insertions(+), 112 deletions(-) diff --git a/dingo/model/llm/agent_eval/llm_agent_error_recovery.py b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py index 0f9cd38d..f16b6013 100644 --- a/dingo/model/llm/agent_eval/llm_agent_error_recovery.py +++ b/dingo/model/llm/agent_eval/llm_agent_error_recovery.py @@ -6,7 +6,6 @@ """ import json -import time from typing import List from dingo.io.input import Data, RequiredField @@ -14,12 +13,6 @@ from dingo.model import Model from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval from dingo.utils import log -from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens - -try: - from pydantic import ValidationError -except ImportError: - ValidationError = Exception @Model.llm_register("LLMAgentErrorRecovery") @@ -120,7 +113,14 @@ def build_messages(cls, input_data: Data) -> List[dict]: @classmethod def eval(cls, input_data: Data) -> EvalDetail: - """Override eval() to handle the no-error special case.""" + """Override eval() to handle the no-error special case. + + When the trace contains no error events, recovery is vacuously perfect, + so we short-circuit to a pass without an LLM call. Otherwise we delegate + to the shared retry/parse pipeline in ``BaseOpenAI.eval``, which drives + this class's ``build_messages`` and inherited ``process_response`` — no + need to duplicate the retry loop and error fallback here. + """ content = getattr(input_data, "content", "") or "" if not cls._has_error_events(content): @@ -132,31 +132,4 @@ def eval(cls, input_data: Data) -> EvalDetail: result.reason = ["No error events found in execution trace; recovery evaluation skipped."] return result - if cls.client is None: - cls.create_client() - - messages = cls.build_messages(input_data) - - attempts = 0 - except_msg = "" - except_name = Exception.__class__.__name__ - while attempts < 3: - try: - response = cls.send_messages(messages) - res: EvalDetail = cls.process_response(response) - return res - except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: - except_msg = str(e) - except_name = e.__class__.__name__ - break - except Exception as e: - attempts += 1 - time.sleep(1) - except_msg = str(e) - except_name = e.__class__.__name__ - - res = EvalDetail(metric=cls.__name__) - res.status = True - res.label = [f"QUALITY_BAD.{except_name}"] - res.reason = [except_msg] - return res + return super().eval(input_data) diff --git a/dingo/model/llm/agent_eval/llm_agent_plan_quality.py b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py index 0d3b85bd..79582ab3 100644 --- a/dingo/model/llm/agent_eval/llm_agent_plan_quality.py +++ b/dingo/model/llm/agent_eval/llm_agent_plan_quality.py @@ -5,7 +5,6 @@ If no planning content is found in the trace, defaults to passing (score=1.0). """ -import time from typing import List from dingo.io.input import Data, RequiredField @@ -13,12 +12,6 @@ from dingo.model import Model from dingo.model.llm.agent_eval.base_llm_agent_eval import BaseLLMAgentEval from dingo.utils import log -from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens - -try: - from pydantic import ValidationError -except ImportError: - ValidationError = Exception @Model.llm_register("LLMAgentPlanQuality") @@ -86,72 +79,31 @@ def build_messages(cls, input_data: Data) -> List[dict]: return [{"role": "user", "content": user_content}] @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - """Override eval() to handle the no-planning special case.""" - if cls.client is None: - cls.create_client() - - messages = cls.build_messages(input_data) - - attempts = 0 - except_msg = "" - except_name = Exception.__class__.__name__ - while attempts < 3: - try: - response = cls.send_messages(messages) - - data = cls._parse_json_response(response) - raw_score = data.get("score", 0) - - try: - raw_score = float(raw_score) - except (TypeError, ValueError): - raw_score = 0.0 - - result = EvalDetail(metric=cls.__name__) - - if raw_score < 0: - # Sentinel value: no planning content found, treat as pass - log.info(f"{cls.__name__}: No planning content found in trace, defaulting to pass") - result.status = False - result.label = [QualityLabel.QUALITY_GOOD] - result.score = 1.0 - result.reason = [data.get("reason", "No planning content found; evaluation skipped.")] - return result - - normalized_score = max(0.0, min(1.0, raw_score / 10.0)) - threshold = cls._get_threshold() - reason_text = data.get("reason", "") - details = {k: v for k, v in data.items() if k not in ("score", "reason")} - - import json - result.score = normalized_score - if normalized_score >= threshold: - result.status = False - result.label = [QualityLabel.QUALITY_GOOD] - else: - result.status = True - result.label = [f"AGENT_QUALITY.{cls.__name__}"] - - reason_parts = [reason_text] if reason_text else [] - if details: - reason_parts.append(json.dumps(details, ensure_ascii=False, default=str)) - result.reason = reason_parts if reason_parts else None - - return result - - except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: - except_msg = str(e) - except_name = e.__class__.__name__ - break - except Exception as e: - attempts += 1 - time.sleep(1) - except_msg = str(e) - except_name = e.__class__.__name__ - - res = EvalDetail(metric=cls.__name__) - res.status = True - res.label = [f"QUALITY_BAD.{except_name}"] - res.reason = [except_msg] - return res + def process_response(cls, response: str) -> EvalDetail: + """Handle the no-planning sentinel, else use the standard agent scoring. + + The prompt instructs the model to return ``score = -1`` when the trace + contains no planning content at all; that is treated as a pass (absence + of planning can be acceptable for simple tasks). Any other score is + delegated to ``BaseLLMAgentEval.process_response`` for the standard + 0~10 → 0.0~1.0 normalization and threshold comparison. By only + overriding ``process_response`` (not ``eval``), the retry loop and error + fallback in ``BaseOpenAI.eval`` are reused instead of duplicated. + """ + data = cls._parse_json_response(response) + raw_score = data.get("score", 0) + try: + raw_score = float(raw_score) + except (TypeError, ValueError): + raw_score = 0.0 + + if raw_score < 0: + log.info(f"{cls.__name__}: No planning content found in trace, defaulting to pass") + result = EvalDetail(metric=cls.__name__) + result.status = False + result.label = [QualityLabel.QUALITY_GOOD] + result.score = 1.0 + result.reason = [data.get("reason", "No planning content found; evaluation skipped.")] + return result + + return super().process_response(response) From a8aa6852f1653dd16d5a261c0a8d731beb6f7666 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 15 Jun 2026 13:52:56 +0800 Subject: [PATCH 5/6] fix(rule): harden agent-trace rules and expose trace-level eval hooks Fix three crashes in the deterministic agent-trace rules and wire them for trace-level orchestration: - _extract_tool_names / _extract_steps: a present-but-null "tool_calls" /"steps" key (JSON null) made dict.get return None and broke iteration (TypeError); use `a or b or []` plus a list guard. - _extract_tool_names: parenthesize the filter so a non-dict item no longer reaches `item.get(...)` and raises AttributeError (operator precedence: `and` binds tighter than `or`). - RuleAgentTraceTokenBudget.dynamic_config: None -> EvaluatorRuleArgs( threshold=500_000) so set_config_rule's model_copy() in the local/spark executors does not raise, matching every other rule. - Add input_data_type="agent_trace_json" / eval_layer to all three rules so agent orchestrators feed them the trace-level tool-call sequence as JSON instead of running them per-span on plain text. Adds test_rule_agent.py (previously zero coverage) covering the malformed-input edge cases and the registration/attribute contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- dingo/model/rule/rule_agent.py | 43 +++++- test/scripts/model/rule/test_rule_agent.py | 170 +++++++++++++++++++++ 2 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 test/scripts/model/rule/test_rule_agent.py diff --git a/dingo/model/rule/rule_agent.py b/dingo/model/rule/rule_agent.py index 2606a81c..4d765c4e 100644 --- a/dingo/model/rule/rule_agent.py +++ b/dingo/model/rule/rule_agent.py @@ -9,6 +9,7 @@ import statistics from typing import List, Optional +from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model @@ -25,6 +26,12 @@ class RuleAgentTraceLoopDetection(BaseRule): repeats 3 or more consecutive times. """ + # Trace-level evaluator: declares input_data_type / eval_layer so agent + # orchestrators (e.g. dingo-saas) feed it the whole tool-call sequence as + # JSON rather than running it per-span on plain text. + eval_layer = "trajectory" + input_data_type = "agent_trace_json" + _required_fields = [RequiredField.CONTENT] @classmethod @@ -56,17 +63,24 @@ def _extract_tool_names(cls, content: str) -> List[str]: except (json.JSONDecodeError, TypeError): return [] + items = [] if isinstance(data, dict): - items = data.get("tool_calls", data.get("steps", [])) + # `or` (not dict.get default): a present-but-null "tool_calls" key + # returns None, which would break iteration — fall through to steps. + items = data.get("tool_calls") or data.get("steps") or [] elif isinstance(data, list): items = data - else: + + if not isinstance(items, list): return [] + # Parenthesize the filter: `and` binds tighter than `or`, so without + # these parens a non-dict item would reach `item.get("name")` and raise + # AttributeError. return [ item.get("tool_name", item.get("name", "")) for item in items - if isinstance(item, dict) and item.get("tool_name") or item.get("name") + if isinstance(item, dict) and (item.get("tool_name") or item.get("name")) ] @classmethod @@ -101,15 +115,22 @@ class RuleAgentTraceTokenBudget(BaseRule): Default budget: 500,000 tokens (configurable via dynamic_config.threshold). """ + # Trace-level evaluator (see RuleAgentTraceLoopDetection for rationale). + eval_layer = "efficiency" + input_data_type = "agent_trace_json" + _required_fields = [RequiredField.CONTENT] - dynamic_config = None + # A real EvaluatorRuleArgs (not None) so set_config_rule's model_copy() in + # the local/spark executors does not raise; the default also documents the + # 500k budget and keeps it overridable via dynamic_config.threshold. + dynamic_config = EvaluatorRuleArgs(threshold=500_000) @classmethod def eval(cls, input_data: Data) -> EvalDetail: result = EvalDetail(metric=cls.__name__) budget = 500_000 - if cls.dynamic_config and hasattr(cls.dynamic_config, "threshold"): + if cls.dynamic_config and cls.dynamic_config.threshold is not None: try: budget = int(cls.dynamic_config.threshold) except (TypeError, ValueError): @@ -160,6 +181,10 @@ class RuleAgentTraceLatencyAnomaly(BaseRule): A step is flagged if its duration exceeds mean + 3*stddev. """ + # Trace-level evaluator (see RuleAgentTraceLoopDetection for rationale). + eval_layer = "efficiency" + input_data_type = "agent_trace_json" + _required_fields = [RequiredField.CONTENT] @classmethod @@ -202,11 +227,15 @@ def _extract_steps(cls, content: str) -> List[dict]: except (json.JSONDecodeError, TypeError): return [] + items = [] if isinstance(data, dict): - items = data.get("steps", data.get("tool_calls", [])) + # `or` (not dict.get default): a present-but-null key returns None, + # which would break the iteration below. + items = data.get("steps") or data.get("tool_calls") or [] elif isinstance(data, list): items = data - else: + + if not isinstance(items, list): return [] return [ diff --git a/test/scripts/model/rule/test_rule_agent.py b/test/scripts/model/rule/test_rule_agent.py new file mode 100644 index 00000000..9db6cd43 --- /dev/null +++ b/test/scripts/model/rule/test_rule_agent.py @@ -0,0 +1,170 @@ +"""Unit tests for the deterministic agent-trace rules in ``rule_agent.py``. + +Covers the three trace-level rules and, importantly, the malformed-input edge +cases (JSON ``null`` values, non-dict items) that previously raised +``TypeError`` / ``AttributeError`` — plus the ``dynamic_config`` / +``input_data_type`` contract the dingo-saas orchestrator depends on. +""" + +import json + +import pytest + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.io.output.eval_detail import QualityLabel +from dingo.model.model import Model +from dingo.model.rule.rule_agent import RuleAgentTraceLatencyAnomaly, RuleAgentTraceLoopDetection, RuleAgentTraceTokenBudget + + +def _data(content: str) -> Data: + return Data(data_id="t", content=content) + + +class TestRuleAgentTraceLoopDetection: + """Loop detection via n-gram repetition on the tool-name sequence.""" + + def test_repeating_pattern_is_flagged(self): + # [search, click] repeated 4 times → pattern len 2 repeats ≥ 3. + content = json.dumps([{"tool_name": "search"}, {"tool_name": "click"}] * 4) + res = RuleAgentTraceLoopDetection.eval(_data(content)) + assert res.status is True + assert res.label == ["AGENT_TRACE_QUALITY.RuleAgentTraceLoopDetection"] + assert res.reason is not None and "Loop detected" in res.reason[0] + + def test_dict_form_with_tool_calls_key(self): + content = json.dumps( + {"tool_calls": [{"tool_name": "a"}, {"tool_name": "b"}, {"tool_name": "c"}] * 3} + ) + res = RuleAgentTraceLoopDetection.eval(_data(content)) + assert res.status is True + + def test_distinct_sequence_passes(self): + content = json.dumps([{"tool_name": f"tool_{i}"} for i in range(8)]) + res = RuleAgentTraceLoopDetection.eval(_data(content)) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_short_sequence_passes(self): + # Fewer than 6 tool names → early pass, no analysis. + content = json.dumps([{"tool_name": "a"}, {"tool_name": "b"}]) + res = RuleAgentTraceLoopDetection.eval(_data(content)) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_null_tool_calls_does_not_crash(self): + # JSON null for a present key previously broke iteration (TypeError). + res = RuleAgentTraceLoopDetection.eval(_data('{"tool_calls": null}')) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_non_dict_items_do_not_crash(self): + # Operator-precedence bug previously raised AttributeError on "a"/null. + res = RuleAgentTraceLoopDetection.eval(_data('["a", null, {"name": "x"}]')) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_non_json_text_passes(self): + # Plain text content (not JSON) must be tolerated, not raise. + res = RuleAgentTraceLoopDetection.eval(_data("just some free text")) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + +class TestRuleAgentTraceTokenBudget: + """Token-budget rule with a default-but-overridable 500k threshold.""" + + def test_default_threshold_is_500k(self): + assert isinstance(RuleAgentTraceTokenBudget.dynamic_config, EvaluatorRuleArgs) + assert RuleAgentTraceTokenBudget.dynamic_config.threshold == 500_000 + + def test_under_budget_passes(self): + res = RuleAgentTraceTokenBudget.eval(_data('{"total_tokens": 1000}')) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert res.score == 1.0 + + def test_over_budget_is_flagged(self): + res = RuleAgentTraceTokenBudget.eval(_data('{"total_tokens": 600000}')) + assert res.status is True + assert res.label == ["AGENT_TRACE_QUALITY.RuleAgentTraceTokenBudget"] + assert res.reason is not None and "exceeds budget" in res.reason[0] + + def test_missing_tokens_passes(self): + res = RuleAgentTraceTokenBudget.eval(_data('{"other_field": 1}')) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_threshold_override_via_dynamic_config(self): + original = RuleAgentTraceTokenBudget.dynamic_config.threshold + try: + RuleAgentTraceTokenBudget.dynamic_config.threshold = 100 + res = RuleAgentTraceTokenBudget.eval(_data('{"total_tokens": 200}')) + assert res.status is True + finally: + RuleAgentTraceTokenBudget.dynamic_config.threshold = original + + +class TestRuleAgentTraceLatencyAnomaly: + """Latency anomaly via mean + 3·stddev outlier detection.""" + + def test_outlier_is_flagged(self): + steps = [{"name": f"s{i}", "duration": 1.0} for i in range(15)] + steps.append({"name": "slow", "duration": 40.0}) + res = RuleAgentTraceLatencyAnomaly.eval(_data(json.dumps({"steps": steps}))) + assert res.status is True + assert res.label == ["AGENT_TRACE_QUALITY.RuleAgentTraceLatencyAnomaly"] + assert res.reason is not None and "slow" in res.reason[0] + + def test_uniform_durations_pass(self): + steps = [{"name": f"s{i}", "duration": 1.0} for i in range(5)] + res = RuleAgentTraceLatencyAnomaly.eval(_data(json.dumps({"steps": steps}))) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_few_steps_pass(self): + # Fewer than 3 valid durations → not enough data, early pass. + steps = [{"name": "a", "duration": 1.0}, {"name": "b", "duration": 99.0}] + res = RuleAgentTraceLatencyAnomaly.eval(_data(json.dumps({"steps": steps}))) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + def test_duration_seconds_alias(self): + steps = [{"name": f"s{i}", "duration_seconds": 1.0} for i in range(15)] + steps.append({"name": "slow", "duration_seconds": 40.0}) + res = RuleAgentTraceLatencyAnomaly.eval(_data(json.dumps({"steps": steps}))) + assert res.status is True + + def test_null_steps_does_not_crash(self): + res = RuleAgentTraceLatencyAnomaly.eval(_data('{"steps": null}')) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + + +class TestRuleAgentRegistration: + """The saas orchestrator keys off rule_name_map + the agent attributes.""" + + @pytest.mark.parametrize( + "name", + [ + "RuleAgentTraceLoopDetection", + "RuleAgentTraceTokenBudget", + "RuleAgentTraceLatencyAnomaly", + ], + ) + def test_registered_in_name_map(self, name): + assert name in Model.rule_name_map + + @pytest.mark.parametrize( + "rule_cls", + [ + RuleAgentTraceLoopDetection, + RuleAgentTraceTokenBudget, + RuleAgentTraceLatencyAnomaly, + ], + ) + def test_declares_agent_eval_attributes(self, rule_cls): + # input_data_type + eval_layer are what dingo-saas' _is_agent_evaluator + # checks to route these as trace-level (not span-level) evaluators. + assert rule_cls.input_data_type == "agent_trace_json" + assert rule_cls.eval_layer From e04aed60d484f8d2fe42bf267c4ff49e59fc8df3 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 15 Jun 2026 14:06:14 +0800 Subject: [PATCH 6/6] refactor(rule): extract shared _load_trace_items parse helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _extract_tool_names and _extract_steps had an identical parse-and-normalize block (JSON string/list/dict → list of items, with the null-key and non-list guards) differing only in key priority. Hoist it into a module-level _load_trace_items() so each extractor keeps only its distinct mapping. No behavior change — covered by the existing test_rule_agent.py edge-case suite. Co-Authored-By: Claude Opus 4.8 (1M context) --- dingo/model/rule/rule_agent.py | 63 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/dingo/model/rule/rule_agent.py b/dingo/model/rule/rule_agent.py index 4d765c4e..dc819682 100644 --- a/dingo/model/rule/rule_agent.py +++ b/dingo/model/rule/rule_agent.py @@ -7,7 +7,7 @@ import json import statistics -from typing import List, Optional +from typing import Any, List, Optional from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data, RequiredField @@ -16,6 +16,31 @@ from dingo.model.rule.base import BaseRule +def _load_trace_items(content: Any, primary_key: str, secondary_key: str) -> List[dict]: + """Parse agent-trace ``content`` into a list of step / tool-call items. + + Accepts a JSON string, an already-parsed list, or a dict wrapper. For a dict + payload, returns the first non-empty list among ``primary_key`` / + ``secondary_key`` — using ``or`` (not ``dict.get`` defaults) so a + present-but-null key falls through instead of yielding ``None``. Returns + ``[]`` for free text, JSON null, or any non-list payload, so callers can + iterate the result without guarding against ``None``. + """ + try: + data = json.loads(content) if isinstance(content, str) else content + except (json.JSONDecodeError, TypeError): + return [] + + if isinstance(data, dict): + items = data.get(primary_key) or data.get(secondary_key) or [] + elif isinstance(data, list): + items = data + else: + return [] + + return items if isinstance(items, list) else [] + + @Model.rule_register("AGENT_TRACE_QUALITY", ["agent_trace_basic"]) class RuleAgentTraceLoopDetection(BaseRule): """Detect repetitive tool call patterns indicating infinite loops. @@ -58,28 +83,12 @@ def eval(cls, input_data: Data) -> EvalDetail: @classmethod def _extract_tool_names(cls, content: str) -> List[str]: - try: - data = json.loads(content) if isinstance(content, str) else content - except (json.JSONDecodeError, TypeError): - return [] - - items = [] - if isinstance(data, dict): - # `or` (not dict.get default): a present-but-null "tool_calls" key - # returns None, which would break iteration — fall through to steps. - items = data.get("tool_calls") or data.get("steps") or [] - elif isinstance(data, list): - items = data - - if not isinstance(items, list): - return [] - # Parenthesize the filter: `and` binds tighter than `or`, so without # these parens a non-dict item would reach `item.get("name")` and raise # AttributeError. return [ item.get("tool_name", item.get("name", "")) - for item in items + for item in _load_trace_items(content, "tool_calls", "steps") if isinstance(item, dict) and (item.get("tool_name") or item.get("name")) ] @@ -222,22 +231,6 @@ def eval(cls, input_data: Data) -> EvalDetail: @classmethod def _extract_steps(cls, content: str) -> List[dict]: - try: - data = json.loads(content) if isinstance(content, str) else content - except (json.JSONDecodeError, TypeError): - return [] - - items = [] - if isinstance(data, dict): - # `or` (not dict.get default): a present-but-null key returns None, - # which would break the iteration below. - items = data.get("steps") or data.get("tool_calls") or [] - elif isinstance(data, list): - items = data - - if not isinstance(items, list): - return [] - return [ { "name": item.get("name", "unknown"), @@ -245,7 +238,7 @@ def _extract_steps(cls, content: str) -> List[dict]: item.get("duration", item.get("duration_seconds")) ), } - for item in items + for item in _load_trace_items(content, "steps", "tool_calls") if isinstance(item, dict) ]