cli.py

from __future__ import annotations

import argparse
import json
import os
import re
import shlex
import subprocess
import sys
from dataclasses import asdict
from pathlib import Path
from urllib.parse import parse_qs, unquote, urlparse

import httpx
import yaml

from .graders import count_markdown_tables, count_words, grade_text
from .scenarios import SCENARIOS
from .types import GradeFinding, RubricCheck, Scenario

_STAGE_DIMENSIONS = {
    "scope": "scope_agent",
    "query": "query_agent",
    "mediator": "mediator",
}


def _int_value(value: object) -> int:
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


def _read_text(path: str) -> str:
    if path == "-":
        return sys.stdin.read()
    return Path(path).read_text(encoding="utf-8")


def _cmd_list() -> int:
    for scenario in SCENARIOS.values():
        demands = ", ".join(scenario.superpower_demand) or "none"
        print(f"{scenario.id}\t{scenario.title}\t{demands}")
    return 0


def _cmd_grade(args: argparse.Namespace) -> int:
    scenario = SCENARIOS[args.scenario]
    output = _read_text(args.output)
    result = grade_text(output, scenario)
    print(json.dumps(asdict(result), indent=2, sort_keys=True))
    return 0 if result.passed else 1


def _hmctl_base(profile: str | None = None) -> list[str]:
    cmd = shlex.split(os.environ.get("HMCTL_BIN", "uv run hmctl"))
    if profile:
        cmd.extend(["--profile", profile])
    return cmd


def _profile_name(profile: str | None) -> str:
    if profile:
        return profile
    if env_profile := os.environ.get("HIVEMIND_PROFILE", "").strip():
        return env_profile
    active_path = Path.home() / ".hivemind" / "active"
    try:
        active = active_path.read_text(encoding="utf-8").strip()
    except OSError:
        active = ""
    return active or "default"


def _profile_api_key(profile: str | None) -> str:
    profile_path = Path.home() / ".hivemind" / "profiles" / f"{_profile_name(profile)}.yaml"
    try:
        config = yaml.safe_load(profile_path.read_text(encoding="utf-8")) or {}
    except OSError:
        return ""
    except yaml.YAMLError:
        return ""
    api_key = str(config.get("api_key") or "").strip()
    return api_key


def _hmroom_run_headers(room_ref: str, profile: str | None) -> tuple[str, dict] | None:
    if not room_ref.startswith("hmroom://"):
        return None
    parsed = urlparse(room_ref)
    qs = parse_qs(parsed.query)
    service = unquote((qs.get("service") or [""])[0]).rstrip("/")
    token = unquote((qs.get("token") or qs.get("share") or [""])[0])
    if not service or not token:
        return None
    headers = {"Authorization": f"Bearer {token}"}
    api_key = _profile_api_key(profile)
    if api_key.startswith("hmk_"):
        headers["X-Hivemind-Api-Key"] = api_key
    return service, headers


def _fetch_hmroom_run_telemetry(
    room_ref: str,
    run_id: str,
    profile: str | None,
) -> subprocess.CompletedProcess | None:
    resolved = _hmroom_run_headers(room_ref, profile)
    if resolved is None:
        return None
    service, headers = resolved
    cmd = ["GET", f"{service}/v1/runs/{run_id}"]
    try:
        resp = httpx.get(
            f"{service}/v1/runs/{run_id}",
            headers=headers,
            timeout=60,
        )
    except httpx.HTTPError as e:
        return subprocess.CompletedProcess(cmd, 1, stdout="", stderr=f"{type(e).__name__}: {e}")
    if resp.status_code >= 400:
        return subprocess.CompletedProcess(
            cmd,
            1,
            stdout=resp.text,
            stderr=f"HTTP {resp.status_code}: {resp.text[:500]}",
        )
    return subprocess.CompletedProcess(cmd, 0, stdout=resp.text, stderr="")


def _stage_seconds(run: dict, stage: str) -> float | None:
    started = run.get(f"{stage}_started_at")
    ended = run.get(f"{stage}_ended_at")
    try:
        if started is None or ended is None:
            return None
        return round(float(ended) - float(started), 3)
    except (TypeError, ValueError):
        return None


def _run_seconds(run: dict) -> float | None:
    try:
        return round(float(run["updated_at"]) - float(run["created_at"]), 3)
    except (KeyError, TypeError, ValueError):
        return None


def _coerce_usage(run: dict) -> dict:
    usage = run.get("usage") or run.get("usage_json") or {}
    if isinstance(usage, str):
        try:
            parsed = json.loads(usage)
        except json.JSONDecodeError:
            return {}
        return parsed if isinstance(parsed, dict) else {}
    return usage if isinstance(usage, dict) else {}


def _usage_stages(usage: dict) -> dict:
    stages = usage.get("stages") if isinstance(usage, dict) else {}
    return stages if isinstance(stages, dict) else {}


def _attestation_body(run: dict) -> dict:
    envelope = run.get("attestation") or {}
    if isinstance(envelope, str):
        try:
            envelope = json.loads(envelope)
        except json.JSONDecodeError:
            return {}
    if not isinstance(envelope, dict):
        return {}
    body = envelope.get("body") or {}
    if isinstance(body, str):
        try:
            body = json.loads(body)
        except json.JSONDecodeError:
            return {}
    return body if isinstance(body, dict) else {}


def _merge_counts(dst: dict[str, int], src: object) -> None:
    if not isinstance(src, dict):
        return
    for key, value in src.items():
        dst[str(key)] = dst.get(str(key), 0) + _int_value(value)


def _extract_bridge_counts(usage: dict) -> tuple[dict[str, int], dict[str, int]]:
    tool_counts: dict[str, int] = {}
    llm_tool_counts: dict[str, int] = {}
    bridge = usage.get("bridge") if isinstance(usage, dict) else {}
    if isinstance(bridge, dict):
        _merge_counts(tool_counts, bridge.get("tool_call_counts"))
        _merge_counts(llm_tool_counts, bridge.get("llm_tool_call_counts"))
    for stage in _usage_stages(usage).values():
        if not isinstance(stage, dict):
            continue
        stage_bridge = stage.get("bridge") or {}
        if isinstance(stage_bridge, dict):
            _merge_counts(tool_counts, stage_bridge.get("tool_call_counts"))
            _merge_counts(
                llm_tool_counts,
                stage_bridge.get("llm_tool_call_counts"),
            )
    return tool_counts, llm_tool_counts


def _extract_stage_bridge_counts(
    usage: dict,
) -> tuple[dict[str, dict[str, int]], dict[str, dict[str, int]]]:
    stage_tool_counts: dict[str, dict[str, int]] = {}
    stage_llm_tool_counts: dict[str, dict[str, int]] = {}
    for stage_name, stage in _usage_stages(usage).items():
        if not isinstance(stage, dict):
            continue
        bridge = stage.get("bridge") or {}
        if not isinstance(bridge, dict):
            continue
        tool_counts: dict[str, int] = {}
        llm_tool_counts: dict[str, int] = {}
        _merge_counts(tool_counts, bridge.get("tool_call_counts"))
        _merge_counts(llm_tool_counts, bridge.get("llm_tool_call_counts"))
        stage_tool_counts[str(stage_name)] = tool_counts
        stage_llm_tool_counts[str(stage_name)] = llm_tool_counts
    return stage_tool_counts, stage_llm_tool_counts


def _first_nonempty(*values: object) -> object:
    for value in values:
        if value not in (None, "", [], {}):
            return value
    return ""


def _extract_run_metrics(run: dict) -> dict:
    usage = _coerce_usage(run)
    usage_stages = _usage_stages(usage)
    attestation_body = _attestation_body(run)
    prompt_tokens = _int_value(usage.get("prompt_tokens"))
    completion_tokens = _int_value(usage.get("completion_tokens"))
    total_tokens = _int_value(usage.get("total_tokens")) or (
        prompt_tokens + completion_tokens
    )
    stages = {
        stage: seconds
        for stage in ("build", "scope", "query", "mediator")
        if (seconds := _stage_seconds(run, stage)) is not None
    }
    tool_counts, llm_tool_counts = _extract_bridge_counts(usage)
    stage_tool_counts, stage_llm_tool_counts = _extract_stage_bridge_counts(usage)
    scope_stage = usage_stages.get("scope") or {}
    query_agent_id = _first_nonempty(
        run.get("query_agent_id"),
        run.get("agent_id"),
        attestation_body.get("query_agent_id"),
    )
    scope_agent_id = _first_nonempty(
        run.get("scope_agent_id"),
        attestation_body.get("scope_agent_id"),
    )
    room_manifest_hash = _first_nonempty(
        run.get("room_manifest_hash"),
        attestation_body.get("room_manifest_hash"),
    )
    return {
        "run_status": run.get("status") or "",
        "billing_status": run.get("billing_status") or "",
        "billing_cost_micro_usd": _int_value(run.get("billing_cost_micro_usd")),
        "room_id": _first_nonempty(run.get("room_id"), attestation_body.get("room_id")),
        "room_manifest_hash": room_manifest_hash,
        "query_agent_id": query_agent_id,
        "scope_agent_id": scope_agent_id,
        "scope_mode": _first_nonempty(
            run.get("scope_mode"),
            attestation_body.get("scope_mode"),
            scope_stage.get("scope_mode") if isinstance(scope_stage, dict) else "",
        ),
        "scope_mode_reason": _first_nonempty(
            run.get("scope_mode_reason"),
            attestation_body.get("scope_mode_reason"),
            scope_stage.get("scope_mode_reason") if isinstance(scope_stage, dict) else "",
        ),
        "query_inspection_mode": _first_nonempty(
            run.get("query_inspection_mode"),
            attestation_body.get("query_inspection_mode"),
            scope_stage.get("query_inspection_mode")
            if isinstance(scope_stage, dict)
            else "",
        ),
        "output_visibility": _first_nonempty(
            run.get("output_visibility"),
            attestation_body.get("output_visibility"),
        ),
        "artifacts_enabled": _first_nonempty(
            run.get("artifacts_enabled"),
            attestation_body.get("artifacts_enabled"),
        ),
        "room_vault_item_count": _first_nonempty(
            run.get("room_vault_item_count"),
            attestation_body.get("room_vault_item_count"),
        ),
        "attestation_present": bool(run.get("attestation")),
        "attestation_body_fields": sorted(str(key) for key in attestation_body),
        "llm_calls": _int_value(usage.get("calls") or usage.get("llm_calls")),
        "prompt_tokens": prompt_tokens,
        "completion_tokens": completion_tokens,
        "total_tokens": total_tokens,
        "duration_seconds": _run_seconds(run),
        "stage_seconds": stages,
        "tool_call_counts": tool_counts,
        "llm_tool_call_counts": llm_tool_counts,
        "stage_tool_call_counts": stage_tool_counts,
        "stage_llm_tool_call_counts": stage_llm_tool_counts,
        "telemetry_artifact_count": len(run.get("artifacts") or []),
    }


def _artifact_name(artifact: object) -> str:
    if isinstance(artifact, dict):
        for key in ("filename", "path", "url"):
            value = artifact.get(key)
            if value:
                return str(value)
    return str(artifact or "")


def _artifact_extensions(artifacts: list[object]) -> set[str]:
    extensions: set[str] = set()
    for artifact in artifacts:
        name = _artifact_name(artifact).split("?", 1)[0].rstrip("/")
        suffix = Path(name).suffix.lower()
        if suffix:
            extensions.add(suffix)
    return extensions


def _artifact_findings(
    artifacts: list[object],
    scenario: Scenario,
) -> list[GradeFinding]:
    if not scenario.required_artifact_extensions:
        return []
    found = _artifact_extensions(artifacts)
    findings: list[GradeFinding] = []
    for ext in scenario.required_artifact_extensions:
        expected = ext.lower() if ext.startswith(".") else f".{ext.lower()}"
        if expected not in found:
            findings.append(
                GradeFinding(
                    kind="required_artifact_missing",
                    pattern=expected,
                    message=(
                        f"Run did not expose a required {expected} artifact."
                    ),
                    matched_text=",".join(sorted(found)),
                    dimension="artifact",
                    severity="fail",
                )
            )
    return findings


def _redact_process_text(text: str) -> str:
    redacted = text
    for pattern in (
        r"hmk_[A-Za-z0-9_-]+",
        r"phak_[A-Za-z0-9_-]+",
        r"Bearer\s+[A-Za-z0-9._~+/=-]+",
    ):
        redacted = re.sub(pattern, "<redacted>", redacted)
    return redacted.strip()[-2000:]


def _command_failure_findings(
    *,
    phase: str,
    returncode: int,
    stderr: str,
) -> list[GradeFinding]:
    if returncode == 0:
        return []
    excerpt = _redact_process_text(stderr)
    return [
        GradeFinding(
            kind=f"{phase}_command_failed",
            pattern=f"returncode={returncode}",
            message=f"hmctl {phase.replace('_', ' ')} command failed.",
            matched_text=excerpt,
            dimension="system",
            severity="fail",
        )
    ]


def _latency_findings(metrics: dict, scenario: Scenario) -> list[GradeFinding]:
    findings: list[GradeFinding] = []
    if scenario.max_duration_seconds is not None:
        actual = metrics.get("duration_seconds")
        if actual is None:
            findings.append(
                GradeFinding(
                    kind="latency_missing",
                    pattern="duration_seconds",
                    message="Run telemetry did not include total duration.",
                    dimension="performance",
                    severity="fail",
                )
            )
        elif float(actual) > scenario.max_duration_seconds:
            findings.append(
                GradeFinding(
                    kind="latency_over_budget",
                    pattern=f"duration_seconds<={scenario.max_duration_seconds:g}",
                    message="Run exceeded the scenario total latency budget.",
                    matched_text=str(actual),
                    dimension="performance",
                    severity="fail",
                )
            )
    stages = metrics.get("stage_seconds") or {}
    if not isinstance(stages, dict):
        stages = {}
    for stage, budget in scenario.max_stage_seconds.items():
        actual = stages.get(stage)
        if actual is None:
            findings.append(
                GradeFinding(
                    kind="latency_missing",
                    pattern=f"stage_seconds.{stage}",
                    message=f"Run telemetry did not include {stage} duration.",
                    dimension="performance",
                    severity="fail",
                )
            )
        elif float(actual) > budget:
            findings.append(
                GradeFinding(
                    kind="latency_over_budget",
                    pattern=f"stage_seconds.{stage}<={budget:g}",
                    message=f"{stage} stage exceeded the scenario latency budget.",
                    matched_text=str(actual),
                    dimension="performance",
                    severity="fail",
                )
            )
    return findings


def _rubric_from_findings(findings: list[GradeFinding]) -> list[RubricCheck]:
    return [
        RubricCheck(
            dimension=finding.dimension,
            severity=finding.severity,
            score=finding.score,
            passed=False,
            kind=finding.kind,
            message=finding.message,
            evidence=finding.matched_text,
            pattern=finding.pattern,
        )
        for finding in findings
    ]


def _artifact_rubric(
    artifacts: list[object],
    scenario: Scenario,
    findings: list[GradeFinding],
) -> list[RubricCheck]:
    if not scenario.required_artifact_extensions:
        return []
    if findings:
        return _rubric_from_findings(findings)
    return [
        RubricCheck(
            dimension="artifact",
            severity="pass",
            score=4,
            passed=True,
            kind="required_artifacts",
            message="Run exposed all required artifact extensions.",
            evidence=",".join(sorted(_artifact_extensions(artifacts))),
            pattern=";".join(scenario.required_artifact_extensions),
        )
    ]


def _latency_rubric(
    metrics: dict,
    scenario: Scenario,
    findings: list[GradeFinding],
) -> list[RubricCheck]:
    has_budget = (
        scenario.max_duration_seconds is not None
        or bool(scenario.max_stage_seconds)
    )
    if not has_budget:
        return []
    if findings:
        return _rubric_from_findings(findings)
    stage_seconds = metrics.get("stage_seconds")
    stage_evidence = json.dumps(stage_seconds, sort_keys=True) if stage_seconds else "{}"
    return [
        RubricCheck(
            dimension="performance",
            severity="pass",
            score=4,
            passed=True,
            kind="latency_budget",
            message="Run stayed within scenario latency budgets.",
            evidence=(
                f"duration_seconds={metrics.get('duration_seconds')}; "
                f"stage_seconds={stage_evidence}"
            ),
            pattern=(
                f"duration_seconds<={scenario.max_duration_seconds}; "
                f"stage_seconds={scenario.max_stage_seconds}"
            ),
        )
    ]


def _stage_dimension(stage: str) -> str:
    return _STAGE_DIMENSIONS.get(stage, "system")


def _stage_tool_count(metrics: dict, stage: str, tool: str) -> int:
    stage_tools = metrics.get("stage_tool_call_counts") or {}
    stage_llm_tools = metrics.get("stage_llm_tool_call_counts") or {}
    tool_counts = stage_tools.get(stage) if isinstance(stage_tools, dict) else {}
    llm_tool_counts = (
        stage_llm_tools.get(stage) if isinstance(stage_llm_tools, dict) else {}
    )
    count = 0
    if isinstance(tool_counts, dict):
        count = max(count, _int_value(tool_counts.get(tool)))
    if isinstance(llm_tool_counts, dict):
        count = max(count, _int_value(llm_tool_counts.get(tool)))
    return count


def _has_observable_value(value: object) -> bool:
    if value is None:
        return False
    if value is False:
        return False
    if value == "":
        return False
    if value == {}:
        return False
    if value == []:
        return False
    if isinstance(value, (int, float)) and value == 0:
        return False
    return True


def _contract_check(
    *,
    dimension: str,
    kind: str,
    passed: bool,
    message: str,
    evidence: str = "",
    pattern: str = "",
    severity: str = "fail",
) -> tuple[GradeFinding | None, RubricCheck]:
    if passed:
        return None, RubricCheck(
            dimension=dimension,
            severity="pass",
            score=4,
            passed=True,
            kind=kind,
            message=message,
            evidence=evidence,
            pattern=pattern,
        )
    finding = GradeFinding(
        kind=kind,
        pattern=pattern,
        message=message,
        matched_text=evidence,
        dimension=dimension,
        severity=severity,
    )
    return finding, RubricCheck(
        dimension=dimension,
        severity=severity,
        score=0,
        passed=False,
        kind=kind,
        message=message,
        evidence=evidence,
        pattern=pattern,
    )


def _runtime_contract_grade(
    metrics: dict,
    scenario: Scenario,
) -> tuple[list[GradeFinding], list[RubricCheck]]:
    findings: list[GradeFinding] = []
    rubric: list[RubricCheck] = []

    def add(
        *,
        dimension: str,
        kind: str,
        passed: bool,
        message: str,
        evidence: str = "",
        pattern: str = "",
        severity: str = "fail",
    ) -> None:
        finding, check = _contract_check(
            dimension=dimension,
            kind=kind,
            passed=passed,
            message=message,
            evidence=evidence,
            pattern=pattern,
            severity=severity,
        )
        if finding is not None:
            findings.append(finding)
        rubric.append(check)

    for stage in scenario.required_runtime_stages:
        seconds = (metrics.get("stage_seconds") or {}).get(stage)
        add(
            dimension=_stage_dimension(stage),
            kind="runtime_stage_present",
            passed=seconds is not None,
            message=f"{stage} stage telemetry is present.",
            evidence="" if seconds is None else str(seconds),
            pattern=stage,
        )

    if scenario.expected_scope_mode:
        actual = str(metrics.get("scope_mode") or "")
        add(
            dimension="scope_agent",
            kind="scope_mode",
            passed=actual == scenario.expected_scope_mode,
            message="Scope agent used the expected routing mode.",
            evidence=actual,
            pattern=scenario.expected_scope_mode,
        )

    if scenario.expected_query_inspection_mode:
        actual = str(metrics.get("query_inspection_mode") or "")
        add(
            dimension="scope_agent",
            kind="query_inspection_mode",
            passed=actual == scenario.expected_query_inspection_mode,
            message="Scope agent exposed the expected query inspection mode.",
            evidence=actual,
            pattern=scenario.expected_query_inspection_mode,
        )

    for stage, tools in scenario.required_stage_tools.items():
        for tool in tools:
            count = _stage_tool_count(metrics, stage, tool)
            add(
                dimension=_stage_dimension(stage),
                kind="required_stage_tool",
                passed=count > 0,
                message=f"{stage} stage used required tool {tool}.",
                evidence=str(count),
                pattern=f"{stage}.{tool}",
            )

    for stage, tools in scenario.forbidden_stage_tools.items():
        for tool in tools:
            count = _stage_tool_count(metrics, stage, tool)
            add(
                dimension=_stage_dimension(stage),
                kind="forbidden_stage_tool",
                passed=count == 0,
                message=f"{stage} stage avoided forbidden tool {tool}.",
                evidence=str(count),
                pattern=f"{stage}.{tool}",
            )

    for field in scenario.required_observability_fields:
        value = metrics.get(field)
        add(
            dimension="observability",
            kind="observability_field",
            passed=_has_observable_value(value),
            message=f"Run telemetry exposed {field}.",
            evidence=json.dumps(value, sort_keys=True)
            if isinstance(value, (dict, list))
            else str(value or ""),
            pattern=field,
        )

    attestation_fields = set(metrics.get("attestation_body_fields") or [])
    for field in scenario.required_attestation_fields:
        add(
            dimension="attestation",
            kind="attestation_field",
            passed=field in attestation_fields,
            message=f"Run attestation body includes {field}.",
            evidence=",".join(sorted(attestation_fields)),
            pattern=field,
            severity="critical",
        )

    return findings, rubric


def _cmd_run_room(args: argparse.Namespace) -> int:
    scenario = SCENARIOS[args.scenario]
    out_dir = Path(args.output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    rows = []
    exit_code = 0

    for model in args.model:
        safe_model = model.replace("/", "_").replace(":", "_")
        run_path = out_dir / f"{scenario.id}__{safe_model}.json"
        cmd = [
            *_hmctl_base(args.hmctl_profile),
            "--yes",
            "--allow-degraded-attestation",
            "room",
            "ask",
            args.room,
            scenario.query,
            "--provider",
            args.provider,
            "--scope-model",
            model,
            "--query-model",
            model,
            "--mediator-model",
            model,
            "--max-tokens",
            str(args.max_tokens),
            "--max-llm-calls",
            str(args.max_llm_calls),
            "--timeout",
            str(args.timeout),
            "--json",
        ]
        if args.fetch:
            cmd.append("--fetch")
        proc = subprocess.run(
            cmd,
            text=True,
            capture_output=True,
            timeout=args.timeout + 120,
        )
        run_path.write_text(proc.stdout or "", encoding="utf-8")
        stderr_path = run_path.with_suffix(".stderr.txt")
        stderr_path.write_text(proc.stderr or "", encoding="utf-8")
        command_findings = _command_failure_findings(
            phase="room_ask",
            returncode=proc.returncode,
            stderr=proc.stderr or "",
        )

        output = ""
        run_id = ""
        artifacts = []
        telemetry = {}
        telemetry_path = None
        telemetry_stderr_path = None
        if proc.stdout.strip():
            try:
                data = json.loads(proc.stdout)
                output = data.get("output") or ""
                run_id = data.get("run_id") or ""
                artifacts = data.get("artifacts") or []
            except json.JSONDecodeError:
                exit_code = 1
        if run_id:
            telemetry_path = out_dir / f"{scenario.id}__{safe_model}__run.json"
            telemetry_stderr_path = telemetry_path.with_suffix(".stderr.txt")
            telemetry_proc = _fetch_hmroom_run_telemetry(
                args.room,
                run_id,
                args.hmctl_profile,
            )
            if telemetry_proc is None:
                telemetry_cmd = [
                    *_hmctl_base(args.hmctl_profile),
                    "--yes",
                    "--allow-degraded-attestation",
                    "room",
                    "runs",
                    run_id,
                    "--json",
                ]
                telemetry_proc = subprocess.run(
                    telemetry_cmd,
                    text=True,
                    capture_output=True,
                    timeout=60,
                )
            command_findings.extend(
                _command_failure_findings(
                    phase="run_telemetry",
                    returncode=telemetry_proc.returncode,
                    stderr=telemetry_proc.stderr or "",
                )
            )
            telemetry_path.write_text(
                telemetry_proc.stdout or "", encoding="utf-8"
            )
            telemetry_stderr_path.write_text(
                telemetry_proc.stderr or "", encoding="utf-8"
            )
            if telemetry_proc.stdout.strip():
                try:
                    parsed = json.loads(telemetry_proc.stdout)
                    if isinstance(parsed, dict):
                        telemetry = parsed
                        artifacts = telemetry.get("artifacts") or artifacts
                except json.JSONDecodeError:
                    exit_code = 1
            if telemetry_proc.returncode != 0:
                exit_code = 1
        metrics = _extract_run_metrics(telemetry)
        text_grade = grade_text(output, scenario)
        artifact_findings = _artifact_findings(artifacts, scenario)
        latency_findings = _latency_findings(metrics, scenario)
        contract_findings, contract_rubric = _runtime_contract_grade(
            metrics,
            scenario,
        )
        findings = (
            command_findings
            or [
                *text_grade.findings,
                *artifact_findings,
                *latency_findings,
                *contract_findings,
            ]
        )
        rubric = (
            _rubric_from_findings(command_findings)
            if command_findings
            else [
                *text_grade.rubric,
                *_artifact_rubric(artifacts, scenario, artifact_findings),
                *_latency_rubric(metrics, scenario, latency_findings),
                *contract_rubric,
            ]
        )
        passed = not findings
        row = {
            "scenario": scenario.id,
            "model": model,
            "returncode": proc.returncode,
            "run_id": run_id,
            "passed": passed,
            "findings": [asdict(finding) for finding in findings],
            "rubric": [asdict(check) for check in rubric],
            "output_chars": len(output),
            "output_words": count_words(output),
            "markdown_table_count": count_markdown_tables(output),
            "artifact_count": len(artifacts),
            "artifact_filenames": [_artifact_name(a) for a in artifacts],
            "stdout_path": str(run_path),
            "stderr_path": str(stderr_path),
            "telemetry_path": str(telemetry_path) if telemetry_path else "",
            "telemetry_stderr_path": (
                str(telemetry_stderr_path) if telemetry_stderr_path else ""
            ),
            **metrics,
        }
        rows.append(row)
        print(json.dumps(row, sort_keys=True))
        if proc.returncode != 0 or not passed:
            exit_code = 1

    summary_path = out_dir / f"{scenario.id}__summary.json"
    summary_path.write_text(json.dumps(rows, indent=2, sort_keys=True), encoding="utf-8")
    return exit_code


def _dimension_status(row: dict, dimension: str) -> str:
    checks = [
        check
        for check in (row.get("rubric") or [])
        if check.get("dimension") == dimension
    ]
    if not checks:
        return "-"
    critical = sum(
        1
        for check in checks
        if not check.get("passed") and check.get("severity") == "critical"
    )
    failed = sum(
        1
        for check in checks
        if not check.get("passed") and check.get("severity") != "critical"
    )
    if critical:
        return f"critical:{critical}"
    if failed:
        return f"fail:{failed}"
    return "pass"


def _format_summary_table(rows: list[dict]) -> str:
    dimensions = (
        "privacy",
        "utility",
        "scope_agent",
        "query_agent",
        "mediator",
        "artifact",
        "attestation",
        "performance",
        "observability",
        "system",
    )
    header = [
        "scenario",
        "model",
        "run_id",
        "passed",
        "duration_s",
        *dimensions,
    ]
    lines = [
        "| " + " | ".join(header) + " |",
        "| " + " | ".join("---" for _ in header) + " |",
    ]
    for row in rows:
        values = [
            str(row.get("scenario") or ""),
            str(row.get("model") or ""),
            str(row.get("run_id") or ""),
            "yes" if row.get("passed") else "no",
            "" if row.get("duration_seconds") is None else str(row["duration_seconds"]),
            *(_dimension_status(row, dimension) for dimension in dimensions),
        ]
        lines.append("| " + " | ".join(values) + " |")
    return "\n".join(lines)


def _cmd_summarize(args: argparse.Namespace) -> int:
    rows: list[dict] = []
    for path in args.summary:
        loaded = json.loads(Path(path).read_text(encoding="utf-8"))
        if isinstance(loaded, list):
            rows.extend(row for row in loaded if isinstance(row, dict))
        elif isinstance(loaded, dict):
            rows.append(loaded)
    print(_format_summary_table(rows))
    return 0 if all(row.get("passed") for row in rows) else 1


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        prog="python -m eval",
        description="Deterministic eval utilities for hivemind room agents.",
    )
    sub = parser.add_subparsers(dest="command", required=True)

    sub.add_parser("list", help="List available deterministic scenarios.")

    summarize = sub.add_parser(
        "summarize",
        help="Print a compact Markdown table for one or more summary JSON files.",
    )
    summarize.add_argument("summary", nargs="+", help="Path to run-room summary JSON.")

    grade = sub.add_parser("grade", help="Grade an output file or stdin.")
    grade.add_argument("scenario", choices=sorted(SCENARIOS))
    grade.add_argument(
        "output",
        help="Path to output text, or '-' to read from stdin.",
    )

    run_room = sub.add_parser(
        "run-room",
        help="Run one scenario against a room across one or more models.",
    )
    run_room.add_argument("scenario", choices=sorted(SCENARIOS))
    run_room.add_argument("room", help="Room id or invite accepted by hmctl room ask.")
    run_room.add_argument(
        "--model",
        action="append",
        required=True,
        help="Model id to test. Repeat for a model matrix.",
    )
    run_room.add_argument("--provider", default="openrouter")
    run_room.add_argument("--max-tokens", type=int, default=1_000_000)
    run_room.add_argument("--max-llm-calls", type=int, default=60)
    run_room.add_argument("--timeout", type=int, default=900)
    run_room.add_argument("--fetch", action="store_true")
    run_room.add_argument(
        "--hmctl-profile",
        help="Optional hmctl profile to pass before room commands.",
    )
    run_room.add_argument(
        "--output-dir",
        default="eval/results/live",
        help="Directory for raw hmctl JSON, stderr, and summary.",
    )
    return parser


def main(argv: list[str] | None = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    if args.command == "list":
        return _cmd_list()
    if args.command == "grade":
        return _cmd_grade(args)
    if args.command == "run-room":
        return _cmd_run_room(args)
    if args.command == "summarize":
        return _cmd_summarize(args)
    parser.error(f"unknown command: {args.command}")
    return 2