From 570d2c68442824e85ef4a7e5ae5b0f0459bd594e Mon Sep 17 00:00:00 2001 From: ernani Date: Sun, 26 Jul 2026 17:39:41 -0300 Subject: [PATCH] =?UTF-8?q?feat(report):=20gecko=20report=20=E2=80=94=20th?= =?UTF-8?q?e=20Agent-Readiness=20Scorecard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained HTML scorecard (inline CSS + inline SVG, no server, no deps): header grade+score, dimension bars, embedded call-graph SVG, ranked fixable findings checklist, a scripted Playground (deterministic intent -> derived tool -> first-call-correct call, no live LLM/network), and a correlation teaser. Deterministic and control-plane clean (structure/scores only, never a payload or secret). Adds InspectionReport.to_dict()+surface_rev and report_diff drift. CLI: gecko report [-o] [--intent]... [--since ]. Co-Authored-By: Claude Opus 4.8 --- gecko/cli.py | 69 +++++++ gecko/inspect.py | 36 +++- gecko/report.py | 441 +++++++++++++++++++++++++++++++++++++++++++ tests/test_report.py | 102 ++++++++++ 4 files changed, 647 insertions(+), 1 deletion(-) create mode 100644 gecko/report.py create mode 100644 tests/test_report.py diff --git a/gecko/cli.py b/gecko/cli.py index 9b08e90..1c036dd 100644 --- a/gecko/cli.py +++ b/gecko/cli.py @@ -37,6 +37,7 @@ ) from .access import public_session, stub_session from .client import AgentApiClient +from .ingest import load_spec from .modes import coerce_mode from .netguard import UnsafeUrlError, validate_public_url @@ -48,6 +49,7 @@ "serve", "test", "inspect", + "report", "from-docs", "scan-image", "scan-doc", @@ -286,6 +288,71 @@ def _cmd_inspect(argv: list[str]) -> int: return 1 if (below or inspect_mod.has_blocking(report)) else 0 +def _cmd_report(argv: list[str]) -> int: + """`gecko report ` — the Agent-Readiness Scorecard (offline, $0). + + Thin transport over :mod:`gecko.report`. Without `--since`, write the self-contained + HTML scorecard (plus a sidecar JSON for later diffs). With `--since `, + print the drift delta (score change + broken/resolved call-paths). + """ + from . import inspect as inspect_mod + from . import report as report_mod + + p = argparse.ArgumentParser( + prog="gecko report", + description="Build a self-contained agent-readiness scorecard (HTML) for an API.", + ) + p.add_argument( + "spec", + help="An API domain, OpenAPI URL, docs URL, or local path — Gecko finds the spec.", + ) + p.add_argument( + "-o", + "--out", + default=None, + help="Write the HTML here (default: .scorecard.html).", + ) + p.add_argument( + "--intent", + action="append", + default=None, + help="A plain-English intent to script the Playground (repeatable).", + ) + p.add_argument( + "--since", + default=None, + help="A prior scorecard JSON — print the drift delta instead of writing HTML.", + ) + args = p.parse_args(argv) + if _reject_unsafe(args.spec, "report"): + return 2 + # Load once, directly — mirrors `gecko graph` (load_spec handles both a local + # yaml/json path and an SSRF-validated URL). The engine does the work. + try: + spec = load_spec(args.spec) + except (UnsafeUrlError, OSError, ValueError) as exc: + print(f" ✗ could not read spec at {args.spec}: {exc}", file=sys.stderr) + return 2 + + name = onboard.safe_name(args.spec) + if args.since: + old = json.loads(Path(args.since).read_text(encoding="utf-8")) + new = inspect_mod.inspect(spec, api=name).to_dict() + print(report_mod.render_diff(report_mod.report_diff(old, new))) + return 0 + + html = report_mod.build_scorecard(spec, intents=args.intent) + out = Path(args.out) if args.out else Path(f"{name}.scorecard.html") + out.write_text(html, encoding="utf-8") + sidecar = out.with_suffix(".json") + sidecar.write_text( + json.dumps(inspect_mod.inspect(spec, api=name).to_dict(), indent=2), + encoding="utf-8", + ) + print(f" → wrote {out} ({len(html)} bytes) and {sidecar}") + return 0 + + def _cmd_test(argv: list[str]) -> int: p = argparse.ArgumentParser( prog="gecko test", @@ -1364,6 +1431,8 @@ def main(argv: list[str] | None = None) -> int: return _cmd_test(rest) if cmd == "inspect": return _cmd_inspect(rest) + if cmd == "report": + return _cmd_report(rest) if cmd == "from-docs": return _cmd_from_docs(rest) if cmd == "scan-image": diff --git a/gecko/inspect.py b/gecko/inspect.py index 2e4a4d6..b9649ac 100644 --- a/gecko/inspect.py +++ b/gecko/inspect.py @@ -62,6 +62,40 @@ class InspectionReport: score: int # 0-100 overall dimensions: list[DimensionResult] summary: str + #: Content hash of the inspected spec (``surfaces.surface_rev``) — lets a stored + #: report be diffed against a later one to catch drift. Defaults empty for callers + #: that build a report without a spec on hand (kept back-compat with positional use). + surface_rev: str = "" + + def to_dict(self) -> dict[str, Any]: + """JSON-serializable form so a report can be persisted and diffed (``report_diff``). + + Control-plane only: structure, scores, and finding metadata — never a payload, + value, or secret.""" + return { + "api": self.api, + "grade": self.grade, + "score": self.score, + "surface_rev": self.surface_rev, + "summary": self.summary, + "dimensions": [ + { + "name": d.name, + "score": d.score, + "findings": [ + { + "dimension": f.dimension, + "severity": f.severity, + "location": f.location, + "message": f.message, + "fix": f.fix, + } + for f in d.findings + ], + } + for d in self.dimensions + ], + } # --------------------------------------------------------------------------- # @@ -279,7 +313,7 @@ def inspect(spec: dict[str, Any], *, api: str) -> InspectionReport: f"{api}: agent-readiness {grade} ({overall}/100) — " f"{n_block} blocking, {n_warn} warnings" ) - return InspectionReport(api, grade, overall, dims, summary) + return InspectionReport(api, grade, overall, dims, summary, client.surface_rev) def has_blocking(report: InspectionReport) -> bool: diff --git a/gecko/report.py b/gecko/report.py new file mode 100644 index 0000000..5aaa378 --- /dev/null +++ b/gecko/report.py @@ -0,0 +1,441 @@ +"""``gecko report`` — the Agent-Readiness Scorecard, a provider leave-behind. + +One self-contained HTML file (inline CSS, inline SVG, no server, no external deps — same +class as :mod:`gecko.surfaceviz`'s SVG and the Skill-Guard artifacts). It is the a-ha, the +surface, and the leave-behind in one file: + + * the big **grade + score** — the number the provider owns + * the four :mod:`gecko.inspect` dimensions as bars + * the **call graph** (``surfaceviz.render_svg``) embedded inline + * the ranked, fixable **findings** as a checklist + * a scripted **Playground** — a chat-style transcript replaying the deterministic + intent → derived tool → first-call-correct call (no live LLM, no real network) + +Deterministic (same spec in → byte-stable HTML out — everything is sorted, no timestamps) +and control-plane clean (structure/scores/finding-names/synthesized call shapes only — +never a response payload, secret, or key). The Playground shows the derived CALL, never a +real response value. +""" + +from __future__ import annotations + +from html import escape +from typing import Any + +from .access import stub_session +from .client import AgentApiClient, ToolNotFound +from .ingest import load_spec +from .inspect import InspectionReport, inspect +from .sample import example_from_schema +from .surfaceviz import graph_data, render_svg + +__all__ = ["build_scorecard", "report_diff", "render_diff"] + +# Severity → (rank for ordering, glyph, css class). Blocking first — it's what breaks a call. +_SEV: dict[str, tuple[int, str, str]] = { + "blocking": (0, "✗", "sev-block"), + "warning": (1, "⚠", "sev-warn"), + "info": (2, "·", "sev-info"), +} +# Grade → accent for the badge (the one place colour carries meaning beyond the accent). +_GRADE_COLOR: dict[str, str] = { + "A": "#0e9f6e", + "B": "#22c55e", + "C": "#d97706", + "D": "#ea580c", + "F": "#dc2626", +} +_MAX_INTENTS = 3 + + +# --------------------------------------------------------------------------- # +# Playground — the deterministic intent → derived-call replay +# --------------------------------------------------------------------------- # +def _auto_intents(client: AgentApiClient) -> list[str]: + """Pick a few plain-English intents from the surface's own tools (deterministic). + + Uses each tool's question-shaped description (already sanitized) as the "user" line, + stable-sorted by tool name so the transcript is byte-identical run to run.""" + intents: list[str] = [] + for tool in sorted(client.list_tools(), key=lambda t: t["name"]): + first = ( + (tool.get("description") or tool["name"]).strip().splitlines()[0].strip() + ) + if first and first not in intents: + intents.append(first[:120]) + if len(intents) >= _MAX_INTENTS: + break + return intents + + +def _derive_call(client: AgentApiClient, intent: str) -> dict[str, Any] | None: + """Replay Gecko's derivation for one intent: search → derived tool → first-call shape. + + Returns the DERIVED CALL (method, path, synthesized param placeholders, optional + supplier-chain plan) — never a real response value (control plane). The param values + are synthesized from the schema, exactly as the $0 recorded path does.""" + hits = client.search(intent, limit=1) + if not hits: + return None + top = hits[0] + name = top["name"] + try: + tool = client.get_tool(name) + except ToolNotFound: + return None + args = example_from_schema(tool.get("inputSchema") or {}) + if not isinstance(args, dict): + args = {} + locations = tool.get("_invoke", {}).get("param_locations", {}) + params = [ + (key, str(locations.get(key, "query")), value) + for key, value in sorted(args.items()) + ] + plan = client.plan_for(intent, name) + return { + "intent": intent, + "tool": name, + "method": top.get("method") or "", + "path": top.get("path") or "", + "params": params, + "plan_steps": (plan or {}).get("steps") if plan else None, + } + + +def _render_playground(client: AgentApiClient, intents: list[str]) -> str: + entries = [c for c in (_derive_call(client, i) for i in intents) if c is not None] + if not entries: + return "" + rows: list[str] = [] + for entry in entries: + params_html = "" + if entry["params"]: + items = "".join( + f'{escape(name)}' + f'{escape(loc)}' + f'{escape(_short(value))}' + for name, loc, value in entry["params"] + ) + params_html = f'{items}
' + plan_html = "" + if entry["plan_steps"]: + steps = " → ".join( + escape(f"{s.get('method', '')} {s.get('path', '')}".strip()) + for s in entry["plan_steps"] + ) + plan_html = f'
supplier chain {steps}
' + rows.append( + '
' + f'
You' + f"{escape(entry['intent'])}
" + '
Gecko' + f'
derived tool {escape(entry["tool"])}
' + f'
{escape(entry["method"])} ' + f"{escape(entry['path'])}
" + f"{params_html}{plan_html}
" + "
" + ) + return ( + '
' + "

The Playground

" + '

Plain-English intent in, the first-call-correct call out — ' + "replayed deterministically from the surface. No live model, no network.

" + f"{''.join(rows)}" + "
" + ) + + +def _short(value: Any, cap: int = 40) -> str: + text = str(value) + return text if len(text) <= cap else text[: cap - 1] + "…" + + +# --------------------------------------------------------------------------- # +# Sections +# --------------------------------------------------------------------------- # +def _render_header(report: InspectionReport) -> str: + color = _GRADE_COLOR.get(report.grade, "#4f46e5") + verdict = ( + f"{report.score}/100 — how often an agent calls " + f"{report.api} right, the first try." + ) + return ( + '
' + '
' + f'
Agent-Readiness Scorecard · {escape(report.api)}
' + f'

{escape(verdict)}

' + '

Your OpenAPI spec is a schema your agent guesses ' + "from. This score is how much it has to guess. Stop letting it guess.

" + "
" + '
' + f'
{escape(report.grade)}
' + f'
{report.score}/100
' + "
" + "
" + ) + + +def _render_dimensions(report: InspectionReport) -> str: + bars: list[str] = [] + for dim in report.dimensions: + bars.append( + '
' + f'
{escape(dim.name)}' + f'{dim.score}
' + f'
' + "
" + ) + return ( + '
' + "

Dimension breakdown

" + '

First-call-correct is weighted 0.4 — it is the promise. ' + "Hygiene, agent-friendliness, and security round it out.

" + f"{''.join(bars)}" + "
" + ) + + +def _render_graph(svg: str) -> str: + return ( + '
' + "

Your API as an agent traverses it

" + '

The derived call graph — operations as nodes, outputs feeding ' + "inputs as arrows, each coloured by provenance.

" + f'
{svg}
' + "
" + ) + + +def _render_findings(report: InspectionReport) -> str: + findings = [f for d in report.dimensions for f in d.findings] + findings.sort(key=lambda f: (_SEV[f.severity][0], f.dimension, f.location)) + if not findings: + return ( + '
' + "

Fixable findings

" + '

Nothing to fix — this surface is clean.

' + "
" + ) + rows: list[str] = [] + for f in findings: + _, glyph, cls = _SEV[f.severity] + rows.append( + f'
  • ' + f'{glyph}' + '
    ' + f'
    {escape(f.dimension)}' + f'{escape(f.location)}
    ' + f'
    {escape(f.message)}
    ' + f'
    → {escape(f.fix)}
    ' + "
    • " + ) + return ( + '
    ' + "

    Fixable findings

    " + '

    Each one is a call an agent gets wrong, and exactly how to fix it. ' + "Improve the score by clearing the list.

    " + f'
      {"".join(rows)}
    ' + "
    " + ) + + +def _render_correlation(gdata: dict[str, Any]) -> str: + if not gdata.get("edges"): + return "" + return ( + '
    ' + "

    Correlation

    " + '

    Your outputs already feed your inputs — the arrows above. ' + "Across a second API, that becomes a provenance-carrying chain your agent can " + "traverse without guessing.

    " + "
    " + ) + + +# --------------------------------------------------------------------------- # +# Public entrypoint +# --------------------------------------------------------------------------- # +def build_scorecard( + spec: str | dict[str, Any], + *, + intents: list[str] | None = None, + base_url: str | None = None, +) -> str: + """Build the full self-contained HTML scorecard for ``spec``. + + Deterministic (same input → byte-stable output) and control-plane clean. ``intents`` + scripts the Playground; when omitted, a few are auto-picked from the surface's tools. + """ + spec_dict = load_spec(spec) if isinstance(spec, str) else spec + api = str((spec_dict.get("info") or {}).get("title") or "API") + + report = inspect(spec_dict, api=api) + client = AgentApiClient(spec_dict, base_url=base_url, session=stub_session()) + graph = client.surface_graph + svg = render_svg(graph, title=f"Agent Surface — {escape(api)}") + gdata = graph_data(graph) + + play_intents = intents if intents else _auto_intents(client) + + body = "".join( + ( + _render_header(report), + _render_dimensions(report), + _render_graph(svg), + _render_findings(report), + _render_playground(client, play_intents), + _render_correlation(gdata), + ) + ) + return _HTML_SHELL.format( + title=escape(f"{api} — Agent-Readiness Scorecard"), body=body + ) + + +# --------------------------------------------------------------------------- # +# Drift-diff — two report dicts in, a delta out +# --------------------------------------------------------------------------- # +def _finding_set(report: dict[str, Any]) -> dict[tuple[str, str, str], dict[str, Any]]: + """Findings keyed by (dimension, location, message) — the stable identity for diffing.""" + out: dict[tuple[str, str, str], dict[str, Any]] = {} + for dim in report.get("dimensions") or []: + for f in dim.get("findings") or []: + out[ + (f.get("dimension", ""), f.get("location", ""), f.get("message", "")) + ] = f + return out + + +def report_diff(old: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]: + """Diff two :meth:`InspectionReport.to_dict` payloads — the drift readout. + + Score delta + added/resolved findings (e.g. "v4 broke 3 agent call-paths"). Pure + structure; no payloads. A negative ``score.delta`` is a regression.""" + old_score = int(old.get("score", 0)) + new_score = int(new.get("score", 0)) + old_f, new_f = _finding_set(old), _finding_set(new) + added = [new_f[k] for k in sorted(new_f.keys() - old_f.keys())] + resolved = [old_f[k] for k in sorted(old_f.keys() - new_f.keys())] + delta = new_score - old_score + verb = "broke" if delta < 0 else ("cleared" if delta > 0 else "changed") + summary = ( + f"{new.get('api', 'API')}: score {old_score} → {new_score} ({delta:+d}); " + f"{len(added)} finding(s) added, {len(resolved)} resolved" + ) + if added: + summary += f" — {verb} {len(added)} agent call-path(s)" + return { + "surface_rev": { + "old": old.get("surface_rev", ""), + "new": new.get("surface_rev", ""), + }, + "grade": {"old": old.get("grade", ""), "new": new.get("grade", "")}, + "score": {"old": old_score, "new": new_score, "delta": delta}, + "added_findings": added, + "resolved_findings": resolved, + "summary": summary, + } + + +def render_diff(diff: dict[str, Any]) -> str: + """A terminal-friendly rendering of :func:`report_diff` (keeps the CLI thin).""" + lines = [f" {diff['summary']}", ""] + for label, key in ( + ("+ added", "added_findings"), + ("- resolved", "resolved_findings"), + ): + for f in diff[key]: + lines.append(f" {label} [{f.get('location', '')}] {f.get('message', '')}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# The shell — a light, professional, single-file readout (system fonts, inline CSS) +# --------------------------------------------------------------------------- # +_HTML_SHELL = """ + + +{title} + +
    +{body} +

    Generated by gecko report — offline, $0, control-plane only. +Structure and scores, never your data.

    +
    +""" diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..95e475d --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,102 @@ +"""Tests for ``gecko report`` — the Agent-Readiness Scorecard (provider leave-behind).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from gecko import report +from gecko.inspect import inspect +from gecko.ingest import load_spec + +FIXTURE = str(Path(__file__).parent / "fixtures" / "txline_openapi.yaml") +# A fixture that produces findings, so the checklist section is exercised. +FINDINGS_FIXTURE = str(Path(__file__).parent / "fixtures" / "pegana_openapi.json") + + +@pytest.fixture(scope="module") +def html() -> str: + return report.build_scorecard(FIXTURE) + + +def test_scorecard_has_grade_score_svg_and_playground(html: str) -> None: + spec = load_spec(FIXTURE) + rep = inspect(spec, api=str(spec["info"]["title"])) + # the number the provider owns + assert str(rep.score) in html + assert rep.grade in html + # the call graph is embedded inline + assert " None: + spec = load_spec(FINDINGS_FIXTURE) + rep = inspect(spec, api=str((spec.get("info") or {}).get("title") or "api")) + a_finding = next(f for d in rep.dimensions for f in d.findings) + html = report.build_scorecard(FINDINGS_FIXTURE) + assert 'class="findings"' in html + assert a_finding.location in html + + +def test_scorecard_is_deterministic(html: str) -> None: + assert report.build_scorecard(FIXTURE) == html + + +def test_scorecard_is_control_plane_clean(html: str) -> None: + lowered = html.lower() + for leak in ("authorization", "bearer ", "x-api-key", "secret", "password"): + assert leak not in lowered, f"control-plane leak: {leak!r} present in scorecard" + + +def test_scorecard_honors_explicit_intents() -> None: + html = report.build_scorecard(FIXTURE, intents=["get a merkle proof for a fixture"]) + assert "get a merkle proof for a fixture" in html.lower() + + +def test_report_diff_flags_a_regression() -> None: + spec = load_spec(FIXTURE) + good = inspect(spec, api="txline").to_dict() + + # Synthesize a lower-scoring "v2" with an extra broken call-path. + worse = dict(good) + worse["score"] = good["score"] - 20 + worse["dimensions"] = [dict(d) for d in good["dimensions"]] + worse["dimensions"][0] = dict(worse["dimensions"][0]) + new_finding = { + "dimension": "first-call-correct", + "severity": "blocking", + "location": "GET /api/new-broken", + "message": "v2 broke this call-path", + "fix": "restore the schema", + } + worse["dimensions"][0]["findings"] = [ + *worse["dimensions"][0]["findings"], + new_finding, + ] + + diff = report.report_diff(good, worse) + assert diff["score"]["delta"] == -20 + assert any(f["location"] == "GET /api/new-broken" for f in diff["added_findings"]) + assert not diff["resolved_findings"] + + +def test_to_dict_includes_surface_rev() -> None: + spec = load_spec(FIXTURE) + d = inspect(spec, api="txline").to_dict() + assert d["surface_rev"] # non-empty content hash + assert isinstance(d["dimensions"], list) + + +def test_cli_report_writes_html(tmp_path: Path) -> None: + from gecko.cli import main + + out = tmp_path / "scorecard.html" + rc = main(["report", FIXTURE, "-o", str(out)]) + assert rc == 0 + assert out.exists() and out.stat().st_size > 0 + assert "