From ed60f0af5f1772f0fef0b313c5265548b89ba736 Mon Sep 17 00:00:00 2001 From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:57:07 +0000 Subject: [PATCH 1/2] feat: add public provider-benchmarks endpoint New GET /api/provider-benchmarks (no auth), returning completed, non-deleted, standalone provider-route evaluation rows (route, benchmark, score, repeat, cost_usd, duration_seconds). Backs the new "Model benchmarks" chart on the public site and mirrors the data already shown in the admin dashboard's provider-tests table. Co-Authored-By: Claude Sonnet 5 --- validator/src/eval_backend/api/routes.py | 39 ++++++++++++++++++++++++ validator/src/eval_backend/schemas.py | 15 +++++++++ 2 files changed, 54 insertions(+) diff --git a/validator/src/eval_backend/api/routes.py b/validator/src/eval_backend/api/routes.py index 950350b..8b1b5fc 100644 --- a/validator/src/eval_backend/api/routes.py +++ b/validator/src/eval_backend/api/routes.py @@ -33,6 +33,8 @@ JobQueueResponse, LeaderboardEntry, LeaderboardResponse, + ProviderBenchmarkEntry, + ProviderBenchmarkResponse, ProviderEvaluationCreateRequest, ProviderEvaluationCreateResponse, ReviewControlOut, @@ -855,6 +857,43 @@ def leaderboard(request: Request, limit: int = 100, include_deleted: bool = Fals session.close() +@router.get("/api/provider-benchmarks", response_model=ProviderBenchmarkResponse) +def provider_benchmarks(request: Request, limit: int = 200) -> ProviderBenchmarkResponse: + session = get_session(request) + try: + stmt = ( + select(EvaluationRun) + .where( + EvaluationRun.submission_id.is_(None), + EvaluationRun.status == "completed", + EvaluationRun.deleted_at.is_(None), + ) + .order_by(EvaluationRun.created_at.desc()) + .limit(max(1, min(limit, 500))) + ) + runs = session.execute(stmt).scalars().all() + items: list[ProviderBenchmarkEntry] = [] + for run in runs: + metrics = json.loads(run.metrics_json) if run.metrics_json else {} + route = metrics.get("provider_route") or metrics.get("pool_model") or "unknown" + repeat = metrics.get("repeat") + items.append( + ProviderBenchmarkEntry( + id=run.id, + route=str(route), + benchmark=run.benchmark, + score=run.score, + repeat=int(repeat) if isinstance(repeat, (int, float)) else None, + cost_usd=run.cost_usd, + duration_seconds=run.duration_seconds, + finished_at=run.finished_at, + ) + ) + return ProviderBenchmarkResponse(items=items) + finally: + session.close() + + @router.get("/api/jobs", response_model=JobQueueResponse) def list_jobs( request: Request, diff --git a/validator/src/eval_backend/schemas.py b/validator/src/eval_backend/schemas.py index d370ab0..66a60e3 100644 --- a/validator/src/eval_backend/schemas.py +++ b/validator/src/eval_backend/schemas.py @@ -156,6 +156,21 @@ class LeaderboardResponse(BaseModel): items: list[LeaderboardEntry] +class ProviderBenchmarkEntry(BaseModel): + id: int + route: str + benchmark: str + score: float | None = None + repeat: int | None = None + cost_usd: float | None = None + duration_seconds: float | None = None + finished_at: datetime | None = None + + +class ProviderBenchmarkResponse(BaseModel): + items: list[ProviderBenchmarkEntry] + + class JobQueueOut(BaseModel): model_config = ConfigDict(from_attributes=True) From 731c2782d02b072bf068511967f19131688e7c0c Mon Sep 17 00:00:00 2001 From: tmimmanuel <14046872+tmimmanuel@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:57:07 +0000 Subject: [PATCH 2/2] feat: add Model benchmarks chart to the leaderboard page Grouped bar chart (score by model, faceted by benchmark), fed by the new /api/provider-benchmarks endpoint. Hand-rolled SVG, no charting dependency added. Uses the dataviz skill's validated dark-mode categorical palette (8 slots, fixed route->color order), verified against this site's dark surface (#060b14) with the palette validator. Mirrors the same chart shipped to the admin dashboard (companion PR in mini-router.github.io) so both surfaces show the same picture. Co-Authored-By: Claude Sonnet 5 --- web/src/components/ProviderBenchmarkChart.tsx | 146 ++++++++++++++++++ .../components/ProviderBenchmarkSection.tsx | 51 ++++++ web/src/hooks/useProviderBenchmarks.ts | 30 ++++ web/src/lib/api.ts | 30 ++++ web/src/pages/Leaderboard.tsx | 2 + 5 files changed, 259 insertions(+) create mode 100644 web/src/components/ProviderBenchmarkChart.tsx create mode 100644 web/src/components/ProviderBenchmarkSection.tsx create mode 100644 web/src/hooks/useProviderBenchmarks.ts diff --git a/web/src/components/ProviderBenchmarkChart.tsx b/web/src/components/ProviderBenchmarkChart.tsx new file mode 100644 index 0000000..c0b3a54 --- /dev/null +++ b/web/src/components/ProviderBenchmarkChart.tsx @@ -0,0 +1,146 @@ +import type { ProviderBenchmarkEntry } from '../lib/api' + +// Grouped bar chart, faceted by benchmark. Hand-rolled SVG, no charting +// library. Mirrors the admin dashboard's chart 1:1 (same palette, same +// layout) so the two surfaces show the same picture -- see +// mini-router.github.io admin/src/components/ProviderBenchmarkChart.tsx. + +// Validated dark-mode categorical palette (8 slots, fixed order -- never +// reassigned by rank/filter). Passes CVD + contrast checks against this +// site's dark surface (#060b14). +const ROUTE_PALETTE = [ + '#3987e5', // blue + '#d95926', // orange + '#199e70', // aqua + '#c98500', // yellow + '#d55181', // magenta + '#008300', // green + '#9085e9', // violet + '#e66767', // red +] + +function buildRouteColors(points: ProviderBenchmarkEntry[]): Map { + const routes = Array.from(new Set(points.map((p) => p.route))).sort() + const map = new Map() + routes.forEach((route, idx) => { + map.set(route, ROUTE_PALETTE[idx % ROUTE_PALETTE.length]) + }) + return map +} + +function fmtPercent(value: number | null): string { + return value == null || Number.isNaN(value) ? '—' : `${(value * 100).toFixed(1)}%` +} + +const FACET_WIDTH = 420 +const FACET_HEIGHT = 220 +const PLOT_TOP = 16 +const PLOT_BOTTOM = 36 +const BAR_GAP = 6 + +export default function ProviderBenchmarkChart({ points }: { points: ProviderBenchmarkEntry[] }) { + const scored = points.filter((p) => p.score != null) + if (!scored.length) { + return ( +
+ No provider benchmark results yet. +
+ ) + } + + const routeColors = buildRouteColors(scored) + const benchmarks = Array.from(new Set(scored.map((p) => p.benchmark))).sort() + + return ( +
+
+ {Array.from(routeColors.entries()).map(([route, color]) => ( +
+ + {route} +
+ ))} +
+
+ {benchmarks.map((benchmark) => { + const rows = scored + .filter((p) => p.benchmark === benchmark) + .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + const plotHeight = FACET_HEIGHT - PLOT_TOP - PLOT_BOTTOM + const barWidth = Math.max(28, (FACET_WIDTH - BAR_GAP * (rows.length - 1)) / rows.length - 4) + + return ( +
+
+ {benchmark} +
+ + {[0, 0.25, 0.5, 0.75, 1].map((frac) => { + const y = PLOT_TOP + plotHeight * (1 - frac) + return ( + + ) + })} + {rows.map((row, idx) => { + const score = row.score ?? 0 + const barHeight = plotHeight * Math.max(0, Math.min(1, score)) + const x = idx * (barWidth + BAR_GAP) + const y = PLOT_TOP + (plotHeight - barHeight) + const color = routeColors.get(row.route) ?? ROUTE_PALETTE[0] + return ( + + + {row.route} — {benchmark}: {fmtPercent(row.score)} + + + + {fmtPercent(row.score)} + + + {row.route.length > 12 ? `${row.route.slice(0, 11)}…` : row.route} + + + ) + })} + + +
+ ) + })} +
+
+ ) +} diff --git a/web/src/components/ProviderBenchmarkSection.tsx b/web/src/components/ProviderBenchmarkSection.tsx new file mode 100644 index 0000000..4540f26 --- /dev/null +++ b/web/src/components/ProviderBenchmarkSection.tsx @@ -0,0 +1,51 @@ +import { motion } from 'framer-motion' +import { useProviderBenchmarks } from '../hooks/useProviderBenchmarks' +import ProviderBenchmarkChart from './ProviderBenchmarkChart' + +export default function ProviderBenchmarkSection() { + const { entries, error } = useProviderBenchmarks() + + return ( +
+
+ + Provider tests + + + Model benchmarks + + + Direct single-route benchmark results for each upstream model, independent of any miner + submission. + +
+ + {error ? ( +
+ {error} +
+ ) : ( + + )} +
+
+ ) +} diff --git a/web/src/hooks/useProviderBenchmarks.ts b/web/src/hooks/useProviderBenchmarks.ts new file mode 100644 index 0000000..5911bfb --- /dev/null +++ b/web/src/hooks/useProviderBenchmarks.ts @@ -0,0 +1,30 @@ +import { useEffect, useState } from 'react' +import { fetchProviderBenchmarks } from '../lib/api' +import type { ProviderBenchmarkEntry } from '../lib/api' + +export function useProviderBenchmarks(limit = 200) { + const [entries, setEntries] = useState([]) + const [error, setError] = useState(null) + + useEffect(() => { + let active = true + + fetchProviderBenchmarks(limit) + .then((nextEntries) => { + if (!active) return + setEntries(nextEntries) + setError(null) + }) + .catch((err: unknown) => { + if (!active) return + setEntries([]) + setError(err instanceof Error ? err.message : 'Failed to load provider benchmarks') + }) + + return () => { + active = false + } + }, [limit]) + + return { entries, error } +} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index ca945da..ead59c6 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -163,6 +163,36 @@ export interface BackendSubmissionOut { trains: BackendTrainOut[] } +export interface ProviderBenchmarkEntry { + id: number + route: string + benchmark: string + score: number | null + repeat: number | null + cost_usd: number | null + duration_seconds: number | null + finished_at: string | null +} + +interface BackendProviderBenchmarkResponse { + items: ProviderBenchmarkEntry[] +} + +export async function fetchProviderBenchmarks(limit = 200): Promise { + const response = await fetch(apiUrl(`/api/provider-benchmarks?limit=${limit}`), { + headers: { + Accept: 'application/json', + }, + }) + + if (!response.ok) { + throw new Error(`Provider benchmark request failed with status ${response.status}`) + } + + const payload = (await response.json()) as BackendProviderBenchmarkResponse + return payload.items +} + export async function fetchSubmission(submissionId: string): Promise { const response = await fetch(apiUrl(`/api/submissions/${submissionId}`), { headers: { diff --git a/web/src/pages/Leaderboard.tsx b/web/src/pages/Leaderboard.tsx index a04cf28..6fb93b9 100644 --- a/web/src/pages/Leaderboard.tsx +++ b/web/src/pages/Leaderboard.tsx @@ -1,5 +1,6 @@ import PageHeader from '../components/PageHeader' import LeaderboardTable from '../components/LeaderboardTable' +import ProviderBenchmarkSection from '../components/ProviderBenchmarkSection' export default function Leaderboard() { return ( @@ -11,6 +12,7 @@ export default function Leaderboard() {
+ ) }