diff --git a/admin/src/App.tsx b/admin/src/App.tsx index 00aca53..31e9e5f 100644 --- a/admin/src/App.tsx +++ b/admin/src/App.tsx @@ -69,6 +69,7 @@ import type { AdminReviewControl, } from './lib/api' import type { LeaderboardEntry } from './types' +import { CostVsScoreChart, ScoreByModelChart, type ProviderBenchmarkPoint } from './components/ProviderBenchmarkChart' type StatusTone = 'ok' | 'warn' | 'bad' | 'idle' @@ -1185,6 +1186,21 @@ function App() { } }, [leaderboard]) + const providerBenchmarkPoints: ProviderBenchmarkPoint[] = useMemo( + () => + providerEvalRuns + .filter((run) => !run.deleted_at) + .map((run) => ({ + id: run.id, + route: metricText(run.metrics, 'provider_route'), + benchmark: run.benchmark_names.join(', ') || metricText(run.metrics, 'benchmark'), + score: run.score, + cost_usd: run.cost_usd, + duration_seconds: run.duration_seconds, + })), + [providerEvalRuns], + ) + if (isCheckingSession) { return (
@@ -1508,6 +1524,22 @@ function App() {
+
+ +
+ +
+ +
+
{ + const routes = Array.from(new Set(points.map((p) => p.route))).sort() + const map = new Map() + routes.forEach((route, idx) => { + map.set(route, ROUTE_PALETTE[idx % ROUTE_PALETTE.length]) + }) + return map +} + +function fmtPercent(value: number | null): string { + return value == null || Number.isNaN(value) ? '—' : `${(value * 100).toFixed(1)}%` +} + +function fmtCost(value: number | null): string { + return value == null || Number.isNaN(value) ? '—' : `$${value.toFixed(3)}` +} + +const FACET_WIDTH = 420 +const FACET_HEIGHT = 220 +const PLOT_TOP = 16 +const PLOT_BOTTOM = 36 +const BAR_GAP = 6 + +export function ScoreByModelChart({ points }: { points: ProviderBenchmarkPoint[] }) { + const scored = points.filter((p) => p.score != null) + if (!scored.length) { + return ( +
+ No scored provider evaluations yet. +
+ ) + } + + const routeColors = buildRouteColors(scored) + const benchmarks = Array.from(new Set(scored.map((p) => p.benchmark))).sort() + + return ( +
+
+ {Array.from(routeColors.entries()).map(([route, color]) => ( +
+ + {route} +
+ ))} +
+
+ {benchmarks.map((benchmark) => { + const rows = scored + .filter((p) => p.benchmark === benchmark) + .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + const plotHeight = FACET_HEIGHT - PLOT_TOP - PLOT_BOTTOM + const barWidth = Math.max(28, (FACET_WIDTH - BAR_GAP * (rows.length - 1)) / rows.length - 4) + + return ( +
+
+ {benchmark} +
+ + {[0, 0.25, 0.5, 0.75, 1].map((frac) => { + const y = PLOT_TOP + plotHeight * (1 - frac) + return ( + + ) + })} + {rows.map((row, idx) => { + const score = row.score ?? 0 + const barHeight = plotHeight * Math.max(0, Math.min(1, score)) + const x = idx * (barWidth + BAR_GAP) + const y = PLOT_TOP + (plotHeight - barHeight) + const color = routeColors.get(row.route) ?? ROUTE_PALETTE[0] + return ( + + + {row.route} — {benchmark}: {fmtPercent(row.score)} (eval #{row.id}) + + + + {fmtPercent(row.score)} + + + {row.route.length > 12 ? `${row.route.slice(0, 11)}…` : row.route} + + + ) + })} + + +
+ ) + })} +
+
+ ) +} + +const SCATTER_WIDTH = 640 +const SCATTER_HEIGHT = 320 +const SCATTER_PAD_LEFT = 48 +const SCATTER_PAD_BOTTOM = 32 +const SCATTER_PAD_TOP = 16 +const SCATTER_PAD_RIGHT = 16 + +// Cost-vs-score is a scatter (an all-pairs form), which caps categorical +// color at 3 series -- with up to 7+ models this would blow that cap, so +// identity here comes from direct text labels + marker shape (per +// benchmark), not from a per-model hue. +export function CostVsScoreChart({ points }: { points: ProviderBenchmarkPoint[] }) { + const rows = points.filter((p) => p.score != null && p.cost_usd != null) + if (!rows.length) { + return ( +
+ No cost/score data yet. +
+ ) + } + + const benchmarks = Array.from(new Set(rows.map((p) => p.benchmark))).sort() + // Cost typically spans multiple orders of magnitude (a cheap model at + // $0.01 next to an expensive one at $0.78), which crushes everything + // cheap into one unreadable cluster on a linear axis -- log scale spreads + // it out. Floor at a small epsilon so a $0 run still has a position. + const costs = rows.map((p) => Math.max(p.cost_usd ?? 0, 0.001)) + const minLog = Math.log10(Math.min(...costs)) + const maxLog = Math.log10(Math.max(...costs, 0.01)) + const logSpan = Math.max(maxLog - minLog, 0.5) + const plotWidth = SCATTER_WIDTH - SCATTER_PAD_LEFT - SCATTER_PAD_RIGHT + const plotHeight = SCATTER_HEIGHT - SCATTER_PAD_TOP - SCATTER_PAD_BOTTOM + + const xFor = (cost: number) => { + const frac = (Math.log10(Math.max(cost, 0.001)) - minLog) / logSpan + return SCATTER_PAD_LEFT + Math.max(0, Math.min(1, frac)) * plotWidth + } + const yFor = (score: number) => SCATTER_PAD_TOP + plotHeight * (1 - Math.max(0, Math.min(1, score))) + + return ( +
+
+ {benchmarks.map((benchmark, idx) => ( +
+ {idx % 2 === 0 ? ( + + ) : ( + + )} + {benchmark} +
+ ))} +
+
+ + {[0, 0.25, 0.5, 0.75, 1].map((frac) => { + const y = SCATTER_PAD_TOP + plotHeight * (1 - frac) + return ( + + + + {Math.round(frac * 100)}% + + + ) + })} + + {[0, 1 / 3, 2 / 3, 1].map((frac) => { + const cost = 10 ** (minLog + frac * logSpan) + const x = SCATTER_PAD_LEFT + frac * plotWidth + return ( + + {fmtCost(cost)} + + ) + })} + {[...rows] + .sort((a, b) => (a.cost_usd ?? 0) - (b.cost_usd ?? 0)) + .map((row, idx) => { + const cx = xFor(row.cost_usd ?? 0) + const cy = yFor(row.score ?? 0) + const benchmarkIdx = benchmarks.indexOf(row.benchmark) + const labelY = idx % 2 === 0 ? cy - 9 : cy + 15 + return ( + + + {row.route} — {row.benchmark}: {fmtPercent(row.score)} at {fmtCost(row.cost_usd)} + {row.duration_seconds != null ? `, ${row.duration_seconds.toFixed(0)}s` : ''} + + {benchmarkIdx % 2 === 0 ? ( + + ) : ( + + )} + + {row.route} + + + ) + })} + + Cost (USD, log scale) + + +
+
+ ) +}