diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8f1d2b8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: ci + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e ".[dev]" + - run: python -m compileall -q bench_loop tests + - run: python -m pytest -q + - if: matrix.python-version == '3.12' + run: python -m pip wheel . --no-deps --wheel-dir dist diff --git a/README.md b/README.md index c1826e7..7acf043 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ **Benchmark local LLMs by what actually matters.** -BenchLoop is a local-first CLI + web app for benchmarking LLMs running on your own hardware or cloud providers. It scores models across seven repeatable suites — quality, speed, reliability, agentic tool use, coding, instruction following — and gives you receipts: per-task outputs, latency, token counts, machine info, scores. +BenchLoop is a local-first CLI + web app for benchmarking LLMs running on your own hardware or cloud providers. It scores models across eight repeatable suites — quality, speed, long-context recall, agentic tool use, coding, instruction following — and gives you receipts: per-task outputs, latency, token counts, machine info, scores. -No accounts, no telemetry. Local models need no API keys; cloud providers use standard OpenAI-compatible auth. Your model, your machine (or your provider), your numbers. +Local runs need no account or API key; cloud providers use standard OpenAI-compatible auth. Publishing is optional (`BENCHLOOP_NO_SUBMIT=1`). Your model, your machine (or your provider), your numbers. ``` $ benchloop run --model qwen3:8b --suites speed,toolcall,agent @@ -76,7 +76,20 @@ benchloop run \ --provider ollama ``` -This runs every default suite, scores them, prints a console report, and persists the full run to `~/.bench-loop/runs/`. +This runs the versioned `core` profile, scores it, prints a console report, and persists the full run to `~/.bench-loop/runs/`. + +### Benchmark profiles + +```bash +benchloop run --model qwen3:8b --profile smoke # 39-task sanity check +benchloop run --model qwen3:8b --profile core # 81-task default +benchloop run --model qwen3:8b --profile full # 93 tasks + agent + 2K–32K context +benchloop run --model qwen3:8b --profile core --trials 5 +``` + +Every run stores its profile, benchmark/scoring version, coverage, and a SHA-256 +manifest of the exact prompts, generation settings, and validators. A custom +`--suites` run is not silently presented as a complete benchmark. ### Run a subset @@ -128,11 +141,14 @@ benchloop run \ --remote ``` -The `--remote` flag (auto-detected for non-localhost endpoints) switches to cloud-aware scoring: +The `--remote` flag (auto-detected for public API endpoints) switches to cloud-aware speed scoring: - **Speed** uses streaming TTFT (time-to-first-token) + effective content tok/s -- **Overall** = 0.50·quality + 0.25·speed + 0.25·reliability (vs local's 0.55/0.20/0.25) - Reasoning models: content tok/s excludes internal thinking tokens +Private/LAN and Tailscale addresses are treated as local hardware. For unusual +DNS or tunnel setups, choose explicitly with `--local` or `--remote` so a remote +4090 is not scored with a hosted-API curve. + ### API key auth Required for vLLM, sglang, and most cloud providers. Two ways to provide it: @@ -188,41 +204,46 @@ benchloop dashboard --dev | Suite | What it scores | |---|---| -| `speed` | Latency, throughput, TTFT, generation tok/s across short/medium/long contexts | +| `speed` | Latency, throughput, TTFT, generation tok/s across short/medium/long output lengths | | `toolcall` | Structured tool-call correctness across realistic tasks (weather, stocks, email, search) | -| `coding` | Executable Python tasks verified in a sandboxed subprocess (10s timeout) | +| `coding` | Executable Python tasks verified in a restricted interpreter with policy, time, output, and resource limits | | `dataextract` | JSON / structured extraction from messy natural language | | `instructfollow` | Constraint following, formatting, exactness | | `reasonmath` | Small reasoning + math tasks with deterministic checks | +| `longcontext` | Deterministic retrieval and prefill telemetry at approximate 2K, 8K, 16K, and 32K prompt tiers | | `agent` | **Multi-turn agentic tool use.** BenchLoop drives a real loop: model emits a tool call, BenchLoop executes it locally, feeds the result back, model iterates until done. Scores correctness, efficiency, no-hallucination, required-tool coverage. | ## Scoring ``` -Local: Overall = 0.55 · quality + 0.20 · speed + 0.25 · reliability -Cloud: Overall = 0.50 · quality + 0.25 · speed + 0.25 · reliability (with streaming speed data) - Overall = 0.65 · quality + 0.35 · reliability (no speed data) +Overall = 0.70 · quality + 0.25 · speed + 0.05 · reliability +No speed suite: 0.90 · quality + 0.10 · reliability ``` -- **Quality** = mean of non-speed suite scores (size-fair). +- **Quality** = fixed profile-specific weighted average of capability suites. - **Speed (local)** = `12.54 · log2(tok/s) + 0.9`, clamped to 0–100. - **Speed (cloud)** = 0.60 · TTFT_score + 0.40 · tok/s_score, where TTFT uses exponential decay (200ms→100, 2000ms→40) and tok/s uses a log curve calibrated for 20-150 tok/s. -- **Reliability** = pass rate across all tasks. +- **Reliability** = endpoint/runtime execution success, separate from model correctness. - **Agent** = `correct_final + efficient + no_hallucinated_tools + all_required_called`, 25 pts each, averaged across tasks. +Speed prompts use three trials by default, discard the first warmup, select the +median post-warmup trial, and persist run-level median plus p50/p95 telemetry. +See [the v3 benchmark specification](docs/BENCHMARK_SPEC_V3.md) for profile +weights, comparability rules, long-context protocol, and coding-execution limits. + ## Local web app A FastAPI backend + React frontend bundle ships alongside the CLI for visualizing runs: ```bash -benchloop dashboard # starts the local web app on :5180 +benchloop dashboard # starts the local web app on :8877 ``` Tabs: Models, Benchmark, Leaderboard, Compare runs, Chat, agent trace viewer. ## Publish a run -Every completed benchmark auto-publishes to via `https://api.bench-loop.com/submit`. Runs are deduped by `(machine_id, run_id)` so the same run from the same machine won't be double-counted. +By default, every completed benchmark publishes to via `https://api.bench-loop.com/submit`. Runs are deduped by `(machine_id, run_id)` so the same run from the same machine won't be double-counted. Opt out: @@ -255,12 +276,12 @@ bench-loop-web/ ← the web app (separate repo) ## Status -BenchLoop is **v0.2 beta**. The benchmark surface, scoring, web app, agent loop, four harnesses, and cloud provider support all work end-to-end. Stuff still on the roadmap: +BenchLoop is **v0.3 beta**. Versioned profiles, provenance manifests, restricted coding execution, long-context retrieval, agent telemetry, four harnesses, and local/cloud provider modes work end-to-end. Next on the roadmap: - ~~Streaming TTFT for OpenAI-compatible providers~~ ✅ (v0.2.3+ with `--remote`) -- Bigger task fixtures (each suite is intentionally small and frozen for v1) -- Hosted submission flow for community runs -- Cloud-specific leaderboard on bench-loop.com (filter by local vs remote) +- Vision and multimodal evaluation +- Peak GPU memory, energy-per-token, and sustained-load telemetry +- Seeded private challenge sets for stronger contamination resistance - More provider adapters (TGI, Bedrock, etc. if there's demand) ## License diff --git a/bench_loop/__init__.py b/bench_loop/__init__.py index 97862d7..38eb940 100644 --- a/bench_loop/__init__.py +++ b/bench_loop/__init__.py @@ -1,2 +1,2 @@ """BenchLoop - Local LLM benchmarking CLI.""" -__version__ = "0.2.3" +__version__ = "0.3.0" diff --git a/bench_loop/benchmark_manifest.py b/bench_loop/benchmark_manifest.py new file mode 100644 index 0000000..40c9c73 --- /dev/null +++ b/bench_loop/benchmark_manifest.py @@ -0,0 +1,184 @@ +"""Versioned benchmark profiles and reproducibility metadata. + +This module is deliberately dependency-light so the CLI, local dashboard, and +public export path all use the same definition of a complete benchmark. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass + +from bench_loop.models import BenchmarkTask + +BENCHMARK_ID = "benchloop" +BENCHMARK_VERSION = "3.0.0" +SCORE_SCHEMA_VERSION = "3.0.0" +DEFAULT_PROFILE = "core" + + +@dataclass(frozen=True) +class BenchmarkProfile: + name: str + suites: tuple[str, ...] + quality_weights: Mapping[str, float] + description: str + + +PROFILES: dict[str, BenchmarkProfile] = { + "smoke": BenchmarkProfile( + name="smoke", + suites=("speed", "toolcall", "reasonmath"), + quality_weights={"toolcall": 0.45, "reasonmath": 0.55}, + description="Fast endpoint and quality sanity check.", + ), + "core": BenchmarkProfile( + name="core", + suites=( + "speed", + "toolcall", + "coding", + "dataextract", + "instructfollow", + "reasonmath", + ), + quality_weights={ + "toolcall": 0.20, + "coding": 0.25, + "dataextract": 0.15, + "instructfollow": 0.15, + "reasonmath": 0.25, + }, + description="Comparable daily-driver benchmark across speed and five quality domains.", + ), + "full": BenchmarkProfile( + name="full", + suites=( + "speed", + "toolcall", + "coding", + "dataextract", + "instructfollow", + "reasonmath", + "longcontext", + "agent", + ), + quality_weights={ + "toolcall": 0.12, + "coding": 0.18, + "dataextract": 0.10, + "instructfollow": 0.10, + "reasonmath": 0.15, + "longcontext": 0.15, + "agent": 0.20, + }, + description="Core benchmark plus long-context retrieval and the multi-turn agent loop.", + ), +} + + +def get_profile(name: str) -> BenchmarkProfile: + try: + return PROFILES[name] + except KeyError as exc: + choices = ", ".join(PROFILES) + raise ValueError( + f"Unknown benchmark profile: {name}. Available: {choices}" + ) from exc + + +def resolve_suites( + profile: str = DEFAULT_PROFILE, suites: Sequence[str] | None = None +) -> list[str]: + """Resolve requested suites while preserving the caller's order.""" + if suites is None: + return list(get_profile(profile).suites) + return list( + dict.fromkeys(str(item).strip() for item in suites if str(item).strip()) + ) + + +def classify_suites(suites: Iterable[str]) -> str: + selected = set(suites) + for name, profile in PROFILES.items(): + if selected == set(profile.suites): + return name + return "custom" + + +def profile_coverage(requested_profile: str, suites: Iterable[str]) -> float: + """Return weighted suite coverage for the requested profile (0..100).""" + expected = get_profile(requested_profile) + selected = set(suites) + # Speed is part of every public comparison but not a quality weight. + speed_weight = 0.20 + quality_weight = 0.80 + covered = ( + speed_weight if "speed" in selected and "speed" in expected.suites else 0.0 + ) + covered += quality_weight * sum( + weight + for suite, weight in expected.quality_weights.items() + if suite in selected + ) + return round(min(100.0, covered * 100.0), 2) + + +def quality_weights_for(profile: str, suites: Iterable[str]) -> dict[str, float]: + """Return normalized quality weights for the suites present in a run.""" + selected = set(suites) + basis = get_profile( + profile if profile in PROFILES else DEFAULT_PROFILE + ).quality_weights + present = {suite: weight for suite, weight in basis.items() if suite in selected} + total = sum(present.values()) + if total <= 0: + return {} + return {suite: weight / total for suite, weight in present.items()} + + +def manifest_hash( + *, + requested_profile: str, + selected_suites: Sequence[str], + tasks_by_suite: Mapping[str, Sequence[BenchmarkTask]], +) -> str: + """Hash the exact prompts, configs, and validators used by a run.""" + payload = { + "benchmark_id": BENCHMARK_ID, + "benchmark_version": BENCHMARK_VERSION, + "score_schema_version": SCORE_SCHEMA_VERSION, + "requested_profile": requested_profile, + "selected_suites": list(selected_suites), + "tasks": { + suite: [asdict(task) for task in tasks_by_suite.get(suite, ())] + for suite in selected_suites + }, + } + canonical = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}" + + +def is_comparable_profile(profile: str, coverage: float) -> bool: + return profile in PROFILES and coverage == 100.0 + + +__all__ = [ + "BENCHMARK_ID", + "BENCHMARK_VERSION", + "DEFAULT_PROFILE", + "PROFILES", + "SCORE_SCHEMA_VERSION", + "BenchmarkProfile", + "classify_suites", + "get_profile", + "is_comparable_profile", + "manifest_hash", + "profile_coverage", + "quality_weights_for", + "resolve_suites", +] diff --git a/bench_loop/cli.py b/bench_loop/cli.py index 589d362..e31b343 100644 --- a/bench_loop/cli.py +++ b/bench_loop/cli.py @@ -1,4 +1,5 @@ """BenchLoop CLI.""" + from __future__ import annotations import asyncio @@ -11,12 +12,19 @@ import click from bench_loop import __version__ +from bench_loop.benchmark_manifest import DEFAULT_PROFILE, PROFILES, classify_suites from bench_loop.harness import list_harnesses from bench_loop.report.console import print_run_report -from bench_loop.runner.orchestrator import DEFAULT_SUITES, SUITE_REGISTRY, run_benchmark +from bench_loop.runner.orchestrator import ( + SUITE_REGISTRY, + endpoint_is_cloud, + run_benchmark, +) from bench_loop.runner.result_writer import save_run -RUNS_DIR = Path(os.environ.get("BENCHLOOP_RUNS", Path.home() / ".bench-loop" / "runs")).expanduser() +RUNS_DIR = Path( + os.environ.get("BENCHLOOP_RUNS", Path.home() / ".bench-loop" / "runs") +).expanduser() @click.group(help="BenchLoop local LLM benchmarking CLI.") @@ -33,7 +41,7 @@ class SuiteSummary: async def _suite_summaries() -> list[SuiteSummary]: summaries: list[SuiteSummary] = [] - for suite_name, suite_cls in SUITE_REGISTRY.items(): + for suite_cls in SUITE_REGISTRY.values(): suite = suite_cls() tasks = await suite.load_tasks() summaries.append(SuiteSummary(name=suite.name, task_count=len(tasks))) @@ -49,6 +57,11 @@ def info() -> None: click.echo("\nSupported suites:") for summary in summaries: click.echo(f" {summary.name}: {summary.task_count} tasks") + counts = {summary.name: summary.task_count for summary in summaries} + click.echo("\nBenchmark profiles:") + for profile in PROFILES.values(): + task_count = sum(counts.get(suite, 0) for suite in profile.suites) + click.echo(f" {profile.name}: {task_count} tasks — {profile.description}") click.echo("\nAvailable harnesses:") for harness_name in list_harnesses(): click.echo(f" {harness_name}") @@ -56,13 +69,34 @@ def info() -> None: @main.command() @click.option("--model", required=True, help="Model name to benchmark.") -@click.option("--endpoint", default="http://localhost:11434", show_default=True, help="Provider endpoint URL.") -@click.option("--provider", default="ollama", show_default=True, help="Provider backend.") +@click.option( + "--endpoint", + default="http://localhost:11434", + show_default=True, + help="Provider endpoint URL.", +) +@click.option( + "--provider", default="ollama", show_default=True, help="Provider backend." +) +@click.option( + "--profile", + "benchmark_profile", + type=click.Choice(list(PROFILES), case_sensitive=False), + default=DEFAULT_PROFILE, + show_default=True, + help="Versioned benchmark profile. --suites overrides its suite selection.", +) @click.option( "--suites", - default=",".join(DEFAULT_SUITES), + default=None, + help="Custom comma-separated suite list. Such runs are labeled custom unless they exactly match a profile.", +) +@click.option( + "--trials", + default=3, show_default=True, - help="Comma-separated suite list.", + type=click.IntRange(1, 20), + help="Speed trials per prompt; the first is warmup when trials > 1.", ) @click.option( "--harness", @@ -110,7 +144,14 @@ def info() -> None: "--remote", is_flag=True, default=False, - help="Mark as remote/cloud benchmark. Skips local speed scoring (tok/s) since cloud inference throughput isn't comparable to local hardware.", + help="Mark as remote/cloud benchmark and use streaming TTFT plus the cloud-aware speed curve.", +) +@click.option( + "--local", + "force_local", + is_flag=True, + default=False, + help="Force the local-hardware speed curve for LAN, SSH-tunnel, or Tailscale endpoints.", ) @click.option( "--api-key", @@ -134,7 +175,9 @@ def run( model: str, endpoint: str, provider: str, - suites: str, + benchmark_profile: str, + suites: str | None, + trials: int, harness: str, hardware: str | None, gpu: str | None, @@ -144,6 +187,7 @@ def run( profile_url: str | None, command_used: str | None, remote: bool, + force_local: bool, api_key: str | None, max_tokens: int | None, ) -> None: @@ -157,25 +201,23 @@ def run( if provider == "ollama": try: from urllib.parse import urlparse + port = urlparse(endpoint).port if port in {1234, 1337, 5001, 8000, 8080, 8081}: provider = "openai_compat" - click.echo(f"[auto-detected provider=openai_compat from port {port}]", err=True) + click.echo( + f"[auto-detected provider=openai_compat from port {port}]", err=True + ) except Exception: pass - # Auto-detect cloud endpoints if --remote not explicitly set - if not remote: - try: - from urllib.parse import urlparse - parsed = urlparse(endpoint) - hostname = parsed.hostname or "" - # If not localhost/127.0.0.1/::1, assume cloud - if hostname and hostname not in ("localhost", "127.0.0.1", "::1", ""): - remote = True - click.echo(f"[auto-detected cloud endpoint: {hostname}]", err=True) - except Exception: - pass + if remote and force_local: + raise click.UsageError("--remote and --local are mutually exclusive") + if not remote and not force_local and endpoint_is_cloud(endpoint): + remote = True + click.echo( + "[auto-detected hosted/cloud endpoint; use --local to override]", err=True + ) # Surface CLI hardware overrides to detect_hardware() via env vars so the # whole detection pipeline picks them up without threading another arg. @@ -191,30 +233,40 @@ def run( "avatar_url": profile_avatar_url, "profile_url": profile_url, } - command_used = (command_used or os.environ.get("BENCHLOOP_COMMAND_USED") or "").strip() or None + command_used = ( + command_used or os.environ.get("BENCHLOOP_COMMAND_USED") or "" + ).strip() or None + + selected_suites = ( + [item.strip() for item in suites.split(",") if item.strip()] + if suites is not None + else None + ) - selected_suites = [item.strip() for item in suites.split(",") if item.strip()] - # Progress callback for CLI output import time + start_time = time.time() total_tasks = 0 completed_tasks = 0 - + def on_progress(event: dict) -> None: nonlocal total_tasks, completed_tasks event_type = event.get("type") - + if event_type == "run_started": total_tasks = event.get("total_tasks", 0) suite_names = event.get("suites", []) - click.echo(f"\n🚀 Starting benchmark: {len(suite_names)} suites, {total_tasks} tasks total", err=True) - + click.echo( + f"\n🚀 Starting benchmark: {len(suite_names)} suites, {total_tasks} tasks total", + err=True, + ) + elif event_type == "suite_started": suite = event.get("suite", "") task_count = event.get("task_count", 0) click.echo(f"\n📊 [{suite}] Running {task_count} tasks...", err=True) - + elif event_type == "task_completed": completed_tasks = event.get("completed_tasks", 0) suite = event.get("suite", "") @@ -222,33 +274,46 @@ def on_progress(event: dict) -> None: passed = event.get("passed", False) score = event.get("score", 0) latency = event.get("latency_ms", 0) - + # Progress bar progress = completed_tasks / total_tasks if total_tasks > 0 else 0 bar_width = 30 filled = int(bar_width * progress) bar = "█" * filled + "░" * (bar_width - filled) - + # ETA calculation elapsed = time.time() - start_time - eta = (elapsed / completed_tasks * (total_tasks - completed_tasks)) if completed_tasks > 0 else 0 + eta = ( + (elapsed / completed_tasks * (total_tasks - completed_tasks)) + if completed_tasks > 0 + else 0 + ) eta_str = f"{int(eta // 60)}m {int(eta % 60)}s" if eta > 0 else "--" - + status = "✓" if passed else "✗" - click.echo(f" [{bar}] {completed_tasks}/{total_tasks} {status} {task_id[:30]:<30} score={score:.0f} {latency/1000:.1f}s ETA={eta_str}", err=True) - + click.echo( + f" [{bar}] {completed_tasks}/{total_tasks} {status} {task_id[:30]:<30} score={score:.0f} {latency / 1000:.1f}s ETA={eta_str}", + err=True, + ) + elif event_type == "suite_completed": suite = event.get("suite", "") score = event.get("score", 0) pass_count = event.get("pass_count", 0) task_count = event.get("task_count", 0) - click.echo(f" ✅ {suite}: {score:.1f}/100 ({pass_count}/{task_count} passed)", err=True) - + click.echo( + f" ✅ {suite}: {score:.1f}/100 ({pass_count}/{task_count} passed)", + err=True, + ) + elif event_type == "run_completed": overall = event.get("overall_score", 0) runtime = event.get("total_runtime_sec", 0) - click.echo(f"\n🎉 Benchmark complete! Overall: {overall:.1f}/100 in {runtime:.1f}s", err=True) - + click.echo( + f"\n🎉 Benchmark complete! Overall: {overall:.1f}/100 in {runtime:.1f}s", + err=True, + ) + try: benchmark = asyncio.run( run_benchmark( @@ -260,6 +325,8 @@ def on_progress(event: dict) -> None: remote=remote, max_tokens=max_tokens, on_progress=on_progress, + profile=benchmark_profile, + runs=trials, ) ) except ValueError as exc: @@ -272,12 +339,19 @@ def on_progress(event: dict) -> None: click.echo(" LM Studio: launch app, enable local server", err=True) click.echo(" • Pull a model first, e.g.:", err=True) click.echo(" ollama pull qwen3:1.7b", err=True) - click.echo(" • If your endpoint isn't Ollama, pass --provider openai_compat", err=True) - click.echo(" • Or launch the dashboard which auto-discovers models:", err=True) + click.echo( + " • If your endpoint isn't Ollama, pass --provider openai_compat", + err=True, + ) + click.echo( + " • Or launch the dashboard which auto-discovers models:", err=True + ) click.echo(" benchloop dashboard", err=True) raise SystemExit(1) except ConnectionError as exc: - click.secho(f"\n✗ Could not reach endpoint {endpoint}: {exc}\n", fg="red", err=True) + click.secho( + f"\n✗ Could not reach endpoint {endpoint}: {exc}\n", fg="red", err=True + ) click.echo("Is your local LLM server running?", err=True) raise SystemExit(1) except Exception as exc: # noqa: BLE001 @@ -286,22 +360,37 @@ def on_progress(event: dict) -> None: # Python traceback. type_name = type(exc).__name__ msg = str(exc) - click.secho(f"\n✗ Benchmark failed ({type_name}): {msg or 'no message'}\n", fg="red", err=True) - if type_name in {"ConnectError", "ConnectionRefusedError", "ConnectionError"} or "connection" in msg.lower(): + click.secho( + f"\n✗ Benchmark failed ({type_name}): {msg or 'no message'}\n", + fg="red", + err=True, + ) + if ( + type_name in {"ConnectError", "ConnectionRefusedError", "ConnectionError"} + or "connection" in msg.lower() + ): click.echo(f"Could not reach endpoint {endpoint}.", err=True) click.echo("Tips:", err=True) click.echo(" • Start your local LLM server:", err=True) click.echo(" Ollama: ollama serve", err=True) click.echo(" LM Studio: launch app, enable local server", err=True) click.echo(" • Verify the endpoint URL and port are right.", err=True) - click.echo(" • If your endpoint isn't Ollama, pass --provider openai_compat", err=True) + click.echo( + " • If your endpoint isn't Ollama, pass --provider openai_compat", + err=True, + ) elif "500" in msg or "Internal Server Error" in msg: click.echo("The provider returned HTTP 500. Common causes:", err=True) click.echo(" • Model context window exceeded for this prompt", err=True) - click.echo(" • GPU OOM (try a smaller model or close other models)", err=True) + click.echo( + " • GPU OOM (try a smaller model or close other models)", err=True + ) click.echo(" • Ollama crashed (check `ollama serve` logs)", err=True) elif "timeout" in msg.lower() or type_name == "ReadTimeout": - click.echo("Timeout. Try a smaller model, fewer suites, or check network stability.", err=True) + click.echo( + "Timeout. Try a smaller model, fewer suites, or check network stability.", + err=True, + ) raise SystemExit(1) # Save before printing -- a console-rendering crash (e.g. legacy Windows # cp1252 terminals choking on an emoji) must not cost the whole run's data. @@ -326,15 +415,24 @@ def suites() -> None: @main.command() -@click.option("--output", "-o", default=None, help="Path to write the leaderboard JSON. Defaults to stdout.") -@click.option("--all", "include_all", is_flag=True, help="Include partial runs (default: only full benchmarks).") +@click.option( + "--output", + "-o", + default=None, + help="Path to write the leaderboard JSON. Defaults to stdout.", +) +@click.option( + "--all", + "include_all", + is_flag=True, + help="Include partial runs (default: named profiles and legacy complete runs).", +) def export(output: str | None, include_all: bool) -> None: """Export local runs to a leaderboard-compatible JSON. The output format matches the schema consumed by https://bench-loop.com/leaderboard so you can submit your own runs via PR. """ - REQUIRED_FULL = {"speed", "toolcall", "dataextract", "instructfollow", "reasonmath"} REQUIRED_QUALITY = {"toolcall", "dataextract", "instructfollow", "reasonmath"} if not RUNS_DIR.exists(): @@ -354,11 +452,13 @@ def export(output: str | None, include_all: bool) -> None: suite_map = data.get("suites") or {} suite_names = set(suite_map.keys()) - is_full = REQUIRED_FULL.issubset(suite_names) + run_profile = data.get("benchmark_profile") or classify_suites(suite_names) + is_full = run_profile == "full" + is_core = run_profile == "core" is_quality = REQUIRED_QUALITY.issubset(suite_names) is_agent_only = suite_names == {"agent"} - if not include_all and not (is_full or is_quality or is_agent_only): + if not include_all and not (is_full or is_core or is_quality or is_agent_only): continue model_id = (data.get("model") or {}).get("model_id", "unknown") @@ -372,15 +472,18 @@ def export(output: str | None, include_all: bool) -> None: "harness": data.get("harness", "raw"), "provider": data.get("provider", ""), "machine": (data.get("machine") or {}).get("gpu") - or (data.get("machine") or {}).get("cpu") - or (data.get("machine") or {}).get("machine_id", ""), + or (data.get("machine") or {}).get("cpu") + or (data.get("machine") or {}).get("machine_id", ""), "overall_score": data.get("overall_score", 0), "quality_score": data.get("quality_score", 0), "speed_score": data.get("speed_score", 0), "reliability_score": data.get("reliability_score", 0), - "generation_tok_per_sec": (data.get("speed_metrics") or {}).get("generation_tok_per_sec", 0), + "generation_tok_per_sec": (data.get("speed_metrics") or {}).get( + "generation_tok_per_sec", 0 + ), "ttft_ms": (data.get("speed_metrics") or {}).get("ttft_ms", 0), "is_full_benchmark": is_full, + "is_core_benchmark": is_core, "is_quality_full": is_quality, "is_agent_only": is_agent_only, "agent_score": (suite_map.get("agent") or {}).get("score"), @@ -389,7 +492,16 @@ def export(output: str | None, include_all: bool) -> None: "profile_name": ((data.get("profile") or {}).get("name") or ""), "profile_avatar_url": ((data.get("profile") or {}).get("avatar_url") or ""), "profile_url": ((data.get("profile") or {}).get("profile_url") or ""), - "suites": {name: {"score": s.get("score", 0)} for name, s in suite_map.items()}, + "suites": { + name: {"score": s.get("score", 0)} for name, s in suite_map.items() + }, + "benchmark_id": data.get("benchmark_id", "benchloop-legacy"), + "benchmark_version": data.get("benchmark_version", "legacy"), + "benchmark_profile": run_profile, + "score_schema_version": data.get("score_schema_version", "legacy"), + "manifest_hash": data.get("manifest_hash", ""), + "coverage_score": data.get("coverage_score", 0), + "comparable": data.get("comparable", is_full or is_core), } # Keep best run per model+harness. @@ -399,7 +511,9 @@ def export(output: str | None, include_all: bool) -> None: rows[key] = row payload = { - "generated_at": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat(), + "generated_at": __import__("datetime") + .datetime.now(__import__("datetime").timezone.utc) + .isoformat(), "count": len(rows), "source": "benchloop export", "runs": sorted(rows.values(), key=lambda r: r["overall_score"], reverse=True), @@ -414,18 +528,46 @@ def export(output: str | None, include_all: bool) -> None: @main.command() @click.option("--host", default="127.0.0.1", show_default=True) -@click.option("--port", "port", default=8877, show_default=True, type=int, help="Port for the dashboard (API + UI).") -@click.option("--api-port", default=None, type=int, help="DEPRECATED. Same as --port; kept for compatibility.") -@click.option("--ui-port", default=None, type=int, help="DEPRECATED. UI is now served by the API.") -@click.option("--api-only", is_flag=True, help="Legacy flag, no-op now that UI is bundled.") -@click.option("--dev/--no-dev", default=False, help="Use the sibling bench-loop-web repo with hot-reload (developer mode).") +@click.option( + "--port", + "port", + default=8877, + show_default=True, + type=int, + help="Port for the dashboard (API + UI).", +) +@click.option( + "--api-port", + default=None, + type=int, + help="DEPRECATED. Same as --port; kept for compatibility.", +) +@click.option( + "--ui-port", default=None, type=int, help="DEPRECATED. UI is now served by the API." +) +@click.option( + "--api-only", is_flag=True, help="Legacy flag, no-op now that UI is bundled." +) +@click.option( + "--dev/--no-dev", + default=False, + help="Use the sibling bench-loop-web repo with hot-reload (developer mode).", +) @click.option( "--service-template", type=click.Choice(["launchd", "systemd", "windows-task"], case_sensitive=False), default=None, help="Print a persistence template instead of launching the dashboard.", ) -def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, api_only: bool, dev: bool, service_template: str | None) -> None: +def dashboard( + host: str, + port: int, + api_port: int | None, + ui_port: int | None, + api_only: bool, + dev: bool, + service_template: str | None, +) -> None: """Launch the local web dashboard. By default this runs the bundled FastAPI + React app that ships inside the @@ -446,7 +588,7 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a if service_template: command = f"benchloop dashboard --host {host} --port {port}" if service_template == "launchd": - click.echo(f''' + click.echo(f""" @@ -462,9 +604,9 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a StandardOutPath~/Library/Logs/benchloop-dashboard.log StandardErrorPath~/Library/Logs/benchloop-dashboard.err -''') +""") elif service_template == "systemd": - click.echo(f'''[Unit] + click.echo(f"""[Unit] Description=BenchLoop dashboard After=network.target @@ -476,7 +618,7 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a Environment=PYTHONUNBUFFERED=1 [Install] -WantedBy=multi-user.target''') +WantedBy=multi-user.target""") else: click.echo(f'''# PowerShell Scheduled Task / startup command # Run this in a persistent shell or wrap it in Task Scheduler: @@ -488,10 +630,12 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a return if dev: - web_dir = Path(os.environ.get( - "BENCHLOOP_WEB_DIR", - Path(__file__).resolve().parent.parent.parent / "bench-loop-web", - )).resolve() + web_dir = Path( + os.environ.get( + "BENCHLOOP_WEB_DIR", + Path(__file__).resolve().parent.parent.parent / "bench-loop-web", + ) + ).resolve() api_dir = web_dir / "api" ui_dir = web_dir / "ui" if not api_dir.is_dir(): @@ -504,10 +648,23 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a sys.exit(1) env = os.environ.copy() env["BENCH_LOOP_DIR"] = str(Path(__file__).resolve().parent.parent) - env["PYTHONPATH"] = env["BENCH_LOOP_DIR"] + os.pathsep + env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + env["BENCH_LOOP_DIR"] + os.pathsep + env.get("PYTHONPATH", "") + ) api_proc = subprocess.Popen( - [sys.executable, "-m", "uvicorn", "main:app", - "--host", host, "--port", str(port), "--app-dir", str(api_dir), "--reload"], + [ + sys.executable, + "-m", + "uvicorn", + "main:app", + "--host", + host, + "--port", + str(port), + "--app-dir", + str(api_dir), + "--reload", + ], env=env, ) click.echo(f"BenchLoop API (dev): http://{host}:{port}") @@ -533,10 +690,22 @@ def dashboard(host: str, port: int, api_port: int | None, ui_port: int | None, a sys.exit(1) env = os.environ.copy() env["BENCH_LOOP_DIR"] = str(Path(__file__).resolve().parent.parent) - env["PYTHONPATH"] = env["BENCH_LOOP_DIR"] + os.pathsep + env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + env["BENCH_LOOP_DIR"] + os.pathsep + env.get("PYTHONPATH", "") + ) api_proc = subprocess.Popen( - [sys.executable, "-m", "uvicorn", "main:app", - "--host", host, "--port", str(port), "--app-dir", str(api_dir)], + [ + sys.executable, + "-m", + "uvicorn", + "main:app", + "--host", + host, + "--port", + str(port), + "--app-dir", + str(api_dir), + ], env=env, ) url = f"http://{host}:{port}" diff --git a/bench_loop/config.py b/bench_loop/config.py index 2e0131e..d0e3efc 100644 --- a/bench_loop/config.py +++ b/bench_loop/config.py @@ -1,10 +1,10 @@ """BenchLoop configuration.""" + from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path - BENCH_LOOP_DIR = Path(__file__).parent TASKS_DIR = BENCH_LOOP_DIR / "tasks" RESULTS_DIR = Path.cwd() / "results" @@ -15,6 +15,8 @@ class RunConfig: model: str = "" provider: str = "ollama" harness: str = "raw" + profile: str = "core" + remote: bool | None = None suites: list[str] = field(default_factory=list) trials: int = 3 warmup: bool = True diff --git a/bench_loop/dashboard/api/routes/benchmark.py b/bench_loop/dashboard/api/routes/benchmark.py index d1c6a9d..d23e727 100644 --- a/bench_loop/dashboard/api/routes/benchmark.py +++ b/bench_loop/dashboard/api/routes/benchmark.py @@ -1,10 +1,10 @@ """Benchmark run management — kick off, stream, list, detail.""" + from __future__ import annotations import asyncio import json import shutil -import time import uuid from datetime import datetime, timezone from pathlib import Path as FsPath @@ -12,12 +12,11 @@ from fastapi import APIRouter, HTTPException, Query from fastapi.responses import StreamingResponse -from pydantic import BaseModel, Field +from pydantic import BaseModel -from bench_loop.config import RunConfig +from bench_loop.benchmark_manifest import DEFAULT_PROFILE, classify_suites from bench_loop.runner.orchestrator import run_benchmark from bench_loop.runner.result_writer import save_failed_run, save_run -from bench_loop.models import BenchmarkRun router = APIRouter() @@ -45,10 +44,12 @@ class BenchmarkRequest(BaseModel): model: str endpoint: str = "http://localhost:11434" provider: str = "ollama" - suites: list[str] = Field(default_factory=lambda: ["speed", "toolcall", "coding", "dataextract", "instructfollow", "reasonmath"]) + profile: str = DEFAULT_PROFILE + suites: list[str] | None = None harness: str = "raw" runs: int = 3 timeout_sec: float = 300.0 + remote: bool | None = None profile_name: str | None = None profile_avatar_url: str | None = None profile_url: str | None = None @@ -62,16 +63,18 @@ async def start_benchmark_route(req: BenchmarkRequest): # Verify API key if provided if req.api_key: from .users import verify_api_key + verified_user_id = verify_api_key(req.api_key) if not verified_user_id: raise HTTPException(status_code=401, detail="Invalid API key") req.user_id = verified_user_id - + run_id = str(uuid.uuid4())[:8] # Use a plain namespace to stay compatible with multiple RunConfig schemas # (the canonical bench_loop uses `base_url`/`suites`/`trials`; the legacy one # used `endpoint`/`suite_names`/`runs`). The orchestrator handles both. from types import SimpleNamespace + config = SimpleNamespace( model=req.model, provider=req.provider, @@ -83,6 +86,8 @@ async def start_benchmark_route(req: BenchmarkRequest): runs=req.runs, trials=req.runs, timeout_sec=req.timeout_sec, + profile=req.profile, + remote=req.remote, ) queue = _endpoint_queues.setdefault(req.endpoint, []) @@ -97,6 +102,8 @@ async def start_benchmark_route(req: BenchmarkRequest): "model": req.model, "endpoint": req.endpoint, "suites": req.suites, + "profile": req.profile, + "remote": req.remote, "harness": req.harness, }, "events": [], @@ -121,7 +128,11 @@ async def _run(): task = asyncio.ensure_future(_run()) _active_runs[run_id]["task"] = task - return {"run_id": run_id, "status": _active_runs[run_id]["status"], "queue_position": position} + return { + "run_id": run_id, + "status": _active_runs[run_id]["status"], + "queue_position": position, + } @router.post("/benchmark/cancel/{run_id}") @@ -146,7 +157,9 @@ async def cancel_benchmark(run_id: str): return {"ok": True, "status": "cancelled"} -async def _execute_run(run_id: str, req: "BenchmarkRequest", config: Any, on_progress: Any) -> None: +async def _execute_run( + run_id: str, req: BenchmarkRequest, config: Any, on_progress: Any +) -> None: try: result = await run_benchmark(config, on_progress=on_progress) _active_runs[run_id]["status"] = "completed" @@ -167,10 +180,12 @@ async def _execute_run(run_id: str, req: "BenchmarkRequest", config: Any, on_pro ) _active_runs[run_id]["saved_path"] = str(saved_path) except Exception as save_exc: - _active_runs[run_id]["events"].append({ - "type": "persist_failed", - "error": str(save_exc), - }) + _active_runs[run_id]["events"].append( + { + "type": "persist_failed", + "error": str(save_exc), + } + ) except asyncio.CancelledError: # User cancelled — already marked in cancel_benchmark, but make sure. if _active_runs[run_id]["status"] not in ("completed", "failed"): @@ -178,17 +193,20 @@ async def _execute_run(run_id: str, req: "BenchmarkRequest", config: Any, on_pro raise except Exception as exc: import traceback + tb = traceback.format_exc() - err_msg = str(exc) or f"{type(exc).__name__}: {repr(exc)}" + err_msg = str(exc) or f"{type(exc).__name__}: {exc!r}" _active_runs[run_id]["status"] = "failed" _active_runs[run_id]["completed_at"] = datetime.now(timezone.utc).isoformat() _active_runs[run_id]["error"] = err_msg _active_runs[run_id]["traceback"] = tb - _active_runs[run_id]["events"].append({ - "type": "run_failed", - "error": err_msg, - "exception_class": type(exc).__name__, - }) + _active_runs[run_id]["events"].append( + { + "type": "run_failed", + "error": err_msg, + "exception_class": type(exc).__name__, + } + ) try: publish_profile = { "name": req.profile_name, @@ -208,19 +226,23 @@ async def _execute_run(run_id: str, req: "BenchmarkRequest", config: Any, on_pro publish_profile=publish_profile, command_used=req.command_used, user_id=req.user_id, + benchmark_profile=req.profile, ) _active_runs[run_id]["saved_path"] = str(saved_path) except Exception as save_exc: - _active_runs[run_id]["events"].append({ - "type": "persist_failed", - "error": str(save_exc), - }) + _active_runs[run_id]["events"].append( + { + "type": "persist_failed", + "error": str(save_exc), + } + ) print(f"[bench-loop-api] run {run_id} failed:\n{tb}", flush=True) @router.get("/benchmark/stream/{run_id}") async def stream_benchmark(run_id: str): """SSE stream for benchmark progress — emits granular task-level events.""" + async def event_generator(): if run_id not in _active_runs: yield f"data: {json.dumps({'type': 'error', 'error': 'Run not found'})}\n\n" @@ -249,16 +271,17 @@ async def event_generator(): @router.get("/benchmark/runs") async def list_runs( limit: int = Query(default=50, le=200), - is_remote: bool | None = Query(default=None, description="Filter by remote/cloud (true) or local (false). Omit for all."), + is_remote: bool | None = Query( + default=None, + description="Filter by remote/cloud (true) or local (false). Omit for all.", + ), ): """List past benchmark runs from disk.""" if not RUNS_DIR.exists(): return {"runs": []} - # "Full benchmark" = at least these quality suites + speed. Coding is bonus. - REQUIRED_FULL_SUITES = {"speed", "toolcall", "dataextract", "instructfollow", "reasonmath"} - import math + def _recompute_speed_score(tok_per_sec: float) -> float: """Match bench_loop.suites.speed v2 curve so older runs use the new scale.""" if tok_per_sec <= 0: @@ -273,7 +296,7 @@ def _recompute_speed_score(tok_per_sec: float) -> float: continue try: data = json.loads(run_file.read_text()) - + # Filter by is_remote if specified run_is_remote = data.get("is_remote", False) if is_remote is not None and run_is_remote != is_remote: @@ -283,65 +306,94 @@ def _recompute_speed_score(tok_per_sec: float) -> float: speed_metrics = data.get("speed_metrics", {}) or {} suite_map = data.get("suites", {}) or {} suite_names = list(suite_map.keys()) - is_full = REQUIRED_FULL_SUITES.issubset(set(suite_names)) + benchmark_profile = data.get("benchmark_profile") or classify_suites( + suite_names + ) + is_full = benchmark_profile == "full" + is_core = benchmark_profile == "core" - # Recompute speed score from tok/s using the v2 curve so historical - # runs (which used the old 25*log2 capped-at-100 curve) display - # comparably to new runs. + # Preserve versioned v3 scores exactly. Legacy runs retain the v2 + # display migration but are visibly marked and never mixed into a + # versioned comparison without that label. gen_tok_per_sec = speed_metrics.get("generation_tok_per_sec", 0) or 0 - recomputed_speed = _recompute_speed_score(gen_tok_per_sec) - speed_score_v2 = recomputed_speed - # Recompute overall using new speed score (quality/reliability unchanged). - quality_v2 = data.get("quality_score", 0) - reliability_v2 = data.get("reliability_score", 0) - overall_v2 = 0.55 * quality_v2 + 0.20 * speed_score_v2 + 0.25 * reliability_v2 - runs.append({ - "id": d.name, - "status": data.get("status", "completed"), - "error": data.get("error", "") or "", - "traceback": data.get("traceback", "") or "", - "timestamp": data.get("timestamp", ""), - "model": model_obj.get("model_id", "unknown"), - "quantization": model_obj.get("quantization", "") or "", - "family": model_obj.get("family", "") or "", - "parameter_count": model_obj.get("parameter_count", "") or "", - "is_remote": run_is_remote, - "overall_score": overall_v2, - "quality_score": quality_v2, - "speed_score": speed_score_v2, - "reliability_score": reliability_v2, - "overall_score_raw": data.get("overall_score", 0), - "speed_score_raw": data.get("speed_score", 0), - "value_score": data.get("value_score", 0), - "total_runtime_sec": data.get("total_runtime_sec", 0), - "harness": data.get("harness", "raw"), - "suites": { - name: { - "score": s.get("score", 0), - "pass_count": s.get("pass_count", 0), - "task_count": s.get("task_count", 0), - } - for name, s in suite_map.items() - }, - "suite_count": len(suite_names), - "suite_names": suite_names, - "is_full_benchmark": is_full, - "provider": data.get("provider", ""), - "backend": machine.get("backend", data.get("provider", "")), - "machine": machine.get("machine_id", ""), - "profile_name": ((data.get("profile") or {}).get("name") or ""), - "profile_avatar_url": ((data.get("profile") or {}).get("avatar_url") or ""), - "profile_url": ((data.get("profile") or {}).get("profile_url") or ""), - "command_used": data.get("command_used", "") or "", - "gpu": machine.get("gpu", ""), - "gpu_memory_gb": machine.get("gpu_memory_gb", 0), - "cpu": machine.get("cpu", ""), - "system_memory_gb": machine.get("system_memory_gb", 0), - "os": machine.get("os", ""), - "generation_tok_per_sec": speed_metrics.get("generation_tok_per_sec", 0), - "prompt_eval_tok_per_sec": speed_metrics.get("prompt_eval_tok_per_sec", 0), - "ttft_ms": speed_metrics.get("ttft_ms", 0), - }) + is_versioned = bool(data.get("score_schema_version")) + quality_display = data.get("quality_score", 0) + reliability_display = data.get("reliability_score", 0) + if is_versioned: + speed_display = data.get("speed_score", 0) + overall_display = data.get("overall_score", 0) + else: + speed_display = _recompute_speed_score(gen_tok_per_sec) + overall_display = ( + 0.55 * quality_display + + 0.20 * speed_display + + 0.25 * reliability_display + ) + runs.append( + { + "id": d.name, + "status": data.get("status", "completed"), + "error": data.get("error", "") or "", + "traceback": data.get("traceback", "") or "", + "timestamp": data.get("timestamp", ""), + "model": model_obj.get("model_id", "unknown"), + "quantization": model_obj.get("quantization", "") or "", + "family": model_obj.get("family", "") or "", + "parameter_count": model_obj.get("parameter_count", "") or "", + "is_remote": run_is_remote, + "overall_score": overall_display, + "quality_score": quality_display, + "speed_score": speed_display, + "reliability_score": reliability_display, + "overall_score_raw": data.get("overall_score", 0), + "speed_score_raw": data.get("speed_score", 0), + "value_score": data.get("value_score", 0), + "total_runtime_sec": data.get("total_runtime_sec", 0), + "harness": data.get("harness", "raw"), + "suites": { + name: { + "score": s.get("score", 0), + "pass_count": s.get("pass_count", 0), + "task_count": s.get("task_count", 0), + } + for name, s in suite_map.items() + }, + "suite_count": len(suite_names), + "suite_names": suite_names, + "is_full_benchmark": is_full, + "is_core_benchmark": is_core, + "benchmark_id": data.get("benchmark_id", "benchloop-legacy"), + "benchmark_version": data.get("benchmark_version", "legacy"), + "benchmark_profile": benchmark_profile, + "score_schema_version": data.get("score_schema_version", "legacy"), + "manifest_hash": data.get("manifest_hash", ""), + "coverage_score": data.get("coverage_score", 0), + "comparable": data.get("comparable", is_full or is_core), + "provider": data.get("provider", ""), + "backend": machine.get("backend", data.get("provider", "")), + "machine": machine.get("machine_id", ""), + "profile_name": ((data.get("profile") or {}).get("name") or ""), + "profile_avatar_url": ( + (data.get("profile") or {}).get("avatar_url") or "" + ), + "profile_url": ( + (data.get("profile") or {}).get("profile_url") or "" + ), + "command_used": data.get("command_used", "") or "", + "gpu": machine.get("gpu", ""), + "gpu_memory_gb": machine.get("gpu_memory_gb", 0), + "cpu": machine.get("cpu", ""), + "system_memory_gb": machine.get("system_memory_gb", 0), + "os": machine.get("os", ""), + "generation_tok_per_sec": speed_metrics.get( + "generation_tok_per_sec", 0 + ), + "prompt_eval_tok_per_sec": speed_metrics.get( + "prompt_eval_tok_per_sec", 0 + ), + "ttft_ms": speed_metrics.get("ttft_ms", 0), + } + ) except Exception: continue @@ -396,7 +448,9 @@ async def delete_run(run_id: str): """Delete a persisted local benchmark run.""" state = _active_runs.get(run_id) if state and state.get("status") not in ("completed", "failed", "cancelled"): - raise HTTPException(status_code=409, detail="Cannot delete an active run. Cancel it first.") + raise HTTPException( + status_code=409, detail="Cannot delete an active run. Cancel it first." + ) runs_root = RUNS_DIR.resolve() run_dir = (RUNS_DIR / run_id).resolve() diff --git a/bench_loop/models.py b/bench_loop/models.py index 0d66dcd..04db7db 100644 --- a/bench_loop/models.py +++ b/bench_loop/models.py @@ -1,4 +1,5 @@ """Core data models for BenchLoop results and configuration.""" + from __future__ import annotations from dataclasses import dataclass, field @@ -80,6 +81,7 @@ class TaskResult: tokens_prompt: int = 0 error: str = "" output: str = "" + execution_ok: bool = True metadata: dict[str, Any] = field(default_factory=dict) @@ -100,16 +102,31 @@ class SpeedMetrics: prompt_eval_tok_per_sec: float = 0.0 generation_tok_per_sec: float = 0.0 total_latency_ms: float = 0.0 + generation_tok_per_sec_p50: float = 0.0 + generation_tok_per_sec_p95: float = 0.0 + ttft_ms_p50: float = 0.0 + ttft_ms_p95: float = 0.0 + sample_count: int = 0 @dataclass class BenchmarkRun: """Top-level result for one complete benchmark run.""" - version: str = "0.1.0" + version: str = "0.3.0" + benchmark_id: str = "benchloop" + benchmark_version: str = "3.0.0" + benchmark_profile: str = "custom" + requested_profile: str = "core" + manifest_hash: str = "" + score_schema_version: str = "3.0.0" + coverage_score: float = 0.0 + comparable: bool = False timestamp: str = "" model: ModelInfo = field(default_factory=lambda: ModelInfo(model_id="unknown")) - machine: MachineInfo = field(default_factory=lambda: MachineInfo(machine_id="unknown")) + machine: MachineInfo = field( + default_factory=lambda: MachineInfo(machine_id="unknown") + ) provider: str = "ollama" harness: str = "raw" harness_version: str = "" @@ -124,42 +141,41 @@ class BenchmarkRun: suites: dict[str, SuiteResult] = field(default_factory=dict) def compute_aggregates(self) -> None: - quality_suites = [ - suite_result - for name, suite_result in self.suites.items() - if name != SuiteName.SPEED and name != SuiteName.SPEED.value - ] - if quality_suites: - self.quality_score = sum(s.score for s in quality_suites) / len(quality_suites) - - speed_suite = self.suites.get(SuiteName.SPEED) or self.suites.get(SuiteName.SPEED.value) + # Import lazily to keep the core dataclasses free of an import cycle. + from bench_loop.benchmark_manifest import quality_weights_for + + weight_profile = ( + self.benchmark_profile + if self.benchmark_profile != "custom" + else self.requested_profile + ) + weights = quality_weights_for(weight_profile, self.suites) + if weights: + self.quality_score = sum( + self.suites[name].score * weight for name, weight in weights.items() + ) + + speed_suite = self.suites.get(SuiteName.SPEED) or self.suites.get( + SuiteName.SPEED.value + ) if speed_suite: self.speed_score = speed_suite.score - total_tasks = sum(s.task_count for s in self.suites.values()) - total_passed = sum(s.pass_count for s in self.suites.values()) - self.reliability_score = (total_passed / total_tasks * 100) if total_tasks > 0 else 0.0 - - if self.is_remote: - # Remote/cloud: use cloud-aware speed scoring when available. - # If speed suite has data (from streaming TTFT + tok/s), include it. - if self.speed_score > 0: - self.overall_score = ( - 0.50 * self.quality_score - + 0.25 * self.speed_score - + 0.25 * self.reliability_score - ) - else: - # No speed data — fall back to quality + reliability only - self.overall_score = ( - 0.65 * self.quality_score - + 0.35 * self.reliability_score - ) + tasks = [task for suite in self.suites.values() for task in suite.tasks] + execution_ok = sum(1 for task in tasks if task.execution_ok) + self.reliability_score = (execution_ok / len(tasks) * 100) if tasks else 0.0 + + # v3 stops counting task correctness twice. Quality owns correctness; + # reliability is reserved for endpoint/runtime execution stability. + if self.speed_score > 0: + self.overall_score = ( + 0.70 * self.quality_score + + 0.25 * self.speed_score + + 0.05 * self.reliability_score + ) else: self.overall_score = ( - 0.55 * self.quality_score - + 0.20 * self.speed_score - + 0.25 * self.reliability_score + 0.90 * self.quality_score + 0.10 * self.reliability_score ) speed_factor = ( @@ -168,7 +184,13 @@ def compute_aggregates(self) -> None: else 0.5 ) reliability_factor = self.reliability_score / 100 - self.value_score = self.quality_score * speed_factor * reliability_factor + self.quality_score = round(self.quality_score, 2) + self.speed_score = round(self.speed_score, 2) + self.reliability_score = round(self.reliability_score, 2) + self.overall_score = round(self.overall_score, 2) + self.value_score = round( + self.quality_score * speed_factor * reliability_factor, 2 + ) def to_dict(self) -> dict[str, Any]: return _asdict_recursive(self) @@ -178,7 +200,10 @@ def _asdict_recursive(obj: Any) -> Any: import dataclasses if dataclasses.is_dataclass(obj) and not isinstance(obj, type): - return {key: _asdict_recursive(value) for key, value in dataclasses.asdict(obj).items()} + return { + key: _asdict_recursive(value) + for key, value in dataclasses.asdict(obj).items() + } if isinstance(obj, dict): return {key: _asdict_recursive(value) for key, value in obj.items()} if isinstance(obj, list): diff --git a/bench_loop/report/console.py b/bench_loop/report/console.py index ce3ce65..5bb0981 100644 --- a/bench_loop/report/console.py +++ b/bench_loop/report/console.py @@ -44,7 +44,9 @@ def _render_header() -> None: def _info_row(label: str, value: str) -> None: - console.print(f" [{BOLD_ACCENT}]◉[/{BOLD_ACCENT}] [{ACCENT}]{label:<10}[/{ACCENT}] {value}") + console.print( + f" [{BOLD_ACCENT}]◉[/{BOLD_ACCENT}] [{ACCENT}]{label:<10}[/{ACCENT}] {value}" + ) def _rule(title: str | None = None) -> None: @@ -58,7 +60,9 @@ def _rule(title: str | None = None) -> None: def _summary_rule(title: str | None = None) -> None: if title: pad = HEADER_WIDTH - len(title) - 6 - console.print(f"\n[{BOLD_ACCENT}]═══ {title} {'═' * max(pad, 4)}[/{BOLD_ACCENT}]") + console.print( + f"\n[{BOLD_ACCENT}]═══ {title} {'═' * max(pad, 4)}[/{BOLD_ACCENT}]" + ) else: console.print(f"[{BOLD_ACCENT}]{'═' * HEADER_WIDTH}[/{BOLD_ACCENT}]") @@ -91,12 +95,23 @@ def render_run_summary(run: BenchmarkRun) -> None: _render_header() _info_row("MODEL", run.model.model_id) _info_row("HARNESS", f"{run.harness} ({run.provider})") + profile_label = f"{run.benchmark_profile} · benchmark {run.benchmark_version}" + if not run.comparable: + profile_label += f" · {run.coverage_score:.0f}% coverage (non-comparable)" + _info_row("PROFILE", profile_label) if run.machine.summary(): _info_row("MACHINE", run.machine.summary()) if run.is_remote: - _info_row("MODE", "☁ remote/cloud (speed reweighted)") + _info_row("MODE", "☁ remote/cloud (cloud-aware speed curve)") if run.speed_metrics.generation_tok_per_sec: _info_row("GEN TOK/S", f"{run.speed_metrics.generation_tok_per_sec:.2f}") + if run.speed_metrics.sample_count: + _info_row( + "SPEED DIST", + f"p50 {run.speed_metrics.generation_tok_per_sec_p50:.2f} · " + f"p95 {run.speed_metrics.generation_tok_per_sec_p95:.2f} · " + f"n={run.speed_metrics.sample_count}", + ) console.print() _rule("Suite Results") @@ -115,7 +130,9 @@ def render_run_summary(run: BenchmarkRun) -> None: _summary_rule("Summary") console.print() if run.is_remote: - speed_label = f"{run.speed_score:.1f}" if run.speed_score > 0 else "N/A (no stream)" + speed_label = ( + f"{run.speed_score:.1f}" if run.speed_score > 0 else "N/A (no stream)" + ) rows: list[tuple[str, Any]] = [ ("QUALITY", run.quality_score), ("SPEED", speed_label), @@ -124,11 +141,17 @@ def render_run_summary(run: BenchmarkRun) -> None: ] for label, val in rows: if isinstance(val, str): - console.print(f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] [{DIM}]{val}[/{DIM}]") + console.print( + f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] [{DIM}]{val}[/{DIM}]" + ) else: - console.print(f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] {val:.1f}") + console.print( + f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] {val:.1f}" + ) if run.speed_metrics.ttft_ms > 0: - console.print(f" [{DIM}]TTFT: {run.speed_metrics.ttft_ms:.0f}ms | tok/s: {run.speed_metrics.generation_tok_per_sec:.1f}[/{DIM}]") + console.print( + f" [{DIM}]TTFT: {run.speed_metrics.ttft_ms:.0f}ms | tok/s: {run.speed_metrics.generation_tok_per_sec:.1f}[/{DIM}]" + ) else: rows = [ ("QUALITY", run.quality_score), @@ -137,7 +160,9 @@ def render_run_summary(run: BenchmarkRun) -> None: ("VALUE", run.value_score), ] for label, val in rows: - console.print(f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] {val:.1f}") + console.print( + f" [{ACCENT}]{label:<14}[/{ACCENT}][{DIM}]│[/{DIM}] {val:.1f}" + ) console.print(f" [{DIM}]{'─' * 24}[/{DIM}]") overall = run.overall_score badge = _score_badge(overall) @@ -146,6 +171,8 @@ def render_run_summary(run: BenchmarkRun) -> None: f" [{BOLD_ACCENT}]OVERALL [/{BOLD_ACCENT}][{DIM}]│[/{DIM}] [{style}]{overall:.1f}[/{style}] {badge}" ) console.print(f"\n [{DIM}]Runtime: {run.total_runtime_sec:.1f}s[/{DIM}]") + if run.manifest_hash: + console.print(f" [{DIM}]Manifest: {run.manifest_hash[:19]}…[/{DIM}]") console.print() diff --git a/bench_loop/runner/orchestrator.py b/bench_loop/runner/orchestrator.py index cb9f1b6..98e4c28 100644 --- a/bench_loop/runner/orchestrator.py +++ b/bench_loop/runner/orchestrator.py @@ -1,6 +1,8 @@ """Benchmark orchestrator.""" + from __future__ import annotations +import ipaddress import time from dataclasses import fields from datetime import datetime, timezone @@ -8,17 +10,32 @@ from typing import Any from urllib.parse import urlparse -from bench_loop.harness import get_harness +from bench_loop.benchmark_manifest import ( + BENCHMARK_ID, + BENCHMARK_VERSION, + DEFAULT_PROFILE, + SCORE_SCHEMA_VERSION, + classify_suites, + is_comparable_profile, + manifest_hash, + profile_coverage, + resolve_suites, +) from bench_loop.hardware import detect_hardware -from bench_loop.models import BenchmarkRun, MachineInfo, ModelInfo, SpeedMetrics, SuiteResult, TaskResult -from bench_loop.providers import ollama -from bench_loop.suites import ( - DEFAULT_SUITES as DEFAULT_SUITES, - SUITE_REGISTRY as SUITE_REGISTRY, +from bench_loop.harness import get_harness +from bench_loop.models import ( + BenchmarkRun, + MachineInfo, + ModelInfo, + SpeedMetrics, + SuiteResult, + TaskResult, ) +from bench_loop.providers import ollama, openai_compat +from bench_loop.suites import DEFAULT_SUITES as _DEFAULT_SUITES +from bench_loop.suites import SUITE_REGISTRY from bench_loop.suites.speed import SpeedSuite -from bench_loop.providers import openai_compat PROVIDER_REGISTRY = { "ollama": ollama, "openai": openai_compat, @@ -26,6 +43,7 @@ "vmlx": openai_compat, # vmlx exposes OpenAI-compatible /v1 } SPEED_TRIALS = 3 +DEFAULT_SUITES = _DEFAULT_SUITES # public back-compat export async def run_benchmark( @@ -37,9 +55,10 @@ async def run_benchmark( suite_names: list[str] | None = None, # alias for API back-compat harness: str = "raw", on_progress=None, - runs: int | None = None, # accepted but currently unused (single-run) + runs: int | None = None, timeout_sec: float | None = None, # accepted but unused - remote: bool = False, # mark as remote/cloud benchmark + remote: bool | None = None, # None=auto, False=local hardware, True=cloud + profile: str = DEFAULT_PROFILE, max_tokens: int | None = None, # override every task's fixture max_tokens ) -> BenchmarkRun: # API back-compat: allow `run_benchmark(config)` where config has @@ -47,11 +66,18 @@ async def run_benchmark( if args and not (model or endpoint): cfg = args[0] model = getattr(cfg, "model", None) or model - endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "base_url", None) or endpoint + endpoint = ( + getattr(cfg, "endpoint", None) or getattr(cfg, "base_url", None) or endpoint + ) provider = getattr(cfg, "provider", None) or provider cfg_suites = getattr(cfg, "suite_names", None) or getattr(cfg, "suites", None) suites = cfg_suites or suites harness = getattr(cfg, "harness", None) or harness + profile = getattr(cfg, "profile", None) or profile + runs = getattr(cfg, "runs", None) or getattr(cfg, "trials", None) or runs + cfg_remote = getattr(cfg, "remote", None) + if cfg_remote is not None: + remote = bool(cfg_remote) max_tokens = getattr(cfg, "max_tokens", None) or max_tokens elif suite_names and not suites: suites = suite_names @@ -61,14 +87,17 @@ async def run_benchmark( if provider not in PROVIDER_REGISTRY: raise ValueError(f"Unsupported provider: {provider}") - # Auto-detect remote if not explicitly set - if not remote: - endpoint_host = _endpoint_host(endpoint) - if endpoint_host and endpoint_host not in {"localhost", "127.0.0.1", "::1", ""}: - remote = True + # Remote means hosted/cloud scoring, not merely "another host." Private, + # LAN, and Tailscale endpoints are local hardware for benchmark purposes. + if remote is None: + remote = endpoint_is_cloud(endpoint) provider_module = PROVIDER_REGISTRY[provider] - selected_suites = suites or DEFAULT_SUITES + selected_suites = resolve_suites(profile, suites) + benchmark_profile = classify_suites(selected_suites) + coverage_profile = benchmark_profile if benchmark_profile != "custom" else profile + coverage = profile_coverage(coverage_profile, selected_suites) + speed_trials = max(1, int(runs or SPEED_TRIALS)) hardware = detect_hardware(endpoint=endpoint) machine_kwargs = { @@ -92,7 +121,9 @@ async def run_benchmark( available_models = await provider_module.list_models(endpoint) if available_models and model not in available_models: - raise ValueError(f"Model '{model}' not found on {endpoint}. Available: {', '.join(available_models)}") + raise ValueError( + f"Model '{model}' not found on {endpoint}. Available: {', '.join(available_models)}" + ) run_started = time.perf_counter() await provider_module.chat( @@ -111,8 +142,15 @@ async def run_benchmark( machine=machine, provider=provider, harness=harness, - harness_version=getattr(harness_adapter, 'version', ''), + harness_version=getattr(harness_adapter, "version", ""), is_remote=remote, + benchmark_id=BENCHMARK_ID, + benchmark_version=BENCHMARK_VERSION, + benchmark_profile=benchmark_profile, + requested_profile=profile, + score_schema_version=SCORE_SCHEMA_VERSION, + coverage_score=coverage, + comparable=is_comparable_profile(benchmark_profile, coverage), ) speed_metric_samples: list[SpeedMetrics] = [] @@ -120,34 +158,48 @@ async def run_benchmark( # Pre-compute suite_task_counts for live API consumers. total_tasks_all = 0 suite_task_counts: dict[str, int] = {} + tasks_by_suite: dict[str, list[Any]] = {} for sn in selected_suites: - if sn in SUITE_REGISTRY: - try: - _tasks_preview = await SUITE_REGISTRY[sn]().load_tasks() - suite_task_counts[sn] = len(_tasks_preview) - total_tasks_all += len(_tasks_preview) - except Exception: - suite_task_counts[sn] = 0 + if sn not in SUITE_REGISTRY: + raise ValueError(f"Unknown suite: {sn}") + tasks_by_suite[sn] = await SUITE_REGISTRY[sn]().load_tasks() + suite_task_counts[sn] = len(tasks_by_suite[sn]) + total_tasks_all += len(tasks_by_suite[sn]) + run.manifest_hash = manifest_hash( + requested_profile=coverage_profile, + selected_suites=selected_suites, + tasks_by_suite=tasks_by_suite, + ) if on_progress: try: - on_progress({ - "type": "run_started", - "total_tasks": total_tasks_all, - "suites": list(selected_suites), - "suite_task_counts": suite_task_counts, - }) + on_progress( + { + "type": "run_started", + "total_tasks": total_tasks_all, + "suites": list(selected_suites), + "suite_task_counts": suite_task_counts, + "benchmark_profile": benchmark_profile, + "requested_profile": profile, + "manifest_hash": run.manifest_hash, + "speed_trials": speed_trials, + } + ) except Exception: pass completed_so_far = 0 for suite_name in selected_suites: - if suite_name not in SUITE_REGISTRY: - raise ValueError(f"Unknown suite: {suite_name}") suite = SUITE_REGISTRY[suite_name]() - tasks = await suite.load_tasks() + tasks = tasks_by_suite[suite_name] if on_progress: try: - on_progress({"type": "suite_started", "suite": suite_name, "task_count": len(tasks)}) + on_progress( + { + "type": "suite_started", + "suite": suite_name, + "task_count": len(tasks), + } + ) except Exception: pass task_results: list[TaskResult] = [] @@ -163,7 +215,7 @@ async def run_benchmark( harness=harness_adapter, provider_name=provider, remote=remote, - max_tokens_override=None, + trials=speed_trials, ) else: result = await suite.run_task( @@ -176,23 +228,29 @@ async def run_benchmark( max_tokens_override=max_tokens, ) task_results.append(result) - speed_meta = result.metadata.get("speed_metrics") if isinstance(result.metadata, dict) else None + speed_meta = ( + result.metadata.get("speed_metrics") + if isinstance(result.metadata, dict) + else None + ) if isinstance(speed_meta, dict): speed_metric_samples.append(SpeedMetrics(**speed_meta)) completed_so_far += 1 if on_progress: try: - on_progress({ - "type": "task_completed", - "suite": suite_name, - "task_id": result.task_id, - "score": result.score, - "passed": result.passed, - "latency_ms": result.latency_ms, - "error": result.error, - "completed_tasks": completed_so_far, - "total_tasks": total_tasks_all, - }) + on_progress( + { + "type": "task_completed", + "suite": suite_name, + "task_id": result.task_id, + "score": result.score, + "passed": result.passed, + "latency_ms": result.latency_ms, + "error": result.error, + "completed_tasks": completed_so_far, + "total_tasks": total_tasks_all, + } + ) except Exception: pass @@ -211,38 +269,57 @@ async def run_benchmark( run.suites[suite_name] = suite_result if on_progress: try: - on_progress({ - "type": "suite_completed", - "suite": suite_name, - "score": suite_result.score, - "pass_count": suite_result.pass_count, - "task_count": suite_result.task_count, - }) + on_progress( + { + "type": "suite_completed", + "suite": suite_name, + "score": suite_result.score, + "pass_count": suite_result.pass_count, + "task_count": suite_result.task_count, + } + ) except Exception: pass run.total_runtime_sec = time.perf_counter() - run_started if speed_metric_samples: + generation_samples = [ + item.generation_tok_per_sec for item in speed_metric_samples + ] + ttft_samples = [ + item.ttft_ms for item in speed_metric_samples if item.ttft_ms > 0 + ] run.speed_metrics = SpeedMetrics( - ttft_ms=sum(item.ttft_ms for item in speed_metric_samples) / len(speed_metric_samples), - prompt_eval_tok_per_sec=sum(item.prompt_eval_tok_per_sec for item in speed_metric_samples) - / len(speed_metric_samples), - generation_tok_per_sec=sum(item.generation_tok_per_sec for item in speed_metric_samples) - / len(speed_metric_samples), - total_latency_ms=sum(item.total_latency_ms for item in speed_metric_samples) - / len(speed_metric_samples), + ttft_ms=median(ttft_samples) if ttft_samples else 0.0, + prompt_eval_tok_per_sec=median( + item.prompt_eval_tok_per_sec for item in speed_metric_samples + ), + generation_tok_per_sec=median(generation_samples), + total_latency_ms=median( + item.total_latency_ms for item in speed_metric_samples + ), + generation_tok_per_sec_p50=median(generation_samples), + generation_tok_per_sec_p95=_percentile(generation_samples, 0.95), + ttft_ms_p50=median(ttft_samples) if ttft_samples else 0.0, + ttft_ms_p95=_percentile(ttft_samples, 0.95), + sample_count=len(speed_metric_samples), ) run.compute_aggregates() if on_progress: try: - on_progress({ - "type": "run_completed", - "overall_score": run.overall_score, - "quality_score": run.quality_score, - "speed_score": run.speed_score, - "reliability_score": run.reliability_score, - "total_runtime_sec": run.total_runtime_sec, - }) + on_progress( + { + "type": "run_completed", + "overall_score": run.overall_score, + "quality_score": run.quality_score, + "speed_score": run.speed_score, + "reliability_score": run.reliability_score, + "total_runtime_sec": run.total_runtime_sec, + "benchmark_profile": run.benchmark_profile, + "coverage_score": run.coverage_score, + "comparable": run.comparable, + } + ) except Exception: pass return run @@ -257,7 +334,7 @@ async def _run_speed_task( harness: Any | None = None, provider_name: str = "ollama", remote: bool = False, - max_tokens_override: int | None = None, + trials: int = SPEED_TRIALS, ) -> TaskResult: trial_results: list[TaskResult] = [] request = ( @@ -265,31 +342,39 @@ async def _run_speed_task( if harness is not None else {"messages": task.messages, **task.config} ) - if max_tokens_override is not None: - request["max_tokens"] = max_tokens_override # Use streaming for remote/cloud to get real TTFT + tok/s use_streaming = remote and hasattr(provider_module, "chat_streaming") - for _ in range(SPEED_TRIALS): - if use_streaming: - response = await provider_module.chat_streaming( - endpoint=endpoint, - model=model, - **request, - ) - else: - response = await provider_module.chat( - endpoint=endpoint, - model=model, - **request, - ) - if harness is not None: - response = harness.postprocess(response, task) + for _ in range(trials): + started = suite.now_ms() + try: + if use_streaming: + response = await provider_module.chat_streaming( + endpoint=endpoint, + model=model, + **request, + ) + else: + response = await provider_module.chat( + endpoint=endpoint, + model=model, + **request, + ) + if harness is not None: + response = harness.postprocess(response, task) + except Exception as exc: # preserve the run and score this as execution failure + response = suite.runtime_error_response(exc, suite.now_ms() - started) + # Never infer cloud/local from the presence of TTFT: local Ollama and + # llama.cpp responses can expose TTFT-like timing fields too. + response["_benchloop_remote"] = remote trial_results.append(suite.evaluate(task, response)) scored_trials = trial_results[1:] if len(trial_results) > 1 else trial_results - selected = min(scored_trials, key=lambda item: abs(item.score - median(t.score for t in scored_trials))) + selected = min( + scored_trials, + key=lambda item: abs(item.score - median(t.score for t in scored_trials)), + ) selected.metadata = { **selected.metadata, "trials": [ @@ -307,7 +392,9 @@ async def _run_speed_task( for index, result in enumerate(trial_results) ], "selected_trial": next( - index + 1 for index, result in enumerate(trial_results) if result is selected + index + 1 + for index, result in enumerate(trial_results) + if result is selected ), "warmup_dropped": len(trial_results) > 1, "selection_method": "median_of_post_warmup_trials", @@ -315,7 +402,38 @@ async def _run_speed_task( return selected - def _endpoint_host(endpoint: str) -> str: parsed = urlparse(endpoint if "://" in endpoint else f"http://{endpoint}") return (parsed.hostname or "").strip().lower() + + +def endpoint_is_cloud(endpoint: str) -> bool: + host = _endpoint_host(endpoint) + if not host or host == "localhost": + return False + try: + address = ipaddress.ip_address(host) + tailscale_cgnat = address in ipaddress.ip_network("100.64.0.0/10") + return not ( + address.is_loopback + or address.is_private + or address.is_link_local + or tailscale_cgnat + ) + except ValueError: + local_suffixes = (".local", ".lan", ".internal", ".tailnet", ".ts.net") + return "." in host and not host.endswith(local_suffixes) + + +def _percentile(values: list[float], percentile: float) -> float: + """Linear percentile with deterministic behavior for short sample lists.""" + if not values: + return 0.0 + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction diff --git a/bench_loop/runner/result_writer.py b/bench_loop/runner/result_writer.py index c57aa67..583c247 100644 --- a/bench_loop/runner/result_writer.py +++ b/bench_loop/runner/result_writer.py @@ -1,4 +1,5 @@ """Persist benchmark results.""" + from __future__ import annotations import json @@ -11,26 +12,45 @@ import httpx from rich.console import Console +from bench_loop.benchmark_manifest import ( + BENCHMARK_ID, + BENCHMARK_VERSION, + DEFAULT_PROFILE, + SCORE_SCHEMA_VERSION, + classify_suites, + profile_coverage, + resolve_suites, +) from bench_loop.hardware import detect_hardware from bench_loop.models import BenchmarkRun - RUNS_DIR = Path.home() / ".bench-loop" / "runs" # Public leaderboard submit endpoint. Set BENCHLOOP_NO_SUBMIT=1 to disable. LEADERBOARD_SUBMIT_URL = os.environ.get( "BENCHLOOP_SUBMIT_URL", "https://api.bench-loop.com/submit" ) -_SUBMIT_DISABLED = os.environ.get("BENCHLOOP_NO_SUBMIT", "").lower() in {"1", "true", "yes"} +_SUBMIT_DISABLED = os.environ.get("BENCHLOOP_NO_SUBMIT", "").lower() in { + "1", + "true", + "yes", +} def _coalesce_profile(publish_profile: dict | None = None) -> dict[str, str]: raw = { - "name": (publish_profile or {}).get("name") or os.environ.get("BENCHLOOP_PROFILE_NAME", ""), - "avatar_url": (publish_profile or {}).get("avatar_url") or os.environ.get("BENCHLOOP_PROFILE_AVATAR_URL", ""), - "profile_url": (publish_profile or {}).get("profile_url") or os.environ.get("BENCHLOOP_PROFILE_URL", ""), + "name": (publish_profile or {}).get("name") + or os.environ.get("BENCHLOOP_PROFILE_NAME", ""), + "avatar_url": (publish_profile or {}).get("avatar_url") + or os.environ.get("BENCHLOOP_PROFILE_AVATAR_URL", ""), + "profile_url": (publish_profile or {}).get("profile_url") + or os.environ.get("BENCHLOOP_PROFILE_URL", ""), + } + return { + key: str(value).strip() + for key, value in raw.items() + if str(value or "").strip() } - return {key: str(value).strip() for key, value in raw.items() if str(value or "").strip()} def _submit_to_leaderboard(payload: dict, console: Console) -> None: @@ -47,7 +67,7 @@ def _submit_to_leaderboard(payload: dict, console: Console) -> None: resp = client.post(LEADERBOARD_SUBMIT_URL, json=payload) if resp.status_code == 200: console.print( - f"[dim green]→ published to https://bench-loop.com/leaderboard[/dim green]" + "[dim green]→ published to https://bench-loop.com/leaderboard[/dim green]" ) else: console.print( @@ -87,7 +107,10 @@ def save_run( console = console or Console() timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") endpoint_id = _endpoint_identifier(endpoint) - run_dir = RUNS_DIR / f"{timestamp}-{_slugify(run.model.model_id)}-{endpoint_id}-{_slugify(run.provider)}" + run_dir = ( + RUNS_DIR + / f"{timestamp}-{_slugify(run.model.model_id)}-{endpoint_id}-{_slugify(run.provider)}" + ) run_dir.mkdir(parents=True, exist_ok=True) output_path = run_dir / "run.json" @@ -116,7 +139,7 @@ def save_failed_run( endpoint: str, provider: str, harness: str, - suites: list[str], + suites: list[str] | None, error: str, traceback_text: str | None = None, events: list[dict] | None = None, @@ -124,14 +147,24 @@ def save_failed_run( command_used: str | None = None, console: Console | None = None, user_id: str | None = None, + benchmark_profile: str = DEFAULT_PROFILE, ) -> Path: console = console or Console() timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") endpoint_id = _endpoint_identifier(endpoint) - run_dir = RUNS_DIR / f"{timestamp}-{_slugify(model)}-{endpoint_id}-{_slugify(provider)}-failed" + run_dir = ( + RUNS_DIR + / f"{timestamp}-{_slugify(model)}-{endpoint_id}-{_slugify(provider)}-failed" + ) run_dir.mkdir(parents=True, exist_ok=True) machine = detect_hardware(endpoint=endpoint) + requested_suites = resolve_suites(benchmark_profile, suites) + classified_profile = classify_suites(requested_suites) + coverage_profile = ( + classified_profile if classified_profile != "custom" else benchmark_profile + ) + coverage = profile_coverage(coverage_profile, requested_suites) run_dict = { "run_id": run_id, "status": "failed", @@ -145,7 +178,15 @@ def save_failed_run( "machine": machine, "provider": provider, "harness": harness, - "requested_suites": suites, + "requested_suites": requested_suites, + "benchmark_id": BENCHMARK_ID, + "benchmark_version": BENCHMARK_VERSION, + "benchmark_profile": classified_profile, + "requested_profile": benchmark_profile, + "score_schema_version": SCORE_SCHEMA_VERSION, + "manifest_hash": "", + "coverage_score": coverage, + "comparable": False, "suites": {}, "overall_score": 0, "quality_score": 0, diff --git a/bench_loop/sandbox/__init__.py b/bench_loop/sandbox/__init__.py index 3bc3f95..267a23f 100644 --- a/bench_loop/sandbox/__init__.py +++ b/bench_loop/sandbox/__init__.py @@ -1 +1,9 @@ """Sandbox helpers.""" + +from bench_loop.sandbox.python_runner import ( + SandboxResult, + run_restricted_python, + validate_python, +) + +__all__ = ["SandboxResult", "run_restricted_python", "validate_python"] diff --git a/bench_loop/sandbox/python_runner.py b/bench_loop/sandbox/python_runner.py new file mode 100644 index 0000000..d75605c --- /dev/null +++ b/bench_loop/sandbox/python_runner.py @@ -0,0 +1,341 @@ +"""Restricted execution for model-generated Python. + +Generated code is parsed before execution, receives a reduced builtin set and +an import allowlist, runs in an isolated interpreter and temporary directory, +and is subject to wall-clock and OS resource limits where available. + +This is defense in depth for benchmark fixtures, not a general multi-tenant +container boundary. Public deployments should still run the BenchLoop worker +inside a disposable VM or container. +""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +ALLOWED_IMPORTS = frozenset( + { + "collections", + "csv", + "dataclasses", + "decimal", + "enum", + "functools", + "heapq", + "itertools", + "json", + "math", + "re", + "statistics", + "string", + "time", + "typing", + } +) +DANGEROUS_NAMES = frozenset( + { + "__import__", + "breakpoint", + "compile", + "delattr", + "dir", + "eval", + "exec", + "getattr", + "globals", + "help", + "input", + "locals", + "memoryview", + "open", + "os", + "pathlib", + "setattr", + "socket", + "subprocess", + "sys", + "vars", + } +) +FORBIDDEN_ATTRIBUTES = frozenset( + { + "bltns", + "builtins", + "ctypes", + "modules", + "os", + "pathlib", + "socket", + "subprocess", + "sys", + } +) +MAX_OUTPUT_BYTES = 64 * 1024 + + +@dataclass(frozen=True) +class SandboxResult: + returncode: int | None + stdout: str = "" + stderr: str = "" + timed_out: bool = False + rejected: bool = False + rejection_reason: str = "" + + +class _PolicyValidator(ast.NodeVisitor): + def __init__(self) -> None: + self.violations: list[str] = [] + + def _reject(self, node: ast.AST, reason: str) -> None: + line = getattr(node, "lineno", "?") + self.violations.append(f"line {line}: {reason}") + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + root = alias.name.partition(".")[0] + if root not in ALLOWED_IMPORTS: + self._reject(node, f"import {root!r} is not allowed") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + root = (node.module or "").partition(".")[0] + if node.level or root not in ALLOWED_IMPORTS: + self._reject(node, f"import from {node.module!r} is not allowed") + for alias in node.names: + if alias.name.startswith("_") or alias.name in FORBIDDEN_ATTRIBUTES: + self._reject(node, f"unsafe import {alias.name!r} is not allowed") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + is_private_instance_state = ( + node.attr.startswith("_") + and not node.attr.startswith("__") + and isinstance(node.value, ast.Name) + and node.value.id in {"self", "cls"} + ) + if node.attr in FORBIDDEN_ATTRIBUTES: + self._reject(node, f"attribute access {node.attr!r} is not allowed") + elif node.attr.startswith("_") and not is_private_instance_state: + self._reject(node, f"private attribute access {node.attr!r} is not allowed") + self.generic_visit(node) + + def visit_Name(self, node: ast.Name) -> None: + if isinstance(node.ctx, ast.Load) and node.id in DANGEROUS_NAMES: + self._reject(node, f"builtin {node.id!r} is not allowed") + self.generic_visit(node) + + +def validate_python(code: str) -> list[str]: + try: + tree = ast.parse(code, mode="exec") + except SyntaxError: + return [] # Syntax reporting remains the CodingSuite's responsibility. + validator = _PolicyValidator() + validator.visit(tree) + return validator.violations + + +def run_restricted_python( + model_code: str, + test_code: str, + *, + timeout_sec: float = 10.0, +) -> SandboxResult: + violations = validate_python(model_code) + if violations: + return SandboxResult( + returncode=None, + rejected=True, + rejection_reason="; ".join(violations[:5]), + ) + + wrapper = _build_wrapper(model_code, test_code) + with tempfile.TemporaryDirectory(prefix="bench-loop-coding-") as temp_dir: + script_path = Path(temp_dir) / "runner.py" + script_path.write_text(wrapper, encoding="utf-8") + stdout_path = Path(temp_dir) / "stdout.txt" + stderr_path = Path(temp_dir) / "stderr.txt" + environment = { + "PATH": os.defpath, + "PYTHONHASHSEED": "0", + "PYTHONDONTWRITEBYTECODE": "1", + } + try: + with ( + stdout_path.open("w", encoding="utf-8") as stdout_file, + stderr_path.open("w", encoding="utf-8") as stderr_file, + ): + completed = subprocess.run( + [sys.executable, "-I", "-S", "-B", str(script_path)], + cwd=temp_dir, + env=environment, + stdout=stdout_file, + stderr=stderr_file, + text=True, + timeout=timeout_sec, + check=False, + preexec_fn=_resource_limits if os.name == "posix" else None, + ) + return SandboxResult( + returncode=completed.returncode, + stdout=_read_limited(stdout_path), + stderr=_read_limited(stderr_path), + ) + except subprocess.TimeoutExpired: + return SandboxResult( + returncode=None, + stdout=_read_limited(stdout_path), + stderr=f"Timed out after {timeout_sec:g}s", + timed_out=True, + ) + + +def _read_limited(path: Path) -> str: + if not path.exists(): + return "" + with path.open("r", encoding="utf-8", errors="replace") as handle: + return handle.read(MAX_OUTPUT_BYTES) + + +def _resource_limits() -> None: + import resource + + limits = [ + (resource.RLIMIT_CPU, 5), + (resource.RLIMIT_FSIZE, MAX_OUTPUT_BYTES), + (resource.RLIMIT_NOFILE, 32), + ] + if hasattr(resource, "RLIMIT_NPROC"): + limits.append((resource.RLIMIT_NPROC, 16)) + # RLIMIT_AS is reliable on Linux; on macOS the interpreter reserves a + # large virtual address space before user code starts. + if sys.platform.startswith("linux") and hasattr(resource, "RLIMIT_AS"): + limits.append((resource.RLIMIT_AS, 512 * 1024 * 1024)) + for kind, value in limits: + try: + resource.setrlimit(kind, (value, value)) + except (OSError, ValueError): + continue + + +def _build_wrapper(model_code: str, test_code: str) -> str: + allowed_imports = repr(sorted(ALLOWED_IMPORTS)) + safe_names = repr( + [ + "ArithmeticError", + "AssertionError", + "AttributeError", + "BaseException", + "Exception", + "IndexError", + "KeyError", + "LookupError", + "NotImplementedError", + "OverflowError", + "RuntimeError", + "StopIteration", + "TypeError", + "ValueError", + "ZeroDivisionError", + "__build_class__", + "abs", + "all", + "any", + "bin", + "bool", + "bytearray", + "bytes", + "callable", + "chr", + "classmethod", + "complex", + "dict", + "divmod", + "enumerate", + "filter", + "float", + "format", + "frozenset", + "hash", + "hex", + "int", + "isinstance", + "issubclass", + "iter", + "len", + "list", + "map", + "max", + "min", + "next", + "object", + "oct", + "ord", + "pow", + "property", + "range", + "repr", + "reversed", + "round", + "set", + "slice", + "sorted", + "staticmethod", + "str", + "sum", + "super", + "tuple", + "type", + "zip", + ] + ) + return f"""\ +import builtins as _builtins + +_ALLOWED_IMPORTS = frozenset({allowed_imports}) +_SAFE_NAMES = {safe_names} +_real_import = _builtins.__import__ +_module_type = type(_builtins) + +def _safe_import(name, globals=None, locals=None, fromlist=(), level=0): + root = name.partition(".")[0] + if level or root not in _ALLOWED_IMPORTS: + raise ImportError("import is not allowed in BenchLoop coding evaluation: " + name) + module = _real_import(name, globals, locals, fromlist, level) + for imported_name in fromlist or (): + value = getattr(module, imported_name, None) + if isinstance(value, _module_type): + imported_root = value.__name__.partition(".")[0] + if imported_root not in _ALLOWED_IMPORTS: + raise ImportError("module re-export is not allowed: " + imported_name) + return module + +_output_bytes = 0 +def _limited_print(*values, sep=" ", end="\\n", file=None, flush=False): + global _output_bytes + if file is not None: + raise ValueError("redirected print is not allowed") + rendered = sep.join(str(value) for value in values) + end + _output_bytes += len(rendered.encode("utf-8", errors="replace")) + if _output_bytes > {MAX_OUTPUT_BYTES}: + raise RuntimeError("output limit exceeded") + _builtins.print(rendered, end="", flush=flush) + +_safe_builtins = {{name: getattr(_builtins, name) for name in _SAFE_NAMES}} +_safe_builtins["__import__"] = _safe_import +_safe_builtins["print"] = _limited_print +_namespace = {{"__builtins__": _safe_builtins, "__name__": "__main__"}} + +exec(compile({model_code!r}, "", "exec"), _namespace, _namespace) +exec(compile({test_code!r}, "", "exec"), _namespace, _namespace) +""" + + +__all__ = ["SandboxResult", "run_restricted_python", "validate_python"] diff --git a/bench_loop/suites/__init__.py b/bench_loop/suites/__init__.py index 3f0bc6f..8ab7795 100644 --- a/bench_loop/suites/__init__.py +++ b/bench_loop/suites/__init__.py @@ -1,15 +1,19 @@ """Benchmark suite registry.""" + from __future__ import annotations +from bench_loop.benchmark_manifest import DEFAULT_PROFILE, get_profile from bench_loop.suites.agent import AgentSuite from bench_loop.suites.coding import CodingSuite from bench_loop.suites.dataextract import DataExtractSuite from bench_loop.suites.instructfollow import InstructFollowSuite +from bench_loop.suites.longcontext import LongContextSuite from bench_loop.suites.reasonmath import ReasonMathSuite from bench_loop.suites.speed import SpeedSuite from bench_loop.suites.toolcall import ToolCallSuite -# v2 shipping suites. `coding` runs Python via subprocess sandbox with 10s timeout. +# v3 shipping suites. Coding uses a restricted interpreter process; public +# multi-tenant workers should additionally run BenchLoop in a disposable VM. # `tool_use` remains deferred (lower-quality fixtures than `toolcall`). SUITE_REGISTRY = { "speed": SpeedSuite, @@ -18,26 +22,21 @@ "instructfollow": InstructFollowSuite, "reasonmath": ReasonMathSuite, "coding": CodingSuite, + "longcontext": LongContextSuite, "agent": AgentSuite, } -DEFAULT_SUITES = [ - "speed", - "toolcall", - "coding", - "dataextract", - "instructfollow", - "reasonmath", -] +DEFAULT_SUITES = list(get_profile(DEFAULT_PROFILE).suites) __all__ = [ + "DEFAULT_SUITES", + "SUITE_REGISTRY", "AgentSuite", "CodingSuite", "DataExtractSuite", - "DEFAULT_SUITES", "InstructFollowSuite", + "LongContextSuite", "ReasonMathSuite", "SpeedSuite", - "SUITE_REGISTRY", "ToolCallSuite", ] diff --git a/bench_loop/suites/agent.py b/bench_loop/suites/agent.py index 87d026d..42ae6f4 100644 --- a/bench_loop/suites/agent.py +++ b/bench_loop/suites/agent.py @@ -26,6 +26,7 @@ Each criterion is worth 25 pts; total 100 per task. Average across tasks = suite score. """ + from __future__ import annotations import json @@ -122,7 +123,12 @@ def _tool_reverse(args: dict[str, Any]) -> str: "description": "Evaluate a math expression. Supports +, -, *, /, %, parentheses.", "parameters": { "type": "object", - "properties": {"expression": {"type": "string", "description": "Math expression to evaluate."}}, + "properties": { + "expression": { + "type": "string", + "description": "Math expression to evaluate.", + } + }, "required": ["expression"], }, }, @@ -136,7 +142,11 @@ def _tool_reverse(args: dict[str, Any]) -> str: "type": "object", "properties": { "location": {"type": "string", "description": "City name."}, - "units": {"type": "string", "enum": ["fahrenheit", "celsius"], "default": "fahrenheit"}, + "units": { + "type": "string", + "enum": ["fahrenheit", "celsius"], + "default": "fahrenheit", + }, }, "required": ["location"], }, @@ -149,7 +159,12 @@ def _tool_reverse(args: dict[str, Any]) -> str: "description": "Get the latest stock price for a ticker symbol.", "parameters": { "type": "object", - "properties": {"ticker": {"type": "string", "description": "Ticker symbol, e.g. AAPL."}}, + "properties": { + "ticker": { + "type": "string", + "description": "Ticker symbol, e.g. AAPL.", + } + }, "required": ["ticker"], }, }, @@ -207,7 +222,9 @@ def _needle_matches(needle: Any, final_lower: str, final_stripped: str) -> bool: return True # 2. Stripped match (handles "$3,948" containing "3948") - needle_stripped = needle_str.replace("$", "").replace(",", "").replace("€", "").replace("£", "") + needle_stripped = ( + needle_str.replace("$", "").replace(",", "").replace("€", "").replace("£", "") + ) if needle_stripped in final_stripped: return True @@ -235,7 +252,7 @@ def _needle_matches(needle: Any, final_lower: str, final_stripped: str) -> bool: # --------------------------------------------------------------------------- # @dataclass class AgentTurn: - role: str # "user" | "assistant" | "tool" + role: str # "user" | "assistant" | "tool" content: str tool_calls: list[dict[str, Any]] | None = None tool_name: str | None = None @@ -252,6 +269,11 @@ class AgentTrace: tool_calls_total: int hallucinated_tools: list[str] required_satisfied: bool + model_turns: int + latency_ms: float + tokens_generated: int + tokens_prompt: int + provider_errors: list[str] class AgentSuite(BenchmarkSuite): @@ -277,11 +299,19 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: final_lower = (trace.final_answer or "").lower() # Strip common numeric formatting ($, commas) for fuzzy matching. # This handles models that write "$3,948" when expected_contains has "3948". - final_stripped = final_lower.replace("$", "").replace(",", "").replace("€", "").replace("£", "") + final_stripped = ( + final_lower.replace("$", "") + .replace(",", "") + .replace("€", "") + .replace("£", "") + ) correct_final = bool(expected_contains) and all( - _needle_matches(needle, final_lower, final_stripped) for needle in expected_contains + _needle_matches(needle, final_lower, final_stripped) + for needle in expected_contains ) - efficient = trace.completed and len(trace.turns) <= max_turns * 2 # user+assistant per turn + efficient = ( + trace.completed and len(trace.turns) <= max_turns * 2 + ) # user+assistant per turn no_hallucination = len(trace.hallucinated_tools) == 0 all_required_called = trace.required_satisfied @@ -298,7 +328,13 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: task=task, passed=passed, score=score, - response={"content": trace.final_answer}, + response={ + "content": trace.final_answer, + "total_ms": trace.latency_ms, + "tokens_generated": trace.tokens_generated, + "tokens_prompt": trace.tokens_prompt, + "error": "; ".join(trace.provider_errors), + }, output=trace.final_answer[:500], metadata={ "agent_components": components, @@ -317,6 +353,8 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: "completed": trace.completed, "stop_reason": trace.stop_reason, "max_turns": max_turns, + "model_turns": trace.model_turns, + "provider_errors": trace.provider_errors, }, ) @@ -338,16 +376,22 @@ async def run_task( validation = task.validation or {} max_turns = int(validation.get("max_turns", self.DEFAULT_MAX_TURNS)) allowed_tools = list(validation.get("tools", list(TOOL_SCHEMAS))) - required_calls = list(validation.get("must_call", [])) # [{"name": "...", "args_contains": {...}}] + required_calls = list( + validation.get("must_call", []) + ) # [{"name": "...", "args_contains": {...}}] # Build the tool definitions the model sees this run. - tool_schemas = [TOOL_SCHEMAS[name] for name in allowed_tools if name in TOOL_SCHEMAS] + tool_schemas = [ + TOOL_SCHEMAS[name] for name in allowed_tools if name in TOOL_SCHEMAS + ] # Start the conversation with the task's first user turn. messages = [dict(m) for m in task.messages] turns: list[AgentTurn] = [] for m in messages: - turns.append(AgentTurn(role=m.get("role", "user"), content=str(m.get("content", "")))) + turns.append( + AgentTurn(role=m.get("role", "user"), content=str(m.get("content", ""))) + ) hallucinated: list[str] = [] tool_calls_total = 0 @@ -355,6 +399,11 @@ async def run_task( stop_reason = "max_turns" final_answer = "" completed = False + model_turns = 0 + latency_ms = 0.0 + tokens_generated = 0 + tokens_prompt = 0 + provider_errors: list[str] = [] for turn_idx in range(max_turns): # Prepare a task-like wrapper so the harness can inject its @@ -374,13 +423,27 @@ async def run_task( else {"messages": messages, "tools": tool_schemas, "max_tokens": per_turn_max_tokens} ) - response = await provider_module.chat( - endpoint=endpoint, - model=model, - **request, - ) - if harness is not None: - response = harness.postprocess(response, synthetic_task) + started = self.now_ms() + try: + response = await provider_module.chat( + endpoint=endpoint, + model=model, + **request, + ) + if harness is not None: + response = harness.postprocess(response, synthetic_task) + except Exception as exc: # preserve the trace and reliability signal + response = self.runtime_error_response(exc, self.now_ms() - started) + + model_turns += 1 + latency_ms += float(response.get("total_ms") or 0.0) + tokens_generated += int(response.get("tokens_generated") or 0) + tokens_prompt += int(response.get("tokens_prompt") or 0) + provider_error = str(response.get("error") or "").strip() + if provider_error: + provider_errors.append(provider_error) + stop_reason = "provider_error" + break content = (response.get("content") or "").strip() tool_calls = response.get("tool_calls") or [] @@ -432,9 +495,13 @@ def _serialize_args(arg_val: Any) -> Any: ( { "function": { - "name": (call.get("function") or {}).get("name") if isinstance(call, dict) else "", + "name": (call.get("function") or {}).get("name") + if isinstance(call, dict) + else "", "arguments": _serialize_args( - (call.get("function") or {}).get("arguments", {}) if isinstance(call, dict) else {} + (call.get("function") or {}).get("arguments", {}) + if isinstance(call, dict) + else {} ), }, } @@ -443,9 +510,13 @@ def _serialize_args(arg_val: Any) -> Any: "id": f"call_{turn_idx}_{i}", "type": "function", "function": { - "name": (call.get("function") or {}).get("name") if isinstance(call, dict) else "", + "name": (call.get("function") or {}).get("name") + if isinstance(call, dict) + else "", "arguments": _serialize_args( - (call.get("function") or {}).get("arguments", "{}") if isinstance(call, dict) else "{}" + (call.get("function") or {}).get("arguments", "{}") + if isinstance(call, dict) + else "{}" ), }, } @@ -476,22 +547,29 @@ def _serialize_args(arg_val: Any) -> Any: try: tool_result = TOOL_IMPL[name](args) except Exception as exc: - tool_result = f"ERROR: tool execution failed: {type(exc).__name__}: {exc}" + tool_result = ( + f"ERROR: tool execution failed: {type(exc).__name__}: {exc}" + ) # Track required calls for req in required_calls: if req.get("name") == name: wanted_args = req.get("args_contains", {}) - if all(str(args.get(k, "")).lower().find(str(v).lower()) >= 0 for k, v in wanted_args.items()): + if all( + str(args.get(k, "")).lower().find(str(v).lower()) >= 0 + for k, v in wanted_args.items() + ): required_seen.add(name) - turns.append(AgentTurn( - role="tool", - content=tool_result, - tool_name=name, - tool_args=args, - tool_result=tool_result, - )) + turns.append( + AgentTurn( + role="tool", + content=tool_result, + tool_name=name, + tool_args=args, + tool_result=tool_result, + ) + ) tool_message: dict[str, Any] = { "role": "tool", "name": name, @@ -505,9 +583,9 @@ def _serialize_args(arg_val: Any) -> Any: # for/else: ran out of turns stop_reason = "max_turns_exceeded" - required_satisfied = ( - len(required_calls) == 0 or {r.get("name") for r in required_calls}.issubset(required_seen) - ) + required_satisfied = len(required_calls) == 0 or { + r.get("name") for r in required_calls + }.issubset(required_seen) trace = AgentTrace( turns=turns, @@ -517,6 +595,11 @@ def _serialize_args(arg_val: Any) -> Any: tool_calls_total=tool_calls_total, hallucinated_tools=hallucinated, required_satisfied=required_satisfied, + model_turns=model_turns, + latency_ms=latency_ms, + tokens_generated=tokens_generated, + tokens_prompt=tokens_prompt, + provider_errors=provider_errors, ) return self.evaluate(task, {"__trace": trace}) diff --git a/bench_loop/suites/base.py b/bench_loop/suites/base.py index 065e613..77cf2f2 100644 --- a/bench_loop/suites/base.py +++ b/bench_loop/suites/base.py @@ -1,4 +1,5 @@ """Base suite helpers.""" + from __future__ import annotations import time @@ -33,9 +34,15 @@ async def load_tasks(self) -> list[BenchmarkTask]: validation.setdefault("scenario_id", str(item.get("id", "")).upper()) metadata = dict(item.get("metadata", {})) - capability_tags = list(item.get("capability_tags", metadata.get("capability_tags", [])) or []) - verifier_type = str(item.get("verifier_type", metadata.get("verifier_type", "")) or "") - difficulty = str(item.get("difficulty", metadata.get("difficulty", "")) or "") + capability_tags = list( + item.get("capability_tags", metadata.get("capability_tags", [])) or [] + ) + verifier_type = str( + item.get("verifier_type", metadata.get("verifier_type", "")) or "" + ) + difficulty = str( + item.get("difficulty", metadata.get("difficulty", "")) or "" + ) expected_turns = item.get("expected_turns", metadata.get("expected_turns")) notes = str(item.get("notes", metadata.get("notes", "")) or "") if capability_tags: @@ -84,20 +91,28 @@ async def run_task( ) if max_tokens_override is not None: request["max_tokens"] = max_tokens_override - response = await provider_module.chat( - endpoint=endpoint, - model=model, - **request, - ) - if harness is not None: - response = harness.postprocess(response, task) + started = self.now_ms() + try: + response = await provider_module.chat( + endpoint=endpoint, + model=model, + **request, + ) + if harness is not None: + response = harness.postprocess(response, task) + except Exception as exc: # noqa: BLE001 - captured as a reliability failure + response = self.runtime_error_response(exc, self.now_ms() - started) return self.evaluate(task, response) def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: raise NotImplementedError def aggregate_score(self, task_results: list[TaskResult]) -> float: - return round(sum(task.score for task in task_results) / len(task_results), 2) if task_results else 0.0 + return ( + round(sum(task.score for task in task_results) / len(task_results), 2) + if task_results + else 0.0 + ) def normalize_text(self, text: str) -> str: return ( @@ -115,7 +130,9 @@ def latency_ms(self, response: dict[str, Any]) -> float: return float(response.get("total_ms") or 0.0) def token_counts(self, response: dict[str, Any]) -> tuple[int, int]: - return int(response.get("tokens_generated") or 0), int(response.get("tokens_prompt") or 0) + return int(response.get("tokens_generated") or 0), int( + response.get("tokens_prompt") or 0 + ) def build_result( self, @@ -129,6 +146,10 @@ def build_result( metadata: dict[str, Any] | None = None, ) -> TaskResult: tokens_generated, tokens_prompt = self.token_counts(response) + provider_error = str(response.get("error") or "").strip() + result_metadata = dict(metadata or {}) + if provider_error: + result_metadata["provider_error"] = provider_error return TaskResult( task_id=task.id, suite=self.name, @@ -137,10 +158,26 @@ def build_result( latency_ms=self.latency_ms(response), tokens_generated=tokens_generated, tokens_prompt=tokens_prompt, - error=error, + error=error or provider_error, output=output[:500], - metadata=metadata or {}, + execution_ok=not bool(provider_error), + metadata=result_metadata, ) def now_ms(self) -> float: return time.perf_counter() * 1000.0 + + def runtime_error_response( + self, exc: Exception, latency_ms: float + ) -> dict[str, Any]: + return { + "content": "", + "tool_calls": [], + "tokens_generated": 0, + "tokens_prompt": 0, + "total_ms": max(0.0, latency_ms), + "generation_tok_per_sec": 0.0, + "prompt_eval_tok_per_sec": 0.0, + "ttft_ms": 0.0, + "error": f"{type(exc).__name__}: {exc}", + } diff --git a/bench_loop/suites/coding.py b/bench_loop/suites/coding.py index 11971f5..66eeebb 100644 --- a/bench_loop/suites/coding.py +++ b/bench_loop/suites/coding.py @@ -1,18 +1,16 @@ """Coding suite execution and evaluation.""" + from __future__ import annotations import re -import subprocess -import sys -import tempfile from pathlib import Path from typing import Any from bench_loop.config import TASKS_DIR from bench_loop.models import BenchmarkTask, TaskResult +from bench_loop.sandbox import run_restricted_python from bench_loop.suites.base import BenchmarkSuite - CODE_BLOCK_RE = re.compile(r"```python\s*(.*?)```", re.DOTALL | re.IGNORECASE) GENERIC_BLOCK_RE = re.compile(r"```\s*(.*?)```", re.DOTALL) @@ -58,26 +56,16 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: metadata={"evaluation_status": "syntax_error"}, ) - script = f"{code}\n\n{test_code}\n" - with tempfile.TemporaryDirectory(prefix="bench-loop-coding-") as temp_dir: - script_path = Path(temp_dir) / "task.py" - script_path.write_text(script, encoding="utf-8") - try: - completed = subprocess.run( - [sys.executable, str(script_path)], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - stdout = completed.stdout or "" - stderr = completed.stderr or "" - except subprocess.TimeoutExpired: - stdout = "" - stderr = "Timed out after 10s" - completed = None + sandbox_result = run_restricted_python(code, test_code, timeout_sec=10.0) + stdout = sandbox_result.stdout + stderr = sandbox_result.stderr - if completed is not None and completed.returncode == 0 and "PASS" in stdout: + if sandbox_result.rejected: + passed = False + score = 0.0 + error = f"Sandbox policy rejected code: {sandbox_result.rejection_reason}" + status = "sandbox_policy_rejected" + elif sandbox_result.returncode == 0 and "PASS" in stdout: passed = True score = 100.0 error = "" @@ -86,15 +74,15 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: passed = False score = 25.0 status = "tests_failed_or_runtime_error" - combined_error = (stderr.strip() or stdout.strip() or "") - if completed is None: + combined_error = stderr.strip() or stdout.strip() or "" + if sandbox_result.timed_out: error = stderr elif "SyntaxError" in combined_error: score = 0.0 status = "syntax_error" error = combined_error - elif completed.returncode != 0: - error = combined_error or f"exit code {completed.returncode}" + elif sandbox_result.returncode != 0: + error = combined_error or f"exit code {sandbox_result.returncode}" else: error = combined_error or "Tests did not report PASS" @@ -109,5 +97,8 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: "stdout": stdout, "stderr": stderr, "evaluation_status": status, + "execution_mode": "restricted_python", + "policy_rejected": sandbox_result.rejected, + "timed_out": sandbox_result.timed_out, }, ) diff --git a/bench_loop/suites/longcontext.py b/bench_loop/suites/longcontext.py new file mode 100644 index 0000000..db95ca9 --- /dev/null +++ b/bench_loop/suites/longcontext.py @@ -0,0 +1,79 @@ +"""Deterministic long-context retrieval suite.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from bench_loop.config import TASKS_DIR +from bench_loop.models import BenchmarkTask, TaskResult +from bench_loop.suites.base import BenchmarkSuite + + +class LongContextSuite(BenchmarkSuite): + """Needle retrieval at several approximate prompt-token tiers. + + Tokenizers differ, so the fixture declares an approximate target and each + result also records the provider-reported prompt-token count when present. + """ + + name = "longcontext" + task_file = Path(TASKS_DIR) / "longcontext" / "tasks.yaml" + + async def load_tasks(self) -> list[BenchmarkTask]: + tasks = await super().load_tasks() + for task in tasks: + target_tokens = int(task.metadata.get("target_context_tokens") or 0) + needle = str(task.validation.get("expected") or "") + position = float(task.metadata.get("needle_position") or 0.5) + context = self._generate_context(target_tokens, needle, position) + for message in task.messages: + message["content"] = message["content"].replace("{{CONTEXT}}", context) + task.metadata["generated_context_chars"] = len(context) + return tasks + + @staticmethod + def _generate_context(target_tokens: int, needle: str, position: float) -> str: + target_chars = max(1_000, target_tokens * 4) + lines: list[str] = [] + index = 0 + generated_chars = 0 + while generated_chars < target_chars: + checksum = (index * 7919 + 104729) % 1_000_003 + line = ( + f"Archive entry {index:05d}: routine telemetry was nominal; " + f"the unrelated checksum was {checksum:06d}." + ) + lines.append(line) + generated_chars += len(line) + 1 + index += 1 + insertion = min(len(lines), max(0, round(len(lines) * position))) + lines.insert( + insertion, + f"AUTHORITATIVE RECORD: Project Aurora's access code is {needle}. " + "This record supersedes all unrelated checksums.", + ) + return "\n".join(lines) + + def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: + output = self.response_text(response).strip() + expected = str(task.validation.get("expected") or "") + passed = bool(expected) and expected.casefold() in output.casefold() + return self.build_result( + task=task, + passed=passed, + score=100.0 if passed else 0.0, + response=response, + output=output, + error="" if passed else "Expected access code was not recovered", + metadata={ + "target_context_tokens": int( + task.metadata.get("target_context_tokens") or 0 + ), + "generated_context_chars": int( + task.metadata.get("generated_context_chars") or 0 + ), + "needle_position": float(task.metadata.get("needle_position") or 0.0), + "provider_prompt_tokens": int(response.get("tokens_prompt") or 0), + }, + ) diff --git a/bench_loop/suites/speed.py b/bench_loop/suites/speed.py index b4e05e0..5273602 100644 --- a/bench_loop/suites/speed.py +++ b/bench_loop/suites/speed.py @@ -1,4 +1,5 @@ """Speed suite fixture loader and evaluation.""" + from __future__ import annotations import math @@ -25,7 +26,7 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: total_latency_ms=float(response.get("total_ms") or 0.0), ) - is_cloud = ttft_ms > 0 and generation_tok_per_sec > 0 + is_cloud = bool(response.get("_benchloop_remote", False)) if is_cloud: score = self._cloud_speed_score(generation_tok_per_sec, ttft_ms) @@ -59,6 +60,7 @@ def evaluate(self, task: BenchmarkTask, response: dict[str, Any]) -> TaskResult: "prompt_eval_duration": int(response.get("prompt_eval_duration") or 0), "load_duration": int(response.get("load_duration") or 0), "is_cloud_speed": is_cloud, + "speed_mode": "remote" if is_cloud else "local", }, ) diff --git a/bench_loop/tasks/longcontext/tasks.yaml b/bench_loop/tasks/longcontext/tasks.yaml new file mode 100644 index 0000000..016e6a4 --- /dev/null +++ b/bench_loop/tasks/longcontext/tasks.yaml @@ -0,0 +1,64 @@ +tasks: + - id: longcontext-2k-early + suite: longcontext + title: "2K retrieval: early needle" + difficulty: easy + capability_tags: [long-context, retrieval, 2k] + messages: + - role: user + content: | + Read the archive below. Ignore routine checksums. What is Project Aurora's access code? + Return only the access code. + + {{CONTEXT}} + config: {max_tokens: 32, temperature: 0.0} + validation: {expected: "AURORA-2K-7319"} + metadata: {target_context_tokens: 2048, needle_position: 0.1} + + - id: longcontext-8k-middle + suite: longcontext + title: "8K retrieval: middle needle" + difficulty: medium + capability_tags: [long-context, retrieval, 8k] + messages: + - role: user + content: | + Read the archive below. Ignore routine checksums. What is Project Aurora's access code? + Return only the access code. + + {{CONTEXT}} + config: {max_tokens: 32, temperature: 0.0} + validation: {expected: "AURORA-8K-2841"} + metadata: {target_context_tokens: 8192, needle_position: 0.5} + + - id: longcontext-16k-late + suite: longcontext + title: "16K retrieval: late needle" + difficulty: hard + capability_tags: [long-context, retrieval, 16k] + messages: + - role: user + content: | + Read the archive below. Ignore routine checksums. What is Project Aurora's access code? + Return only the access code. + + {{CONTEXT}} + config: {max_tokens: 32, temperature: 0.0} + validation: {expected: "AURORA-16K-9157"} + metadata: {target_context_tokens: 16384, needle_position: 0.9} + + - id: longcontext-32k-middle + suite: longcontext + title: "32K retrieval: middle needle" + difficulty: hard + capability_tags: [long-context, retrieval, 32k] + messages: + - role: user + content: | + Read the archive below. Ignore routine checksums. What is Project Aurora's access code? + Return only the access code. + + {{CONTEXT}} + config: {max_tokens: 32, temperature: 0.0} + validation: {expected: "AURORA-32K-4603"} + metadata: {target_context_tokens: 32768, needle_position: 0.5} diff --git a/docs/BENCHMARK_SPEC_V3.md b/docs/BENCHMARK_SPEC_V3.md new file mode 100644 index 0000000..9e3d169 --- /dev/null +++ b/docs/BENCHMARK_SPEC_V3.md @@ -0,0 +1,105 @@ +# BenchLoop Benchmark Specification v3 + +BenchLoop v3 makes benchmark results identifiable and reproducible. A result is +not just a score: it records the benchmark version, score schema, named profile, +coverage, and a SHA-256 manifest of every prompt, generation setting, and +validator used in the run. + +## Profiles + +| Profile | Suites | Tasks | Intended use | +|---|---|---:|---| +| `smoke` | speed, toolcall, reasonmath | 39 | Endpoint and quality sanity check | +| `core` | speed, toolcall, coding, dataextract, instructfollow, reasonmath | 81 | Default comparable daily-driver run | +| `full` | core + longcontext + agent | 93 | Capability audit including 2K–32K retrieval and multi-turn tools | + +Supplying `--suites` creates a `custom` run unless the suite set exactly matches +a named profile. Custom and partial runs retain their component scores but are +marked `comparable=false`; they must not be presented as complete profile runs. + +## Provenance fields + +Each `run.json` includes: + +- `benchmark_id` and `benchmark_version` +- `benchmark_profile` and `requested_profile` +- `score_schema_version` +- `manifest_hash` +- `coverage_score` and `comparable` + +Scores should only be compared directly when benchmark version, score schema, +profile, and manifest hash match. Harness, provider mode, sampling settings, and +hardware must remain visible dimensions of the comparison. + +## Scoring + +Quality uses fixed, declared weights instead of changing every time a suite is +added or removed. Core weights are: + +| Suite | Weight | +|---|---:| +| toolcall | 20% | +| coding | 25% | +| dataextract | 15% | +| instructfollow | 15% | +| reasonmath | 25% | + +Full gives agent 20% and long-context 15%, with the remaining 65% distributed +across the core quality domains as declared in `benchmark_manifest.py`. + +The v3 composite is: + +```text +overall = 0.70 × quality + 0.25 × speed + 0.05 × reliability +``` + +When no speed suite is present: + +```text +overall = 0.90 × quality + 0.10 × reliability +``` + +Reliability means provider/runtime execution success. It is intentionally not +the task pass rate; task correctness is already represented by quality and must +not be counted twice. + +## Speed protocol + +- Each speed prompt runs three trials by default (`--trials` changes this). +- Trial 1 is warmup when more than one trial is requested. +- The representative post-warmup trial is the median-scoring trial. +- Run-level throughput and latency are medians; p50/p95 and sample count are + persisted alongside the headline values. +- Local and remote/cloud curves are selected from explicit run mode. Timing + fields returned by a local engine never silently switch the scoring curve. +- The speed fixtures vary output length. Long-context prefill and retrieval are + measured separately by the `longcontext` suite. + +## Long-context protocol + +The full profile performs deterministic needle retrieval at approximate 2K, +8K, 16K, and 32K context tiers and records both target size and provider-reported +prompt tokens. Approximate tiers are used because tokenization differs across +model families. The exact generated context is included in the manifest hash. + +## Generated-code execution + +Coding responses are never run as unrestricted host Python. The evaluator: + +- parses the AST and blocks non-allowlisted imports, filesystem/process/network + primitives, private reflection, and dynamic-code builtins; +- executes with reduced builtins and an import allowlist; +- uses isolated interpreter flags, a fresh temporary working directory, a + minimal environment, wall-clock timeout, output cap, and POSIX resource caps. + +This is defense in depth for local benchmark fixtures, not a general hostile +multi-tenant security boundary. A public BenchLoop worker should additionally +run inside a disposable VM or container. + +## Known limits + +The fixtures and deterministic validators are public, so v3 is a reproducible +engineering benchmark rather than a secret contamination-proof evaluation. +Vision, energy-per-token, peak GPU memory, and seeded private challenge sets are +recommended future profile additions. They should receive new benchmark and +score-schema versions instead of changing historical scores in place. diff --git a/pyproject.toml b/pyproject.toml index ed7d3a4..dc7c32c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "benchloop-cli" -version = "0.2.3" +version = "0.3.0" description = "Local-first CLI + web dashboard for benchmarking LLMs across quality, speed, reliability, and a real multi-turn agent loop. Hardware-aware, deterministic, reproducible." # Note: the PyPI name `benchloop` was already taken by an unrelated dataset # library, so the published distribution is `benchloop-cli`. The installed diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py new file mode 100644 index 0000000..c61c0f6 --- /dev/null +++ b/tests/test_agent_telemetry.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import asyncio + +from bench_loop.models import BenchmarkTask +from bench_loop.suites.agent import AgentSuite + + +class _TwoTurnProvider: + calls = 0 + + @classmethod + async def chat(cls, **_kwargs): + cls.calls += 1 + if cls.calls == 1: + return { + "content": "", + "tool_calls": [ + { + "function": { + "name": "calculator", + "arguments": '{"expression": "2+2"}', + } + } + ], + "total_ms": 100, + "tokens_generated": 3, + "tokens_prompt": 10, + } + return { + "content": "4", + "tool_calls": [], + "total_ms": 200, + "tokens_generated": 2, + "tokens_prompt": 20, + } + + +class _ErrorProvider: + @staticmethod + async def chat(**_kwargs): + return {"content": "", "error": "endpoint unavailable", "total_ms": 50} + + +def _task() -> BenchmarkTask: + return BenchmarkTask( + id="agent-telemetry", + suite="agent", + messages=[{"role": "user", "content": "Calculate 2+2."}], + validation={ + "max_turns": 3, + "tools": ["calculator"], + "must_call": [{"name": "calculator", "args_contains": {}}], + "expected_contains": ["4"], + }, + ) + + +def test_agent_accumulates_metrics_across_model_turns() -> None: + _TwoTurnProvider.calls = 0 + result = asyncio.run( + AgentSuite().run_task( + _TwoTurnProvider, + "http://localhost", + "test-model", + _task(), + provider_name="openai_compat", + ) + ) + assert result.passed is True + assert result.execution_ok is True + assert result.latency_ms == 300 + assert result.tokens_generated == 5 + assert result.tokens_prompt == 30 + assert result.metadata["model_turns"] == 2 + + +def test_agent_provider_failure_is_a_reliability_failure() -> None: + result = asyncio.run( + AgentSuite().run_task( + _ErrorProvider, + "http://localhost", + "test-model", + _task(), + ) + ) + assert result.execution_ok is False + assert result.metadata["stop_reason"] == "provider_error" + assert result.metadata["provider_errors"] == ["endpoint unavailable"] diff --git a/tests/test_benchmark_manifest.py b/tests/test_benchmark_manifest.py new file mode 100644 index 0000000..e432010 --- /dev/null +++ b/tests/test_benchmark_manifest.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from bench_loop.benchmark_manifest import ( + PROFILES, + classify_suites, + manifest_hash, + profile_coverage, + resolve_suites, +) +from bench_loop.models import BenchmarkTask + + +def test_profiles_have_unique_exact_classification() -> None: + for name, profile in PROFILES.items(): + assert classify_suites(profile.suites) == name + assert profile_coverage(name, profile.suites) == 100.0 + + +def test_custom_suite_selection_is_not_mislabeled_full() -> None: + suites = ["speed", "toolcall", "dataextract", "instructfollow", "reasonmath"] + assert classify_suites(suites) == "custom" + assert profile_coverage("core", suites) < 100.0 + + +def test_profile_resolution_preserves_default_and_custom_order() -> None: + assert resolve_suites("smoke") == list(PROFILES["smoke"].suites) + assert resolve_suites("core", ["reasonmath", "speed", "reasonmath"]) == [ + "reasonmath", + "speed", + ] + + +def test_manifest_hash_is_deterministic_and_fixture_sensitive() -> None: + task = BenchmarkTask( + id="example", + suite="reasonmath", + messages=[{"role": "user", "content": "What is 2+2?"}], + validation={"expected": "4"}, + ) + kwargs = { + "requested_profile": "smoke", + "selected_suites": ["reasonmath"], + "tasks_by_suite": {"reasonmath": [task]}, + } + first = manifest_hash(**kwargs) + second = manifest_hash(**kwargs) + assert first == second + assert first.startswith("sha256:") + + changed_task = BenchmarkTask( + id="example", + suite="reasonmath", + messages=[{"role": "user", "content": "What is 2+3?"}], + validation={"expected": "5"}, + ) + changed = manifest_hash( + **{**kwargs, "tasks_by_suite": {"reasonmath": [changed_task]}} + ) + assert changed != first diff --git a/tests/test_endpoint_mode.py b/tests/test_endpoint_mode.py new file mode 100644 index 0000000..32906eb --- /dev/null +++ b/tests/test_endpoint_mode.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import pytest + +from bench_loop.runner.orchestrator import endpoint_is_cloud + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:8080", + "http://127.0.0.1:8080", + "http://192.168.1.96:8080", + "http://100.95.10.4:8080", + "http://luxecorp-pc1:8080", + "http://pc1.example-tailnet.ts.net:8080", + ], +) +def test_local_and_tailscale_endpoints_use_local_mode(endpoint: str) -> None: + assert endpoint_is_cloud(endpoint) is False + + +@pytest.mark.parametrize( + "endpoint", + [ + "https://api.openai.com/v1", + "https://openrouter.ai/api/v1", + "https://inference.example.com/v1", + ], +) +def test_public_api_endpoints_use_cloud_mode(endpoint: str) -> None: + assert endpoint_is_cloud(endpoint) is True diff --git a/tests/test_longcontext.py b/tests/test_longcontext.py new file mode 100644 index 0000000..afa2894 --- /dev/null +++ b/tests/test_longcontext.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import asyncio + +from bench_loop.suites.longcontext import LongContextSuite + + +def test_longcontext_fixtures_expand_deterministically() -> None: + suite = LongContextSuite() + first = asyncio.run(suite.load_tasks()) + second = asyncio.run(suite.load_tasks()) + + assert [task.id for task in first] == [task.id for task in second] + assert len(first) == 4 + for left, right in zip(first, second, strict=True): + left_prompt = left.messages[0]["content"] + right_prompt = right.messages[0]["content"] + expected = left.validation["expected"] + assert left_prompt == right_prompt + assert "{{CONTEXT}}" not in left_prompt + assert left_prompt.count(expected) == 1 + assert ( + left.metadata["generated_context_chars"] + >= left.metadata["target_context_tokens"] * 4 + ) + + +def test_longcontext_scorer_records_context_metadata() -> None: + suite = LongContextSuite() + task = asyncio.run(suite.load_tasks())[0] + result = suite.evaluate( + task, + { + "content": f"The code is {task.validation['expected']}", + "tokens_prompt": 2100, + "tokens_generated": 8, + "total_ms": 100, + }, + ) + assert result.passed is True + assert result.score == 100 + assert result.metadata["provider_prompt_tokens"] == 2100 diff --git a/tests/test_orchestrator_v3.py b/tests/test_orchestrator_v3.py new file mode 100644 index 0000000..f218076 --- /dev/null +++ b/tests/test_orchestrator_v3.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import asyncio + +from bench_loop.runner import orchestrator + + +class _FakeProvider: + calls = 0 + requests: list[dict] = [] + + @staticmethod + async def list_models(_endpoint: str) -> list[str]: + return ["fake-model"] + + @staticmethod + async def get_system_info(endpoint: str) -> dict: + return {"endpoint": endpoint} + + @classmethod + async def chat(cls, **kwargs) -> dict: + cls.calls += 1 + cls.requests.append(kwargs) + return { + "content": "benchmark output", + "tokens_prompt": 20, + "tokens_generated": 40, + "ttft_ms": 25, + "total_ms": 1000, + "generation_tok_per_sec": 42, + "prompt_eval_tok_per_sec": 200, + } + + +def test_orchestrator_stamps_provenance_and_trial_distribution(monkeypatch) -> None: + _FakeProvider.calls = 0 + _FakeProvider.requests = [] + monkeypatch.setitem(orchestrator.PROVIDER_REGISTRY, "fake", _FakeProvider) + run = asyncio.run( + orchestrator.run_benchmark( + model="fake-model", + endpoint="http://100.95.10.4:8080", + provider="fake", + suites=["speed"], + profile="core", + runs=2, + remote=False, + ) + ) + + assert run.benchmark_version == "3.0.0" + assert run.score_schema_version == "3.0.0" + assert run.benchmark_profile == "custom" + assert run.coverage_score == 20.0 + assert run.comparable is False + assert run.manifest_hash.startswith("sha256:") + assert run.is_remote is False + assert run.speed_metrics.generation_tok_per_sec == 42 + assert run.speed_metrics.generation_tok_per_sec_p95 == 42 + assert run.speed_metrics.sample_count == 9 + assert run.reliability_score == 100 + # One global warmup + two trials for each of nine speed prompts. + assert _FakeProvider.calls == 19 + for task in run.suites["speed"].tasks: + assert task.metadata["warmup_dropped"] is True + assert len(task.metadata["trials"]) == 2 + assert task.metadata["speed_mode"] == "local" + + +def test_quality_token_override_never_changes_speed_fixture_caps(monkeypatch) -> None: + _FakeProvider.calls = 0 + _FakeProvider.requests = [] + monkeypatch.setitem(orchestrator.PROVIDER_REGISTRY, "fake", _FakeProvider) + + asyncio.run( + orchestrator.run_benchmark( + model="fake-model", + endpoint="http://localhost:8080", + provider="fake", + suites=["speed"], + runs=1, + max_tokens=8192, + remote=False, + ) + ) + + # The first call is the global warmup. Every subsequent call must retain + # the speed task's own small generation cap instead of the quality override. + speed_caps = [request["max_tokens"] for request in _FakeProvider.requests[1:]] + assert speed_caps == [32, 48, 64, 160, 160, 160, 384, 384, 384] diff --git a/tests/test_result_writer_v3.py b/tests/test_result_writer_v3.py new file mode 100644 index 0000000..f01fc8d --- /dev/null +++ b/tests/test_result_writer_v3.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import json + +from bench_loop.runner import result_writer + + +def test_failed_run_retains_requested_profile_provenance(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(result_writer, "RUNS_DIR", tmp_path) + monkeypatch.setattr( + result_writer, + "detect_hardware", + lambda endpoint: {"machine_id": "test", "endpoint": endpoint}, + ) + output = result_writer.save_failed_run( + run_id="failed-1", + model="test-model", + endpoint="http://localhost:8080", + provider="openai_compat", + harness="raw", + suites=None, + benchmark_profile="full", + error="intentional failure", + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["benchmark_profile"] == "full" + assert payload["requested_profile"] == "full" + assert payload["coverage_score"] == 100 + assert payload["comparable"] is False + assert payload["manifest_hash"] == "" + assert payload["requested_suites"][-2:] == ["longcontext", "agent"] diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100644 index 0000000..a68b832 --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import pytest + +from bench_loop.sandbox import run_restricted_python, validate_python + + +def test_restricted_python_runs_valid_fixture_code() -> None: + result = run_restricted_python( + """ +import csv +def parse_csv(text): + rows = csv.DictReader(text.splitlines()) + return list(rows) +""", + """ +assert parse_csv('name,age\\nAda,36\\n') == [{'name': 'Ada', 'age': '36'}] +print('PASS') +""", + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "PASS" + + +@pytest.mark.parametrize( + "code, expected", + [ + ("open('/tmp/escape', 'w')", "builtin 'open'"), + ("import socket", "import 'socket'"), + ("getattr(object, '__subclasses__')()", "builtin 'getattr'"), + ( + "import collections\ncollections._sys.modules", + "private attribute access '_sys'", + ), + ("from dataclasses import sys\nsys.modules", "unsafe import 'sys'"), + ("import enum\nenum.bltns.open('escape', 'w')", "attribute access 'bltns'"), + ], +) +def test_policy_rejects_host_escape_primitives(code: str, expected: str) -> None: + violations = validate_python(code) + assert any(expected in violation for violation in violations) + result = run_restricted_python(code, "print('PASS')") + assert result.rejected is True + assert result.returncode is None + + +def test_private_instance_state_remains_available() -> None: + result = run_restricted_python( + """ +class Counter: + def __init__(self): + self._value = 0 + def increment(self): + self._value += 1 + return self._value +""", + "assert Counter().increment() == 1\nprint('PASS')", + ) + assert result.returncode == 0, result.stderr diff --git a/tests/test_scoring_v3.py b/tests/test_scoring_v3.py new file mode 100644 index 0000000..ca89607 --- /dev/null +++ b/tests/test_scoring_v3.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from bench_loop.models import BenchmarkRun, SpeedMetrics, SuiteResult, TaskResult + + +def _suite( + name: str, score: float, *, passed: bool = False, execution_ok: bool = True +) -> SuiteResult: + task = TaskResult( + task_id=f"{name}-1", + suite=name, + passed=passed, + score=score, + execution_ok=execution_ok, + ) + return SuiteResult( + suite=name, + score=score, + task_count=1, + pass_count=int(passed), + fail_count=int(not passed), + tasks=[task], + ) + + +def test_v3_weighted_quality_and_execution_reliability() -> None: + run = BenchmarkRun( + requested_profile="core", + speed_metrics=SpeedMetrics(generation_tok_per_sec=50), + suites={ + "speed": _suite("speed", 70), + "toolcall": _suite("toolcall", 80), + "coding": _suite("coding", 60), + "dataextract": _suite("dataextract", 100), + "instructfollow": _suite("instructfollow", 40), + "reasonmath": _suite("reasonmath", 20, execution_ok=False), + }, + ) + run.compute_aggregates() + + assert run.quality_score == 57.0 + # Correctness failures do not masquerade as transport/runtime failures. + assert run.reliability_score == 83.33 + assert run.speed_score == 70.0 + assert run.overall_score == 61.57 + + +def test_missing_speed_uses_quality_reliability_formula() -> None: + run = BenchmarkRun( + requested_profile="smoke", + suites={ + "toolcall": _suite("toolcall", 100, passed=True), + "reasonmath": _suite("reasonmath", 0, passed=False), + }, + ) + run.compute_aggregates() + assert run.quality_score == 45.0 + assert run.reliability_score == 100.0 + assert run.overall_score == 50.5 diff --git a/tests/test_speed_v3.py b/tests/test_speed_v3.py new file mode 100644 index 0000000..c7b36e1 --- /dev/null +++ b/tests/test_speed_v3.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import math + +from bench_loop.models import BenchmarkTask +from bench_loop.runner.orchestrator import _percentile +from bench_loop.suites.speed import SpeedSuite + + +def _task() -> BenchmarkTask: + return BenchmarkTask(id="speed-test", suite="speed", messages=[]) + + +def _response(*, remote: bool) -> dict: + return { + "content": "ok", + "generation_tok_per_sec": 30.0, + "prompt_eval_tok_per_sec": 100.0, + "ttft_ms": 2000.0, + "total_ms": 3000.0, + "_benchloop_remote": remote, + } + + +def test_local_timing_is_never_inferred_as_cloud() -> None: + result = SpeedSuite().evaluate(_task(), _response(remote=False)) + expected = 12.54 * math.log2(30.0) + 0.9 + assert result.score == round(expected, 2) + assert result.metadata["speed_mode"] == "local" + + +def test_remote_mode_uses_cloud_curve_explicitly() -> None: + result = SpeedSuite().evaluate(_task(), _response(remote=True)) + assert result.metadata["speed_mode"] == "remote" + assert result.score != SpeedSuite().evaluate(_task(), _response(remote=False)).score + + +def test_percentiles_are_deterministic_for_short_runs() -> None: + assert _percentile([], 0.95) == 0.0 + assert _percentile([10], 0.95) == 10 + assert _percentile([10, 20, 30], 0.5) == 20 + assert _percentile([10, 20, 30], 0.95) == 29 diff --git a/tests/test_suite_scorers.py b/tests/test_suite_scorers.py new file mode 100644 index 0000000..30ab182 --- /dev/null +++ b/tests/test_suite_scorers.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import asyncio + +from bench_loop.models import BenchmarkTask +from bench_loop.suites.dataextract import DataExtractSuite +from bench_loop.suites.instructfollow import InstructFollowSuite +from bench_loop.suites.reasonmath import ReasonMathSuite +from bench_loop.suites.toolcall import ToolCallSuite + + +def _task(task_id: str, suite: str, *, validation: dict | None = None) -> BenchmarkTask: + return BenchmarkTask( + id=task_id, + suite=suite, + messages=[], + validation=validation or {}, + ) + + +def _call(name: str, arguments: dict) -> dict: + return {"function": {"name": name, "arguments": arguments}} + + +def test_toolcall_multi_call_golden_scores() -> None: + task = _task("tc-09", "toolcall") + complete = ToolCallSuite().evaluate( + task, + { + "content": "", + "tool_calls": [ + _call("get_weather", {"location": "London"}), + _call("get_stock_price", {"ticker": "MSFT"}), + ], + }, + ) + partial = ToolCallSuite().evaluate( + task, + {"content": "", "tool_calls": [_call("get_weather", {"location": "London"})]}, + ) + assert complete.score == 100 + assert complete.passed is True + assert partial.score == 50 + assert partial.passed is False + + +def test_dataextract_atomic_field_golden_scores() -> None: + task = _task( + "de-golden", + "dataextract", + validation={"expected": {"name": "Ada", "age": 36}, "scenario_id": "DE-GOLDEN"}, + ) + exact = DataExtractSuite().evaluate(task, {"content": '{"name":"Ada","age":36}'}) + partial = DataExtractSuite().evaluate(task, {"content": '{"name":"Ada","age":99}'}) + invalid = DataExtractSuite().evaluate(task, {"content": "not json"}) + assert exact.score == 100 + assert partial.score == 50 + assert invalid.score == 0 + + +def test_dataextract_recovers_json_without_relaxing_field_scoring() -> None: + task = _task( + "de-golden", + "dataextract", + validation={"expected": {"name": "Ada", "age": 36}, "scenario_id": "DE-GOLDEN"}, + ) + fenced = DataExtractSuite().evaluate( + task, + {"content": 'Result:\n```json\n{"name":"Ada","age":36}\n```'}, + ) + prose_wrapped = DataExtractSuite().evaluate( + task, + {"content": 'The extracted record is {"name":"Ada","age":99}.'}, + ) + + assert fenced.score == 100 + assert fenced.metadata["json_extraction_method"] == "fenced" + assert prose_wrapped.score == 50 + assert prose_wrapped.metadata["json_extraction_method"] == "bracket_scan" + + +def test_instruction_following_golden_scores() -> None: + task = _task("if-02", "instructfollow") + exact = InstructFollowSuite().evaluate( + task, + {"content": "Oceans cover Earth\nWaves shape rocky shores\nCurrents move heat"}, + ) + partial = InstructFollowSuite().evaluate( + task, + {"content": "Oceans cover Earth\nWaves shape shores\nCurrents move heat"}, + ) + assert exact.score == 100 + assert partial.score == 50 + + +def test_reasonmath_pair_golden_scores() -> None: + task = _task("rm-03", "reasonmath") + exact = ReasonMathSuite().evaluate( + task, + { + "content": "Calculation complete.\nANSWER: new_original_price=100; saved_money=yes" + }, + ) + partial = ReasonMathSuite().evaluate( + task, + {"content": "ANSWER: new_original_price=100; saved_money=no"}, + ) + assert exact.score == 100 + assert partial.score == 50 + + +def test_provider_error_is_separate_from_quality_failure() -> None: + task = _task("if-02", "instructfollow") + result = InstructFollowSuite().evaluate( + task, + {"content": "", "error": "HTTP 500 from endpoint"}, + ) + assert result.passed is False + assert result.execution_ok is False + assert result.metadata["provider_error"] == "HTTP 500 from endpoint" + + +def test_raised_provider_error_becomes_a_task_reliability_failure() -> None: + class RaisingProvider: + @staticmethod + async def chat(**_kwargs): + raise TimeoutError("endpoint stalled") + + task = _task("if-02", "instructfollow") + result = asyncio.run( + InstructFollowSuite().run_task( + RaisingProvider, + "http://localhost", + "test-model", + task, + ) + ) + assert result.execution_ok is False + assert "TimeoutError" in result.metadata["provider_error"]