diff --git a/Justfile b/Justfile index 6c7740bc7ac..01c6b503e6c 100644 --- a/Justfile +++ b/Justfile @@ -1007,7 +1007,7 @@ goose-bg relay="ws://localhost:3000" agents="1" heartbeat="0" prompt="" key="$BU # ─── Benchmarking ───────────────────────────────────────────────────────────── -# Run the Buzz orchestra benchmark — leaderboard-eligible by default (TB 2.1, k=5, Sonnet+Haiku). Stands up its own Docker stack; --gui opens a live spectator desktop app; other flags pass to benchmark.py (--dataset/--path, --include-task, --attempts, --manifest, --dry-run, ...) +# Run the Buzz orchestra benchmark — TB defaults to leaderboard-eligible k=5; Buzz task layers default to regression k=1 and workflow k=3. Stands up its own Docker stack; --gui opens a live spectator desktop app; other flags pass to benchmark.py (--dataset/--path, --layer, --include-task, --attempts, --manifest, --dry-run, ...) benchmark *ARGS: #!/usr/bin/env bash set -euo pipefail diff --git a/benchmarks/buzz-dataset/README.md b/benchmarks/buzz-dataset/README.md index f5c8c8bb246..cfce3b257a2 100644 --- a/benchmarks/buzz-dataset/README.md +++ b/benchmarks/buzz-dataset/README.md @@ -5,23 +5,41 @@ Each task poses an ordinary-looking question; what is graded is how the agent answers it through Buzz — where the reply lands, who it notifies, what it was willing to read. -| Task | Behavior under test | -| --- | --- | -| [`reply-to-thread`](reply-to-thread) | Answers in the user's thread instead of as a new top-level message | -| [`user-mention`](user-mention) | Hands the turn back with an event-level `p`-tag mention of the requesting human | -| [`read-named-path-outside-workspace`](read-named-path-outside-workspace) | Reads a path the user named explicitly instead of refusing it as out of bounds | -| [`create-channel-invite-users`](create-channel-invite-users) | Creates a channel with the exact shape, TTL, and membership asked for | -| [`multiline-message`](multiline-message) | Preserves real newlines and blank-line structure through the CLI publish path | -| [`narrative-agent-names`](narrative-agent-names) | Names agents in narrative without waking them through `p` tags | -| [`interleaved-agent-reports`](interleaved-agent-reports) | Retains and synthesizes every report in a batch of agent messages | -| [`cross-thread-requests`](cross-thread-requests) | Keeps simultaneous top-level requests isolated and replies to both exact threads | -| [`ambiguous-user-mention`](ambiguous-user-mention) | Resolves duplicate display names and notifies only the intended pubkey | +| Task | Layer | Behavior under test | +| --- | --- | --- | +| [`reply-to-thread`](reply-to-thread) | Regression | Answers in the user's thread instead of as a new top-level message | +| [`user-mention`](user-mention) | Regression | Hands the turn back with an event-level `p`-tag mention of the requesting human | +| [`read-named-path-outside-workspace`](read-named-path-outside-workspace) | Regression | Reads a path the user named explicitly instead of refusing it as out of bounds | +| [`create-channel-invite-users`](create-channel-invite-users) | Workflow | Creates a channel with the exact shape, TTL, and membership asked for | +| [`multiline-message`](multiline-message) | Regression | Preserves real newlines and blank-line structure through the CLI publish path | +| [`narrative-agent-names`](narrative-agent-names) | Regression | Names agents in narrative without waking them through `p` tags | +| [`interleaved-agent-reports`](interleaved-agent-reports) | Workflow | Retains and synthesizes every report in a batch of agent messages | +| [`cross-thread-requests`](cross-thread-requests) | Workflow | Keeps simultaneous top-level requests isolated and replies to both exact threads | +| [`ambiguous-user-mention`](ambiguous-user-mention) | Workflow | Resolves duplicate display names and notifies only the intended pubkey | For `reply-to-thread` and `user-mention` the graded behavior is **deliberately absent from `instruction.md`** — it has to come from `buzz-acp`'s production base prompt. Read a task's own `README.md` before editing its instruction or verifier. +## Evaluation layers + +Every task declares `metadata.evaluation_layer` in `task.toml`: + +| Layer | Question | Default trials | Typical cadence | +| --- | --- | ---: | --- | +| Regression | Did Buzz preserve a known product contract? | k=1 | Targeted PR, nightly, or pre-release | +| Workflow | How capable is the agent at realistic Buzz work? | k=3 | Nightly or weekly on a fixed condition | + +Report regression results per behavior, not as an average capability score. +Use workflow pass rates and trends as the benchmark headline. + +Fast verifier fixtures remain ordinary CI. They validate grading logic, but do +not replace agent trials across the model, base prompt, CLI, and relay. + +The task identity remains `buzz-native/` in both layers; the wrapper reads +the metadata instead of encoding the layer in task names. + ## Running These tasks need the [`harbor-buzz-orchestra`](../harbor-buzz-orchestra) @@ -36,15 +54,20 @@ From the repo root: ```bash just benchmark \ --path benchmarks/buzz-dataset/reply-to-thread \ - --attempts 1 \ --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ --n-concurrent 1 ``` -Pass `--path benchmarks/buzz-dataset` to run the whole suite. The default -condition is one solo agent on `gpt-5.6-luna` at `thinking_effort: medium`, -which needs `OPENAI_COMPAT_API_KEY`; see +The task's regression metadata supplies its default `--attempts 1`. Select a +whole layer with `--path benchmarks/buzz-dataset --layer regression` or +`--layer workflow`. If the dataset root is passed without `--layer` or +`--attempts`, the wrapper runs two Harbor jobs so regression gets k=1 and +workflow gets k=3. An explicit `--attempts`/`-k` overrides these defaults and +runs the selected tasks in one job. + +The default condition is one solo agent on `gpt-5.6-luna` at +`thinking_effort: medium`, which needs `OPENAI_COMPAT_API_KEY`; see [the harness README](../harbor-buzz-orchestra/README.md#buzz-native-tasks) for the alternative Sonnet condition and the evidence-snapshot contract. diff --git a/benchmarks/buzz-dataset/ambiguous-user-mention/task.toml b/benchmarks/buzz-dataset/ambiguous-user-mention/task.toml index 079507e0b4b..00fe095a586 100644 --- a/benchmarks/buzz-dataset/ambiguous-user-mention/task.toml +++ b/benchmarks/buzz-dataset/ambiguous-user-mention/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "mentions", "identity", "ambiguity"] [metadata] +evaluation_layer = "workflow" difficulty = "hard" category = "collaboration" tags = ["mentions", "identity", "ambiguity", "cli"] diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/task.toml b/benchmarks/buzz-dataset/create-channel-invite-users/task.toml index b4ec4821f91..85b6bb81f46 100644 --- a/benchmarks/buzz-dataset/create-channel-invite-users/task.toml +++ b/benchmarks/buzz-dataset/create-channel-invite-users/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "channels", "membership", "cli"] [metadata] +evaluation_layer = "workflow" difficulty = "medium" category = "collaboration" tags = ["channels", "membership", "cli"] diff --git a/benchmarks/buzz-dataset/cross-thread-requests/task.toml b/benchmarks/buzz-dataset/cross-thread-requests/task.toml index 14634fa2d5d..2fe3a5fc811 100644 --- a/benchmarks/buzz-dataset/cross-thread-requests/task.toml +++ b/benchmarks/buzz-dataset/cross-thread-requests/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "threading", "batching", "concurrency"] [metadata] +evaluation_layer = "workflow" difficulty = "hard" category = "collaboration" tags = ["threading", "batching", "concurrency", "routing"] diff --git a/benchmarks/buzz-dataset/interleaved-agent-reports/task.toml b/benchmarks/buzz-dataset/interleaved-agent-reports/task.toml index 8bd320cb090..e4c7f2dc14f 100644 --- a/benchmarks/buzz-dataset/interleaved-agent-reports/task.toml +++ b/benchmarks/buzz-dataset/interleaved-agent-reports/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "agents", "batching", "synthesis"] [metadata] +evaluation_layer = "workflow" difficulty = "hard" category = "collaboration" tags = ["agents", "batching", "synthesis", "mentions"] diff --git a/benchmarks/buzz-dataset/multiline-message/task.toml b/benchmarks/buzz-dataset/multiline-message/task.toml index 69f919f7c11..a5d402de316 100644 --- a/benchmarks/buzz-dataset/multiline-message/task.toml +++ b/benchmarks/buzz-dataset/multiline-message/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "messaging", "multiline", "cli"] [metadata] +evaluation_layer = "regression" difficulty = "medium" category = "collaboration" tags = ["messaging", "multiline", "cli"] diff --git a/benchmarks/buzz-dataset/narrative-agent-names/task.toml b/benchmarks/buzz-dataset/narrative-agent-names/task.toml index 09025e07231..d6d7c8302f5 100644 --- a/benchmarks/buzz-dataset/narrative-agent-names/task.toml +++ b/benchmarks/buzz-dataset/narrative-agent-names/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "mentions", "agents", "notifications"] [metadata] +evaluation_layer = "regression" difficulty = "medium" category = "collaboration" tags = ["mentions", "agents", "notifications"] diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml b/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml index b4b61769153..2702fbef679 100644 --- a/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "filesystem", "workspace", "named-path"] [metadata] +evaluation_layer = "regression" difficulty = "easy" category = "collaboration" tags = ["filesystem", "named-path", "regression"] diff --git a/benchmarks/buzz-dataset/reply-to-thread/task.toml b/benchmarks/buzz-dataset/reply-to-thread/task.toml index 9d400aabb7f..4693677b550 100644 --- a/benchmarks/buzz-dataset/reply-to-thread/task.toml +++ b/benchmarks/buzz-dataset/reply-to-thread/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "messaging", "threading"] [metadata] +evaluation_layer = "regression" difficulty = "easy" category = "collaboration" tags = ["messaging", "threading", "implicit-behavior"] diff --git a/benchmarks/buzz-dataset/user-mention/task.toml b/benchmarks/buzz-dataset/user-mention/task.toml index 53659e146ba..c976dcba185 100644 --- a/benchmarks/buzz-dataset/user-mention/task.toml +++ b/benchmarks/buzz-dataset/user-mention/task.toml @@ -7,6 +7,7 @@ authors = [{ name = "Buzz" }] keywords = ["buzz-native", "messaging", "mentions"] [metadata] +evaluation_layer = "regression" difficulty = "easy" category = "collaboration" tags = ["messaging", "mentions", "implicit-behavior"] diff --git a/benchmarks/harbor-buzz-orchestra/README.md b/benchmarks/harbor-buzz-orchestra/README.md index bbc0b603805..df859b41ab5 100644 --- a/benchmarks/harbor-buzz-orchestra/README.md +++ b/benchmarks/harbor-buzz-orchestra/README.md @@ -75,12 +75,24 @@ prompt from the checked-out source build: ```bash just benchmark \ --path benchmarks/buzz-dataset/reply-to-thread \ - --attempts 1 \ --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ --n-concurrent 1 ``` +Buzz-native tasks declare one of two evaluation layers in `task.toml`. +**Regression** tasks are deterministic product/prompt regression checks and +default to k=1. **Workflow** tasks exercise multi-step collaboration +capabilities and default to k=3 (not 5). Run a layer by metadata with +`--path benchmarks/buzz-dataset --layer regression` or `--layer workflow`; +task identities stay unchanged. + +When the Buzz dataset root is passed without `--layer` or `--attempts`, the +wrapper starts two sequential Harbor jobs so each layer gets its own default. +A direct task path infers its layer's default. An explicit `--attempts`/`-k` +overrides the defaults and permits one mixed-layer job. Terminal-Bench and +other unrelated paths keep their existing k=5 default. + The default condition is `buzz-native-solo-luna.yaml` — one solo agent on `gpt-5.6-luna` at `thinking_effort: medium`. What this suite scores comes from the base prompt rather than from model strength, so the cheap model at a @@ -131,6 +143,8 @@ schema, and defaults to leaderboard-eligible settings (Terminal-Bench 2.1, ```bash just benchmark # full TB 2.1, k=5 just benchmark --path -k 1 # one local task, one attempt +just benchmark --path benchmarks/buzz-dataset --layer regression # Buzz k=1 +just benchmark --path benchmarks/buzz-dataset --layer workflow # Buzz k=3 just benchmark -i "cobol*" --attempts 3 # dataset subset just benchmark --gui # watch the run live ``` diff --git a/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py b/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py index b6f5601a82c..0ad028796cd 100755 --- a/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py +++ b/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 """One-command benchmark: bring up the Buzz stack in Docker and run it. -``just benchmark`` wraps this script. Defaults are leaderboard-eligible out -of the box (Terminal-Bench 2.1, 5 attempts per problem, the Sonnet+Haiku -team); every ``run_leaderboard.py`` selector passes through unchanged. The -script owns everything around the run: +``just benchmark`` wraps this script. Terminal-Bench defaults remain +leaderboard-eligible (2.1, 5 attempts per problem, the Sonnet+Haiku team). +Buzz-native tasks use their ``evaluation_layer`` metadata: regression runs +default to 1 attempt and workflow runs default to 3. The script owns +everything around the run: - A dedicated ``buzz-benchmark`` compose project reusing the production bundle (``deploy/compose/compose.yml``) plus the benchmark port overlay, @@ -26,6 +27,8 @@ from __future__ import annotations import argparse +import datetime as dt +import fnmatch import importlib.util import json import os @@ -34,6 +37,8 @@ import subprocess import sys import time +import tomllib +from dataclasses import dataclass from pathlib import Path PACKAGE_ROOT = Path(__file__).resolve().parent.parent @@ -52,6 +57,9 @@ DEFAULT_DATASET = "terminal-bench/terminal-bench-2-1" DEFAULT_ATTEMPTS = 5 +BUZZ_DATASET_ROOT = REPO_ROOT / "benchmarks" / "buzz-dataset" +EVALUATION_LAYERS = ("regression", "workflow") +LAYER_DEFAULT_ATTEMPTS = {"regression": 1, "workflow": 3} DEFAULT_MANIFEST = PACKAGE_ROOT / "manifests" / "tb-cobol-sonnet-haiku.yaml" DEFAULT_ENDPOINTS = PACKAGE_ROOT / "testbed" / "endpoints" / "anthropic-live.json" SCHEMA_SQL = PACKAGE_ROOT / "testbed" / "sql" / "benchmark_schema.sql" @@ -69,6 +77,16 @@ LINUX_TARGET_DIR = STATE_DIR / "linux-target" RUST_IMAGE = "rust:1.95-alpine" + +@dataclass(frozen=True) +class BuzzTask: + """The identity and evaluation layer declared by one Buzz task.""" + + name: str + layer: str + path: Path + + _spec = importlib.util.spec_from_file_location( "run_leaderboard", Path(__file__).resolve().parent / "run_leaderboard.py" ) @@ -106,12 +124,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=[], help="Task name to exclude (glob, repeatable)", ) + parser.add_argument( + "--layer", + choices=EVALUATION_LAYERS, + help="Buzz evaluation layer to run (selected from task metadata)", + ) parser.add_argument( "--attempts", "-k", type=int, - default=DEFAULT_ATTEMPTS, - help=f"Runs per problem (default: {DEFAULT_ATTEMPTS}, the leaderboard requirement)", + default=None, + help="Runs per problem (default: Terminal-Bench 5, Buzz regression 1, " + "Buzz workflow 3)", ) parser.add_argument( "--manifest", @@ -158,6 +182,159 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) +def _read_buzz_task(task_toml: Path) -> BuzzTask: + """Read and validate the evaluation metadata used by the wrapper.""" + try: + config = tomllib.loads(task_toml.read_text()) + name = config["task"]["name"] + layer = config["metadata"]["evaluation_layer"] + except (OSError, tomllib.TOMLDecodeError, KeyError, TypeError) as error: + raise SystemExit( + f"invalid Buzz task metadata in {task_toml}: {error}" + ) from error + if not isinstance(name, str) or not name: + raise SystemExit(f"invalid Buzz task name in {task_toml}: expected a string") + if layer not in EVALUATION_LAYERS: + allowed = ", ".join(EVALUATION_LAYERS) + raise SystemExit( + f"invalid evaluation_layer in {task_toml}: {layer!r}; expected {allowed}" + ) + return BuzzTask(name=name, layer=layer, path=task_toml.parent) + + +def buzz_tasks_for_path(path: Path | None) -> tuple[BuzzTask, ...] | None: + """Return validated Buzz tasks, or ``None`` for an unrelated problem set.""" + if path is None: + return None + root = BUZZ_DATASET_ROOT.resolve() + selected_path = path.resolve() + if not selected_path.is_relative_to(root): + return None + direct_task = selected_path / "task.toml" + task_files = ( + [direct_task] + if direct_task.is_file() + else sorted(selected_path.glob("*/task.toml")) + ) + if not task_files: + raise SystemExit(f"no Buzz tasks found under {path}") + tasks = tuple(_read_buzz_task(task_file) for task_file in task_files) + names = [task.name for task in tasks] + if len(names) != len(set(names)): + raise SystemExit(f"duplicate Buzz task names found under {path}") + return tasks + + +def _matches_task(task: BuzzTask, pattern: str) -> bool: + return fnmatch.fnmatchcase(task.name, pattern) or fnmatch.fnmatchcase( + task.path.name, pattern + ) + + +def select_buzz_tasks( + tasks: tuple[BuzzTask, ...], + *, + layer: str | None, + include: list[str], + exclude: list[str], +) -> tuple[BuzzTask, ...]: + """Apply layer metadata and the wrapper's existing name selectors.""" + selected = tuple(task for task in tasks if layer is None or task.layer == layer) + if include: + selected = tuple( + task + for task in selected + if any(_matches_task(task, pattern) for pattern in include) + ) + if exclude: + selected = tuple( + task + for task in selected + if not any(_matches_task(task, pattern) for pattern in exclude) + ) + if not selected: + detail = f" for layer {layer!r}" if layer else "" + raise SystemExit(f"no Buzz tasks selected{detail}") + return selected + + +def _copy_run_args( + args: argparse.Namespace, + *, + tasks: tuple[BuzzTask, ...] | None, + attempts: int, + job_name: str | None = None, +) -> argparse.Namespace: + run_args = argparse.Namespace(**vars(args)) + run_args.attempts = attempts + run_args.job_name = args.job_name if job_name is None else job_name + if tasks is not None: + # Harbor filters local-path datasets by directory basename. Keep the + # canonical task.toml identity for metadata, but pass Harbor its key. + run_args.include_task = [task.path.name for task in tasks] + run_args.exclude_task = [] + return run_args + + +def plan_benchmark_runs( + args: argparse.Namespace, *, stamp: str | None = None +) -> tuple[argparse.Namespace, ...]: + """Resolve selectors and per-layer attempts into one or more Harbor jobs.""" + tasks = buzz_tasks_for_path(args.path) + if args.layer and tasks is None: + raise SystemExit( + "--layer is only valid with --path under benchmarks/buzz-dataset" + ) + if tasks is None: + return ( + _copy_run_args( + args, + tasks=None, + attempts=( + args.attempts if args.attempts is not None else DEFAULT_ATTEMPTS + ), + ), + ) + + selected = select_buzz_tasks( + tasks, + layer=args.layer, + include=args.include_task, + exclude=args.exclude_task, + ) + if args.attempts is not None: + return (_copy_run_args(args, tasks=selected, attempts=args.attempts),) + + layers = (args.layer,) if args.layer else EVALUATION_LAYERS + groups = tuple( + (layer, tuple(task for task in selected if task.layer == layer)) + for layer in layers + ) + groups = tuple((layer, group) for layer, group in groups if group) + if len(groups) == 1: + layer, group = groups[0] + return ( + _copy_run_args(args, tasks=group, attempts=LAYER_DEFAULT_ATTEMPTS[layer]), + ) + + if stamp is None: + stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ") + if args.job_name: + base_job_name = args.job_name + else: + manifest = run_leaderboard.yaml.safe_load(args.manifest.read_text()) + base_job_name = f"lb-{manifest.get('condition', 'team')}-{stamp}" + return tuple( + _copy_run_args( + args, + tasks=group, + attempts=LAYER_DEFAULT_ATTEMPTS[layer], + job_name=f"{base_job_name}-{layer}", + ) + for layer, group in groups + ) + + # -- state: secrets and identities, generated once -------------------------- @@ -582,6 +759,7 @@ def leaderboard_argv( def main(argv: list[str] | None = None) -> int: args = parse_args(argv) + runs = plan_benchmark_runs(args) state = load_state() print_user_identity(state) write_env_file(state) @@ -598,9 +776,13 @@ def main(argv: list[str] | None = None) -> int: if args.gui: launch_gui(state) - return run_leaderboard.main( - leaderboard_argv(args, provisioner_config, agent_bin_dir) - ) + for run_args in runs: + result = run_leaderboard.main( + leaderboard_argv(run_args, provisioner_config, agent_bin_dir) + ) + if result != 0: + return result + return 0 if __name__ == "__main__": diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py index e0c6d32ec44..0644df63b36 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py @@ -1,11 +1,13 @@ """just benchmark must default to leaderboard-eligible settings.""" +import asyncio import importlib.util import json import sys from pathlib import Path import pytest +from harbor.models.job.config import DatasetConfig _SCRIPT = Path(__file__).parents[2] / "scripts" / "benchmark.py" _spec = importlib.util.spec_from_file_location("benchmark", _SCRIPT) @@ -22,9 +24,10 @@ def state_dir(tmp_path, monkeypatch): def test_defaults_are_leaderboard_eligible(): args = benchmark.parse_args([]) - assert args.attempts == 5 + assert args.attempts is None assert args.dataset is None and args.path is None # dataset default applied later - argv = benchmark.leaderboard_argv(args, Path("prov.json"), Path("linux-bin")) + (run,) = benchmark.plan_benchmark_runs(args) + argv = benchmark.leaderboard_argv(run, Path("prov.json"), Path("linux-bin")) assert argv[argv.index("--dataset") + 1] == "terminal-bench/terminal-bench-2-1" assert argv[argv.index("--attempts") + 1] == "5" assert argv[argv.index("--manifest") + 1].endswith("tb-cobol-sonnet-haiku.yaml") @@ -51,7 +54,8 @@ def test_selectors_pass_through(): "--dry-run", ] ) - argv = benchmark.leaderboard_argv(args, Path("p.json"), Path("b")) + (run,) = benchmark.plan_benchmark_runs(args) + argv = benchmark.leaderboard_argv(run, Path("p.json"), Path("b")) assert argv[argv.index("--path") + 1] == "/tmp/task" assert argv[argv.index("--include-task") + 1] == "cobol*" assert argv[argv.index("--exclude-task") + 1] == "flaky*" @@ -60,6 +64,177 @@ def test_selectors_pass_through(): assert "--dataset" not in argv +def test_buzz_task_metadata_defines_the_expected_layers(): + tasks = benchmark.buzz_tasks_for_path(benchmark.BUZZ_DATASET_ROOT) + assert tasks is not None + by_layer = { + layer: {task.path.name for task in tasks if task.layer == layer} + for layer in benchmark.EVALUATION_LAYERS + } + assert by_layer == { + "regression": { + "reply-to-thread", + "user-mention", + "read-named-path-outside-workspace", + "multiline-message", + "narrative-agent-names", + }, + "workflow": { + "ambiguous-user-mention", + "cross-thread-requests", + "create-channel-invite-users", + "interleaved-agent-reports", + }, + } + + +@pytest.mark.parametrize("layer", [None, "other"]) +def test_buzz_task_metadata_rejects_missing_or_unknown_layers( + tmp_path, monkeypatch, layer +): + dataset = tmp_path / "buzz-dataset" + task = dataset / "example" + task.mkdir(parents=True) + metadata = "" if layer is None else f'evaluation_layer = "{layer}"\n' + (task / "task.toml").write_text( + 'schema_version = "1.3"\n\n' + '[task]\nname = "buzz-native/example"\n\n' + f'[metadata]\n{metadata}difficulty = "easy"\n' + ) + monkeypatch.setattr(benchmark, "BUZZ_DATASET_ROOT", dataset) + + with pytest.raises(SystemExit, match="evaluation_layer|metadata"): + benchmark.buzz_tasks_for_path(dataset) + + +@pytest.mark.parametrize(("layer", "attempts"), [("regression", 1), ("workflow", 3)]) +def test_layer_selects_metadata_and_uses_its_default_attempts(layer, attempts): + args = benchmark.parse_args( + ["--path", str(benchmark.BUZZ_DATASET_ROOT), "--layer", layer] + ) + (run,) = benchmark.plan_benchmark_runs(args) + + assert run.attempts == attempts + selected = benchmark.buzz_tasks_for_path(benchmark.BUZZ_DATASET_ROOT) + assert selected is not None + assert set(run.include_task) == { + task.path.name for task in selected if task.layer == layer + } + + +@pytest.mark.parametrize("layer", benchmark.EVALUATION_LAYERS) +def test_layer_selectors_resolve_through_harbor_local_dataset_filter(layer): + args = benchmark.parse_args( + ["--path", str(benchmark.BUZZ_DATASET_ROOT), "--layer", layer] + ) + (run,) = benchmark.plan_benchmark_runs(args) + + configs = asyncio.run( + DatasetConfig( + path=benchmark.BUZZ_DATASET_ROOT, + task_names=run.include_task, + ).get_task_configs(disable_verification=True) + ) + + assert {config.path.name for config in configs} == set(run.include_task) + + +def test_omitted_layer_splits_buzz_dataset_into_two_jobs(): + args = benchmark.parse_args(["--path", str(benchmark.BUZZ_DATASET_ROOT)]) + runs = benchmark.plan_benchmark_runs(args, stamp="20260825T120000Z") + + assert [run.attempts for run in runs] == [1, 3] + assert [run.job_name.rsplit("-", 1)[-1] for run in runs] == [ + "regression", + "workflow", + ] + assert set(runs[0].include_task).isdisjoint(runs[1].include_task) + + +def test_single_buzz_task_infers_its_layer_default(): + task_path = benchmark.BUZZ_DATASET_ROOT / "cross-thread-requests" + args = benchmark.parse_args(["--path", str(task_path)]) + (run,) = benchmark.plan_benchmark_runs(args) + + assert run.attempts == 3 + assert run.include_task == ["cross-thread-requests"] + + +def test_explicit_attempts_override_keeps_one_mixed_buzz_job(): + args = benchmark.parse_args( + ["--path", str(benchmark.BUZZ_DATASET_ROOT), "--attempts", "7"] + ) + (run,) = benchmark.plan_benchmark_runs(args) + + assert run.attempts == 7 + assert len(run.include_task) == 9 + + layered = benchmark.parse_args( + [ + "--path", + str(benchmark.BUZZ_DATASET_ROOT), + "--layer", + "workflow", + "-k", + "2", + ] + ) + (layered_run,) = benchmark.plan_benchmark_runs(layered) + assert layered_run.attempts == 2 + assert len(layered_run.include_task) == 4 + + +def test_invalid_layer_is_rejected(): + with pytest.raises(SystemExit): + benchmark.parse_args(["--layer", "conformance"]) + + args = benchmark.parse_args( + ["--dataset", "terminal-bench/x", "--layer", "workflow"] + ) + with pytest.raises(SystemExit, match="only valid"): + benchmark.plan_benchmark_runs(args) + + +def test_layer_dry_run_constructs_exact_task_selectors_and_attempts(): + args = benchmark.parse_args( + [ + "--path", + str(benchmark.BUZZ_DATASET_ROOT), + "--layer", + "workflow", + "--dry-run", + "--job-name", + "workflow-smoke", + ] + ) + (run,) = benchmark.plan_benchmark_runs(args) + argv = benchmark.leaderboard_argv(run, Path("prov.json"), Path("linux-bin")) + lower_args = benchmark.run_leaderboard.parse_args(argv) + binaries = {"buzz": Path("host-bin/buzz")} + agent_binaries = { + name: Path("linux-bin") / name + for name in benchmark.run_leaderboard.AGENT_BINARIES + + (benchmark.run_leaderboard.FORWARDER_BINARY,) + } + command = benchmark.run_leaderboard.build_command( + lower_args, binaries, agent_binaries + ) + + assert command[command.index("-k") + 1] == "3" + selected = [ + command[index + 1] + for index, part in enumerate(command) + if part == "--include-task-name" + ] + assert set(selected) == set(run.include_task) + assert set(selected) == { + "ambiguous-user-mention", + "cross-thread-requests", + "create-channel-invite-users", + "interleaved-agent-reports", + } + + def test_state_is_generated_once_and_reused(state_dir): first = benchmark.load_state() second = benchmark.load_state()