From a3339dc2776ee0c977fb3313fe89f56da7a91555 Mon Sep 17 00:00:00 2001 From: Hao Li <18546749+haofrank@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:55:38 -0700 Subject: [PATCH 01/16] Resolve generic benchmark dependencies before eval changes cwd --- Magpie/scripts/benchmark/atom_mi300x.sh | 7 +- Magpie/scripts/benchmark/atom_mi355x.sh | 7 +- Magpie/scripts/benchmark/sglang_mi300x.sh | 7 +- Magpie/scripts/benchmark/sglang_mi355x.sh | 7 +- .../scripts/benchmark/sglang_radeon8060s.sh | 7 +- Magpie/scripts/benchmark/vllm_mi300x.sh | 7 +- Magpie/scripts/benchmark/vllm_mi300x_mm.sh | 7 +- Magpie/scripts/benchmark/vllm_mi355x.sh | 7 +- Magpie/scripts/benchmark/vllm_mi355x_mm.sh | 7 +- Magpie/scripts/benchmark/vllm_radeon8060s.sh | 7 +- tests/test_eval_source_paths.py | 94 +++++++++++++++++++ 11 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 tests/test_eval_source_paths.py diff --git a/Magpie/scripts/benchmark/atom_mi300x.sh b/Magpie/scripts/benchmark/atom_mi300x.sh index eb3eff5..9e22259 100644 --- a/Magpie/scripts/benchmark/atom_mi300x.sh +++ b/Magpie/scripts/benchmark/atom_mi300x.sh @@ -20,10 +20,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch, no server-side cleanup, no SERVER_PID monitoring). -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/atom_mi355x.sh b/Magpie/scripts/benchmark/atom_mi355x.sh index bd8e2b1..c88143c 100644 --- a/Magpie/scripts/benchmark/atom_mi355x.sh +++ b/Magpie/scripts/benchmark/atom_mi355x.sh @@ -16,10 +16,11 @@ # with vLLM, so the bench client uses --backend vllm unchanged. See # atom_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_mi300x.sh b/Magpie/scripts/benchmark/sglang_mi300x.sh index bde2280..74cdefa 100644 --- a/Magpie/scripts/benchmark/sglang_mi300x.sh +++ b/Magpie/scripts/benchmark/sglang_mi300x.sh @@ -15,10 +15,11 @@ # cluster, externally managed). Leave the env unset to keep the # default behaviour of launching a local server. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_mi355x.sh b/Magpie/scripts/benchmark/sglang_mi355x.sh index ea5aeaf..1c68444 100644 --- a/Magpie/scripts/benchmark/sglang_mi355x.sh +++ b/Magpie/scripts/benchmark/sglang_mi355x.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See sglang_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/sglang_radeon8060s.sh b/Magpie/scripts/benchmark/sglang_radeon8060s.sh index 12402ab..40d6f3d 100755 --- a/Magpie/scripts/benchmark/sglang_radeon8060s.sh +++ b/Magpie/scripts/benchmark/sglang_radeon8060s.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See sglang_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi300x.sh b/Magpie/scripts/benchmark/vllm_mi300x.sh index d852f6d..0bf5997 100644 --- a/Magpie/scripts/benchmark/vllm_mi300x.sh +++ b/Magpie/scripts/benchmark/vllm_mi300x.sh @@ -19,10 +19,11 @@ # cluster, externally managed). Leave the env unset to keep the # default behaviour of launching a local server. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi300x_mm.sh b/Magpie/scripts/benchmark/vllm_mi300x_mm.sh index b3ee431..aef3c1f 100644 --- a/Magpie/scripts/benchmark/vllm_mi300x_mm.sh +++ b/Magpie/scripts/benchmark/vllm_mi300x_mm.sh @@ -26,10 +26,11 @@ # Phases (via MAGPIE_RUN_PHASE): all | server | client (default all). # Server-only writes PID to MAGPIE_SERVER_PID_FILE then disowns and exits. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -[[ -f "$(dirname "$0")/magpie_bench_remote_compat.sh" ]] && source "$(dirname "$0")/magpie_bench_remote_compat.sh" +[[ -f "$SCRIPT_DIR/magpie_bench_remote_compat.sh" ]] && source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" # Fallback: some server_cleanup.sh variants (e.g. InferenceX's) do not define # magpie_stop_benchmark_server_stack. Provide a safe inline version so the diff --git a/Magpie/scripts/benchmark/vllm_mi355x.sh b/Magpie/scripts/benchmark/vllm_mi355x.sh index 1b42e62..a3f8e3f 100644 --- a/Magpie/scripts/benchmark/vllm_mi355x.sh +++ b/Magpie/scripts/benchmark/vllm_mi355x.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See vllm_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/Magpie/scripts/benchmark/vllm_mi355x_mm.sh b/Magpie/scripts/benchmark/vllm_mi355x_mm.sh index fd025a3..d7c20ee 100644 --- a/Magpie/scripts/benchmark/vllm_mi355x_mm.sh +++ b/Magpie/scripts/benchmark/vllm_mi355x_mm.sh @@ -26,10 +26,11 @@ # Phases (via MAGPIE_RUN_PHASE): all | server | client (default all). # Server-only writes PID to MAGPIE_SERVER_PID_FILE then disowns and exits. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -[[ -f "$(dirname "$0")/magpie_bench_remote_compat.sh" ]] && source "$(dirname "$0")/magpie_bench_remote_compat.sh" +[[ -f "$SCRIPT_DIR/magpie_bench_remote_compat.sh" ]] && source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" # Fallback: some server_cleanup.sh variants (e.g. InferenceX's) do not define # magpie_stop_benchmark_server_stack. Provide a safe inline version so the diff --git a/Magpie/scripts/benchmark/vllm_radeon8060s.sh b/Magpie/scripts/benchmark/vllm_radeon8060s.sh index a49aae7..ba30c11 100755 --- a/Magpie/scripts/benchmark/vllm_radeon8060s.sh +++ b/Magpie/scripts/benchmark/vllm_radeon8060s.sh @@ -16,10 +16,11 @@ # instead of localhost:$PORT, and forces PHASE=client (no local server # launch). See vllm_mi300x.sh for the full contract. -source "$(dirname "$0")/benchmark_lib.sh" -source "$(dirname "$0")/server_cleanup.sh" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" || exit $? +source "$SCRIPT_DIR/benchmark_lib.sh" +source "$SCRIPT_DIR/server_cleanup.sh" # shellcheck source=magpie_bench_remote_compat.sh -source "$(dirname "$0")/magpie_bench_remote_compat.sh" +source "$SCRIPT_DIR/magpie_bench_remote_compat.sh" PHASE="${MAGPIE_RUN_PHASE:-all}" case "$PHASE" in diff --git a/tests/test_eval_source_paths.py b/tests/test_eval_source_paths.py new file mode 100644 index 0000000..4fd2854 --- /dev/null +++ b/tests/test_eval_source_paths.py @@ -0,0 +1,94 @@ +"""Generic scripts keep sourced eval resources valid after persisted eval changes cwd.""" + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).parents[1] / "Magpie" / "scripts" / "benchmark" +EVAL_SCRIPTS = [ + path.name + for path in sorted(SCRIPTS.glob("*.sh")) + if '/benchmark_lib.sh"' in path.read_text(encoding="utf-8") +] + +# InferenceX 3d5581562 resolves these paths inside functions, after Magpie's +# persisted eval wrapper changes cwd. Keep real Bash source-stack resolution; +# a stub that captures the root at source time would conceal the regression. +BENCHMARK_LIB = r''' +_eval_patches_dir() { + cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd +} + +run_eval() { + local _repo_root + _repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" || return 1 + [[ "$_repo_root" == "$EXPECTED_REPO_ROOT" ]] || return 2 + [[ "$PWD" == "$EVAL_RESULT_DIR" ]] || return 3 + cp "$(_eval_patches_dir)/lm_eval_sitecustomize.py" "$EVAL_RESULT_DIR/sitecustomize.py" || return 4 + cp "$_repo_root/utils/evals/gsm8k.yaml" "$EVAL_RESULT_DIR/task.yaml" || return 5 + printf '%s\n' '{"results":{"gsm8k":{"exact_match,strict-match":0.5}}}' > "$EVAL_RESULT_DIR/results_fixture.json" +} + +# Exercise the entrypoint's actual dependency loading and persisted eval +# wrapper, exiting before any server, GPU command, or throughput client runs. +check_env_vars() { + magpie_run_eval_persisted --framework lm-eval --port 8888 + local eval_rc=$? + [[ "$PWD" == "$EXPECTED_REPO_ROOT" ]] || exit 6 + exit "$eval_rc" +} +''' + + +@pytest.mark.parametrize("script_name", EVAL_SCRIPTS) +@pytest.mark.parametrize("absolute_entrypoint", [False, True]) +def test_persisted_eval_keeps_deferred_source_paths( + tmp_path: Path, script_name: str, absolute_entrypoint: bool +): + checkout = tmp_path / "InferenceX checkout" + scripts = checkout / "benchmarks" + shutil.copytree(SCRIPTS, scripts) + library = scripts / "benchmark_lib.sh" + library.write_text(BENCHMARK_LIB, encoding="utf-8") + patches = checkout / "utils" / "evals" / "patches" + patches.mkdir(parents=True) + (patches / "lm_eval_sitecustomize.py").write_text("# eval patch fixture\n") + (patches.parent / "gsm8k.yaml").write_text("task: gsm8k\n") + result_dir = tmp_path / "session results" + eval_dir = result_dir / "eval output" + env = { + "PATH": os.environ["PATH"], + "MAGPIE_RUN_PHASE": "client", + "MAGPIE_ACCURACY_REPORT_PYTHON": sys.executable, + "RESULT_DIR": str(result_dir), + "EVAL_RESULT_DIR": str(eval_dir), + "EXPECTED_REPO_ROOT": str(checkout), + "MODEL": "fixture-model", + "CONC": "1", + } + entrypoint = scripts / script_name + if not absolute_entrypoint: + entrypoint = entrypoint.relative_to(checkout) + completed = subprocess.run( + ["bash", str(entrypoint)], + cwd=checkout, + env=env, + capture_output=True, + text=True, + check=False, + timeout=15, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert (eval_dir / "sitecustomize.py").read_text() == "# eval patch fixture\n" + assert (eval_dir / "task.yaml").read_text() == "task: gsm8k\n" + summary = json.loads((result_dir / "accuracy_report.json").read_text()) + assert summary["status"] == "COMPLETED" + assert summary["score"] == 0.5 + assert (result_dir / summary["source_result"]).is_file() + assert library.read_text() == BENCHMARK_LIB + assert not (checkout / "results_fixture.json").exists() From 10052e21a9d2024180f1875dd8495a619c6e58d7 Mon Sep 17 00:00:00 2001 From: Hao Li <18546749+haofrank@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:23:40 -0700 Subject: [PATCH 02/16] feat(agentx): resolve launchers and verify server launch extensions Resolve declared InferenceX launchers and persist structured server overrides. Require matching launch evidence before accepting candidate results. --- Magpie/modes/benchmark/agentx.py | 70 +++++ Magpie/modes/benchmark/agentx_launch.py | 269 +++++++++++++++++ Magpie/modes/benchmark/benchmarker.py | 15 + Magpie/modes/benchmark/config.py | 20 +- docs/how-to/benchmarking/benchmark.md | 11 +- docs/reference/benchmark-config.md | 52 +++- tests/unit/modes/test_agentx.py | 10 +- tests/unit/modes/test_agentx_launch.py | 365 ++++++++++++++++++++++++ 8 files changed, 788 insertions(+), 24 deletions(-) create mode 100644 Magpie/modes/benchmark/agentx_launch.py create mode 100644 tests/unit/modes/test_agentx_launch.py diff --git a/Magpie/modes/benchmark/agentx.py b/Magpie/modes/benchmark/agentx.py index 2ab1a88..87b5a4c 100644 --- a/Magpie/modes/benchmark/agentx.py +++ b/Magpie/modes/benchmark/agentx.py @@ -517,6 +517,8 @@ def resolve_agentx_recipe( entry["image"] = config.docker_image entry["recipe-fingerprint"] = _recipe_fingerprint(entry) + _resolve_launcher(config, inferencex_root, recipe, entry, runner_type) + _apply_launch_entry(config, entry) try: config_file_label = str(config_file.relative_to(inferencex_root)) @@ -532,6 +534,74 @@ def resolve_agentx_recipe( return spec +def _resolve_launcher( + config: BenchmarkConfig, + root: Path, + recipe: str, + entry: Dict[str, Any], + runner_type: Optional[str], +) -> None: + """Use InferenceX's declared recipe/launcher mapping, never filename guesses.""" + manifest = root / "configs" / "agentx-launchers.json" + row = None + if manifest.is_file(): + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"Invalid AgentX launcher manifest: {manifest}") from exc + if ( + not isinstance(data, dict) + or data.get("version") != 1 + or not isinstance(data.get("recipes"), dict) + ): + raise ValueError(f"Unsupported AgentX launcher manifest: {manifest}") + row = data["recipes"].get(recipe) + if row is not None: + if not isinstance(row, dict) or row.get("framework") != config.framework: + raise ValueError(f"AgentX launcher manifest framework mismatch for {recipe}") + for field, expected in ( + ("model", config.model), ("precision", config.precision) + ): + if row.get(field) != expected: + raise ValueError( + f"AgentX launcher manifest {field} mismatch for {recipe}" + ) + if runner_type and row.get("runner_type") != runner_type.lower(): + raise ValueError(f"AgentX launcher manifest runner mismatch for {recipe}") + script = row.get("benchmark_script") + if not isinstance(script, str) or not script.startswith( + "single_node/agentic/" + ): + raise ValueError(f"Invalid AgentX launcher path for {recipe}") + path = (root / "benchmarks" / script).resolve() + agentic_root = (root / "benchmarks" / "single_node" / "agentic").resolve() + if not path.is_relative_to(agentic_root) or not path.is_file(): + raise ValueError( + "AgentX declared launcher is missing or escapes its directory: " + f"{script}" + ) + if config.benchmark_script and config.benchmark_script != script: + raise ValueError( + f"AgentX recipe {recipe} declares launcher {script}, " + f"not {config.benchmark_script}" + ) + config.benchmark_script = script + entry["benchmark-script"] = script + entry["launch-overrides-version"] = row.get("launch_overrides_version") + if not config.benchmark_script: + raise ValueError( + f"InferenceX has no declared AgentX launcher for {recipe}; " + "use a checkout with configs/agentx-launchers.json or pin benchmark_script" + ) + assert config.agentx is not None + if config.agentx.launch_overrides is not None and ( + row is None or row.get("launch_overrides_version") != 1 + ): + raise ValueError( + f"AgentX launcher for {recipe} does not declare launch_overrides v1 support" + ) + + def _apply_launch_entry(config: BenchmarkConfig, entry: Dict[str, Any]) -> None: assert config.agentx is not None config.framework = str(entry["framework"]).lower() diff --git a/Magpie/modes/benchmark/agentx_launch.py b/Magpie/modes/benchmark/agentx_launch.py new file mode 100644 index 0000000..1990aa2 --- /dev/null +++ b/Magpie/modes/benchmark/agentx_launch.py @@ -0,0 +1,269 @@ +"""Versioned AgentX launch requests and independently persisted launch evidence.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .config import BenchmarkConfig + +PROTECTED_ARGS = frozenset( + { + "--model", + "--model-path", + "--served-model-name", + "--host", + "--port", + "--tp", + "--tp-size", + "--tensor-parallel-size", + "--pp", + "--pp-size", + "--pipeline-parallel-size", + "--ep", + "--ep-size", + "--expert-parallel-size", + "--dp", + "--dp-size", + "--data-parallel-size", + "--data-parallel-rank", + "--data-parallel-start-rank", + "--data-parallel-size-local", + "--data-parallel-address", + "--data-parallel-rpc-port", + "--enable-dp-attention", + "--enable-expert-parallel", + "--decode-context-parallel-size", + "--prefill-context-parallel-size", + "--dcp-size", + "--pcp-size", + "--nnodes", + "--node-rank", + "--dist-init-addr", + } +) +PROTECTED_ENV = frozenset( + { + "MODEL", + "MODEL_PATH", + "MODEL_NAME", + "SERVED_MODEL_NAME", + "MODEL_PREFIX", + "TP", + "PP_SIZE", + "EP_SIZE", + "DP_ATTENTION", + "DCP_SIZE", + "PCP_SIZE", + "CONC", + "DURATION", + "PORT", + "RUN_EVAL", + "EVAL_ONLY", + "IS_AGENTIC", + "SCENARIO_TYPE", + "SCENARIO_SUBDIR", + "RECIPE_FINGERPRINT", + "ROCR_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + } +) +PROTECTED_ENV_PREFIXES = ("AGENTX_", "AGENTIC_", "AIPERF_", "SGLANG_SIMULATE_ACC_") +FIELDS = { + "version", + "append_args", + "remove_args", + "replace_args", + "env", + "unset_env", + "source_files", + "absent_source_files", + "executable", +} + + +def digest(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + + +def validate_overrides(value: Any) -> dict[str, Any]: + if ( + not isinstance(value, dict) + or type(value.get("version")) is not int + or value["version"] != 1 + ): + raise ValueError("AgentX launch_overrides requires version: 1") + if unknown := value.keys() - FIELDS: + raise ValueError(f"Unknown AgentX launch override fields: {sorted(unknown)}") + result = { + "version": 1, + "append_args": [], + "remove_args": [], + "replace_args": False, + "env": {}, + "unset_env": [], + "source_files": {}, + "absent_source_files": [], + "executable": None, + **value, + } + for key in ("append_args", "remove_args", "unset_env", "absent_source_files"): + if not isinstance(result[key], list) or any( + not isinstance(item, str) or "\0" in item for item in result[key] + ): + raise ValueError(f"{key} must be a list of strings without NUL bytes") + if type(result["replace_args"]) is not bool: + raise ValueError("replace_args must be a boolean") + for key in ("env", "source_files"): + if not isinstance(result[key], dict) or any( + not isinstance(name, str) + or not isinstance(item, str) + or "\0" in name + or "\0" in item + for name, item in result[key].items() + ): + raise ValueError(f"{key} must map strings to strings without NUL bytes") + if len(set(result["remove_args"])) != len(result["remove_args"]): + raise ValueError("remove_args must not contain duplicates") + for name in result["remove_args"]: + if not re.fullmatch(r"--[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise ValueError(f"remove_args must contain long option names: {name!r}") + if name in PROTECTED_ARGS: + raise ValueError(f"Cannot remove protocol option {name}") + names = set(result["env"]) | set(result["unset_env"]) + if set(result["env"]) & set(result["unset_env"]): + raise ValueError("An environment variable cannot be both set and unset") + for name in names: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Invalid environment variable name: {name!r}") + if name in PROTECTED_ENV or name.startswith(PROTECTED_ENV_PREFIXES): + raise ValueError(f"Cannot override protocol environment variable {name}") + for name, checksum in result["source_files"].items(): + if not Path(name).is_absolute() or not re.fullmatch(r"[0-9a-f]{64}", checksum): + raise ValueError( + "source_files requires absolute paths and lowercase SHA256 values" + ) + for name in result["absent_source_files"]: + if not Path(name).is_absolute() or name in result["source_files"]: + raise ValueError("absent_source_files requires distinct absolute paths") + executable = result["executable"] + if executable is not None and ( + not isinstance(executable, str) + or "\0" in executable + or not Path(executable).is_absolute() + ): + raise ValueError("executable must be an absolute path or null") + return result + + +def launch_environment( + config: BenchmarkConfig, + workspace: Path, + *, + docker: bool = False, +) -> dict[str, str]: + """Persist the exact normalized request and bind this run's output path.""" + result = { + "AGENTX_LAUNCH_OVERRIDES_FILE": "", + "AGENTX_LAUNCH_OVERRIDES_SHA256": "", + "AGENTX_SERVER_LAUNCH_FILE": "", + } + if config.agentx is None or config.agentx.launch_overrides is None: + return result + overrides = validate_overrides(config.agentx.launch_overrides) + if overrides != config.agentx.launch_overrides: + raise ValueError("AgentX launch overrides changed after config normalization") + workspace.mkdir(parents=True, exist_ok=True) + request = workspace / "agentx_launch_overrides.json" + request.write_text(json.dumps(overrides, indent=2) + "\n", encoding="utf-8") + # A stale receipt must never make a failed new launch appear verified. + (workspace / "agentx_server_launch.json").unlink(missing_ok=True) + runtime_workspace = Path("/workspace") if docker else workspace.resolve() + result.update( + { + "AGENTX_LAUNCH_OVERRIDES_FILE": str(runtime_workspace / request.name), + "AGENTX_LAUNCH_OVERRIDES_SHA256": digest(overrides), + "AGENTX_SERVER_LAUNCH_FILE": str( + runtime_workspace / "agentx_server_launch.json" + ), + } + ) + return result + + +def read_launch_evidence(config: BenchmarkConfig, workspace: Path) -> dict[str, Any]: + """Verify the receipt against the immutable request persisted for this run.""" + assert config.agentx is not None and config.agentx.launch_overrides is not None + overrides = config.agentx.launch_overrides + request = json.loads( + (workspace / "agentx_launch_overrides.json").read_text(encoding="utf-8") + ) + if request != overrides: + raise ValueError( + "Persisted AgentX launch request differs from the benchmark config" + ) + evidence = json.loads( + (workspace / "agentx_server_launch.json").read_text(encoding="utf-8") + ) + if not isinstance(evidence, dict): + raise ValueError("AgentX server launch evidence must be an object") # noqa: TRY004 + payload = { + key: value for key, value in evidence.items() if key != "evidence_sha256" + } + if evidence.get("evidence_sha256") != digest(payload): + raise ValueError("AgentX server launch evidence hash mismatch") + if ( + type(evidence.get("version")) is not int + or evidence.get("version") != 1 + or evidence.get("framework") != config.framework + ): + raise ValueError("AgentX server launch evidence identity mismatch") + if evidence.get("overrides_sha256") != digest(overrides): + raise ValueError("AgentX server did not apply this run's launch overrides") + if evidence.get("source_files") != overrides["source_files"]: + raise ValueError("AgentX source file launch evidence mismatch") + if evidence.get("absent_source_files") != overrides["absent_source_files"]: + raise ValueError("AgentX deleted source file launch evidence mismatch") + names = sorted(set(overrides["env"]) | set(overrides["unset_env"])) + expected_env = {name: overrides["env"].get(name) for name in names} + if evidence.get("effective_env") != expected_env: + raise ValueError("AgentX effective server environment mismatch") + if not isinstance(evidence.get("base_env"), dict) or set( + evidence["base_env"] + ) != set(names): + raise ValueError("AgentX base server environment evidence is incomplete") + for key in ("base_argv", "effective_argv"): + argv = evidence.get(key) + if ( + not isinstance(argv, list) + or not argv + or any(not isinstance(value, str) for value in argv) + ): + raise ValueError(f"AgentX {key} evidence must contain server argv tokens") + if ( + overrides["executable"] + and evidence["effective_argv"][0] != overrides["executable"] + ): + raise ValueError("AgentX server executable override was not applied") + executable = evidence.get("resolved_executable") + if not isinstance(executable, str) or not Path(executable).is_absolute(): + raise ValueError("AgentX resolved executable evidence is missing") + runtime = evidence.get("runtime_environment") + if not isinstance(runtime, dict) or any( + not isinstance(name, str) or not isinstance(value, str) + for name, value in runtime.items() + ): + raise ValueError("AgentX runtime environment evidence is incomplete") + return evidence diff --git a/Magpie/modes/benchmark/benchmarker.py b/Magpie/modes/benchmark/benchmarker.py index d85d3c9..86c3525 100644 --- a/Magpie/modes/benchmark/benchmarker.py +++ b/Magpie/modes/benchmark/benchmarker.py @@ -33,6 +33,7 @@ ensure_agentx_dependencies, resolve_agentx_recipe, ) +from .agentx_launch import launch_environment, read_launch_evidence from .config import BenchmarkConfig from .image_selector import ImageSelector from .inferencex import ensure_inferencex_available @@ -526,6 +527,18 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: "Benchmark produced no valid throughput/latency metrics" ) + if self.config.is_agentx and self.config.agentx.launch_overrides is not None: + try: + evidence = read_launch_evidence(self.config, workspace) + if result.agentx_metrics is None: + result.agentx_metrics = {} + result.agentx_metrics["server_launch"] = evidence + except (OSError, ValueError) as exc: + result.success = False + result.benchmark_valid = False + result.publishable = False + result.errors.append(f"AgentX server launch evidence rejected: {exc}") + # Parse torch trace if available if self.config.profiler.torch_profiler.enabled: torch_trace_dir = workspace / "torch_trace" @@ -870,6 +883,7 @@ def _build_docker_command( env_vars["INFMAX_CONTAINER_WORKSPACE"] = "/opt/InferenceX" env_vars["AGENTIC_OUTPUT_DIR"] = "/workspace" env_vars["AIPERF_RUNTIME_DIR"] = "/tmp/inferencex-agentx" + env_vars.update(launch_environment(self.config, workspace, docker=True)) if phase == "server": env_vars["MAGPIE_SERVER_PID_FILE"] = "/workspace/reuse_server_spawn.pid" env_vars["MAGPIE_KEEP_CONTAINER_ALIVE"] = "1" @@ -977,6 +991,7 @@ def _build_local_command( env_vars["AIPERF_RUNTIME_DIR"] = str( workspace / ".agentx-runtime" ) + env_vars.update(launch_environment(self.config, workspace)) if phase == "server" and server_pid_file is not None: env_vars["MAGPIE_SERVER_PID_FILE"] = str(server_pid_file) diff --git a/Magpie/modes/benchmark/config.py b/Magpie/modes/benchmark/config.py index 63c0476..799ea9f 100644 --- a/Magpie/modes/benchmark/config.py +++ b/Magpie/modes/benchmark/config.py @@ -12,6 +12,8 @@ from enum import Enum from typing import Any, Dict, List, Optional +from .agentx_launch import validate_overrides + logger = logging.getLogger(__name__) @@ -668,7 +670,7 @@ class AgentXConfig: """Configuration for an InferenceX AgentX trace replay. The workload switch is simply ``agentx: enable``; ``docker_image`` and - ``benchmark_script`` remain explicit top-level benchmark pins. ``recipe`` + ``benchmark_script`` can override pins resolved from InferenceX. ``recipe`` and ``selector`` are escape hatches for configurations that cannot be selected unambiguously from model, framework, precision, GPU, and concurrency. @@ -681,6 +683,7 @@ class AgentXConfig: config_file: Optional[str] = None selector: Dict[str, Any] = field(default_factory=dict) failed_request_threshold: float = 0.10 + launch_overrides: Optional[Dict[str, Any]] = None resolved: Optional[Dict[str, Any]] = field(default=None, repr=False) def __post_init__(self) -> None: @@ -693,6 +696,8 @@ def __post_init__(self) -> None: raise ValueError("agentx.mode must be 'canonical' or 'fast'") if not 0.0 <= self.failed_request_threshold <= 1.0: raise ValueError("agentx.failed_request_threshold must be between 0 and 1") + if self.launch_overrides is not None: + self.launch_overrides = validate_overrides(self.launch_overrides) def to_dict(self) -> Dict[str, Any]: result: Dict[str, Any] = { @@ -710,6 +715,8 @@ def to_dict(self) -> Dict[str, Any]: result["selector"] = self.selector if self.resolved is not None: result["resolved"] = self.resolved + if self.launch_overrides is not None: + result["launch_overrides"] = self.launch_overrides return result @classmethod @@ -737,6 +744,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "AgentXConfig": config_file=data.get("config_file"), selector=dict(data.get("selector") or {}), failed_request_threshold=float(data.get("failed_request_threshold", 0.10)), + launch_overrides=data.get("launch_overrides"), ) @classmethod @@ -919,16 +927,6 @@ def __post_init__(self): ) if self.is_server_lifecycle: raise ValueError("AgentX cannot be combined with server_lifecycle") - if not self.benchmark_script: - raise ValueError( - "AgentX requires benchmark_script so the InferenceX " - "launcher is explicitly pinned" - ) - if self.run_mode == "docker" and not self.docker_image: - raise ValueError( - "Docker AgentX requires docker_image so the runtime is " - "explicitly pinned" - ) incompatible = [] if self.profiler.torch_profiler.enabled: incompatible.append("torch_profiler") diff --git a/docs/how-to/benchmarking/benchmark.md b/docs/how-to/benchmarking/benchmark.md index 6de3c01..56b3eeb 100644 --- a/docs/how-to/benchmarking/benchmark.md +++ b/docs/how-to/benchmarking/benchmark.md @@ -74,10 +74,17 @@ For AgentX, Magpie does not implement a separate replay client. It resolves the matching single-node recipe from InferenceX, runs the corresponding script under `benchmarks/single_node/agentic/`, and normalizes the resulting AgentX aggregate into `benchmark_report.json`. In YAML, `agentx: enable` is the -workload switch; `docker_image` and `benchmark_script` explicitly pin the two -runtime assets. Deployment details such as TP and KV offload remain owned by +workload switch; the image defaults to the recipe and the launcher is resolved +from InferenceX's `configs/agentx-launchers.json` when available. Explicit +`docker_image` and `benchmark_script` pins remain supported; older checkouts +require the script pin. Deployment details such as TP and KV offload remain owned by the InferenceX recipe. +Supported launchers also accept `agentx.launch_overrides: {version: 1}` to +record verified server launch evidence without changing the command. Add +structured argument/environment/source overrides for candidate measurements; +see [the launch extension contract](../../reference/benchmark-config.md#verified-server-launch-overrides). + AgentX gets input lengths and target output lengths from the trace dataset. Omit `ISL`, `OSL`, and `RANDOM_RANGE_RATIO` from YAML and `--input-len` / `--output-len` from the AgentX CLI; Magpie warns and discards these options diff --git a/docs/reference/benchmark-config.md b/docs/reference/benchmark-config.md index 9130fb0..9abcd0a 100644 --- a/docs/reference/benchmark-config.md +++ b/docs/reference/benchmark-config.md @@ -127,8 +127,11 @@ benchmark: ## AgentX trace replay -AgentX workload semantics are enabled with one line. The Docker image and -InferenceX launcher path are selected explicitly. The recipe and launcher +AgentX workload semantics are enabled with one line. The Docker image defaults +to the resolved recipe image. The InferenceX launcher path is resolved from +`configs/agentx-launchers.json` when that checkout declares the recipe; an +explicit `benchmark_script` must match this mapping. Older checkouts without +the manifest still require an explicit launcher. The recipe and launcher contents still come from the checkout at `inferencex_path`; pin that checkout to a commit outside Magpie when exact source reproducibility is required. Magpie detects the GPU, matches `model`/`framework`/`precision` against that @@ -187,11 +190,50 @@ benchmark: `MODEL_PREFIX`, `KV_OFFLOADING`, `KV_OFFLOAD_BACKEND`, and `TOTAL_CPU_DRAM_GB` are not AgentX YAML requirements. They are resolved -from the matching InferenceX recipe. `benchmark_script` is required in all -AgentX modes, and `docker_image` is required for Docker mode. These select the -launcher path and runtime image; they do not pin the InferenceX checkout. +from the matching InferenceX recipe. `benchmark_script` and `docker_image` +can explicitly select the launcher path and runtime image; they do not pin +the InferenceX checkout. Missing or ambiguous recipes/arms fail resolution. `agentx.recipe` remains an advanced ambiguity override. +### Verified server launch overrides + +For launchers declaring `launch_overrides_version: 1` in the InferenceX +manifest, `agentx.launch_overrides` applies a structured server-only extension: + +```yaml +agentx: + enabled: true + launch_overrides: + version: 1 + append_args: [--mem-fraction-static, '0.8'] + remove_args: [--mem-fraction-static] + replace_args: false + env: {SGLANG_USE_AITER: '0'} + unset_env: [] + executable: null + source_files: {} # absolute path -> expected lowercase SHA256 + absent_source_files: [] # absolute paths that must remain absent +``` + +Arguments are tokens, never shell expressions. Removal must match exactly one +existing long option and removes its following values up to the next long +option. Ambiguous short-option forms fail. `replace_args: true` retains the +executable, positionals, and model/host/port/topology options while replacing +the other serving options. Protocol options and replay environment controls +cannot be overridden. `executable`, when supplied, must be an absolute executable +path: use the Python interpreter for SGLang or the vLLM entrypoint for vLLM. +Environment changes affect the server process, not the replay/router process. + +Source hashes and required absences are checked after launcher setup and +immediately before server start. The exact normalized request is saved as +`agentx_launch_overrides.json`. `agentx_server_launch.json` records base/effective +argv, changed environment, filtered runtime controls, resolved executable, and +verified source identities. Both request and evidence have canonical JSON +SHA256 identities. Magpie rejects missing, corrupt, or mismatched evidence and +publishes verified evidence under `agentx_metrics.server_launch`. An empty +`{version: 1}` request records evidence without changing the canonical command. +Omitting `launch_overrides` preserves the older execution contract. + Magpie AgentX v1 is single-node and supports Docker or local execution. Its own trace-replay loop is incompatible with Ray, persistent-server reuse, PyTorch/system profiling, TraceLens, and gap analysis. Those profilers default diff --git a/tests/unit/modes/test_agentx.py b/tests/unit/modes/test_agentx.py index e2ba9e7..af5a23e 100644 --- a/tests/unit/modes/test_agentx.py +++ b/tests/unit/modes/test_agentx.py @@ -190,12 +190,10 @@ def test_agentx_rejects_magpie_profiler(): ) -def test_agentx_requires_pinned_script_and_docker_image(): - with pytest.raises(ValueError, match="benchmark_script"): - _minimal_config(benchmark_script=None) - - with pytest.raises(ValueError, match="docker_image"): - _minimal_config(docker_image=None) +def test_agentx_defers_optional_script_and_image_to_recipe_resolution(): + config = _minimal_config(benchmark_script=None, docker_image=None) + assert config.benchmark_script is None + assert config.docker_image is None def test_resolve_agentx_uses_inferencex_recipe(tmp_path): diff --git a/tests/unit/modes/test_agentx_launch.py b/tests/unit/modes/test_agentx_launch.py new file mode 100644 index 0000000..b6c65f4 --- /dev/null +++ b/tests/unit/modes/test_agentx_launch.py @@ -0,0 +1,365 @@ +"""The native launch extension must be explicit and backed by run evidence.""" + +import json + +import pytest +from test_agentx import _fake_inferencex, _minimal_config + +from Magpie.modes.benchmark import benchmarker +from Magpie.modes.benchmark.agentx import resolve_agentx_recipe +from Magpie.modes.benchmark.agentx_launch import ( + digest, + launch_environment, + read_launch_evidence, + validate_overrides, +) +from Magpie.modes.benchmark.benchmarker import BenchmarkMode +from Magpie.modes.benchmark.config import AgentXConfig, BenchmarkConfig +from Magpie.modes.benchmark.result import BenchmarkResult + +RECIPE = "dsv4-fp4-mi355x-sglang-agentic-mtp" +SCRIPT = "single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh" + + +def _declare_launcher(root, **changes): + row = { + "framework": "sglang", + "runner_type": "mi355x", + "model": "deepseek-ai/DeepSeek-V4-Pro-0813", + "precision": "fp4", + "benchmark_script": SCRIPT, + "launch_overrides_version": 1, + **changes, + } + path = root / "benchmarks" / SCRIPT + path.parent.mkdir(parents=True) + path.write_text("#!/bin/bash\n") + (root / "configs/agentx-launchers.json").write_text( + json.dumps({"version": 1, "recipes": {RECIPE: row}}) + ) + + +def _config(**overrides): + return _minimal_config(agentx={"launch_overrides": {"version": 1, **overrides}}) + + +def _evidence(config, workspace): + overrides = config.agentx.launch_overrides + names = set(overrides["env"]) | set(overrides["unset_env"]) + evidence = { + "version": 1, + "framework": "sglang", + "base_argv": ["python3", "-m", "sglang.launch_server"], + "effective_argv": [ + overrides["executable"] or "python3", + "-m", + "sglang.launch_server", + ], + "base_env": {name: "original" for name in names}, + "effective_env": {name: overrides["env"].get(name) for name in names}, + "source_files": overrides["source_files"], + "absent_source_files": overrides["absent_source_files"], + "overrides_sha256": digest(overrides), + "resolved_executable": "/usr/bin/python3", + "runtime_environment": {"PATH": "/usr/bin"}, + } + _write_evidence(workspace, evidence) + return evidence + + +def _write_evidence(workspace, evidence): + evidence["evidence_sha256"] = digest( + {key: value for key, value in evidence.items() if key != "evidence_sha256"} + ) + (workspace / "agentx_server_launch.json").write_text(json.dumps(evidence)) + + +def test_launcher_and_image_resolve_from_declared_recipe_without_user_pins(tmp_path): + root = _fake_inferencex(tmp_path) + _declare_launcher(root) + config = _minimal_config( + benchmark_script=None, + docker_image=None, + agentx={"launch_overrides": {"version": 1}}, + ) + spec = resolve_agentx_recipe(config, str(root), runner_type="mi355x") + assert config.benchmark_script == SCRIPT + assert config.docker_image == "sglang:test" + assert spec.entry["benchmark-script"] == SCRIPT + assert spec.entry["launch-overrides-version"] == 1 + assert config.to_dict()["benchmark_script"] == SCRIPT + + +def test_old_checkout_requires_explicit_launcher_and_cannot_ignore_overrides(tmp_path): + root = _fake_inferencex(tmp_path) + with pytest.raises(ValueError, match="no declared AgentX launcher"): + resolve_agentx_recipe( + _minimal_config(benchmark_script=None), str(root), runner_type="mi355x" + ) + with pytest.raises(ValueError, match="does not declare launch_overrides"): + resolve_agentx_recipe(_config(), str(root), runner_type="mi355x") + resolve_agentx_recipe(_minimal_config(), str(root), runner_type="mi355x") + + +@pytest.mark.parametrize( + "changes,match", + [ + ({"framework": "vllm"}, "framework mismatch"), + ({"model": "wrong/model"}, "model mismatch"), + ({"precision": "bf16"}, "precision mismatch"), + ({"runner_type": "mi300x"}, "runner mismatch"), + ({"benchmark_script": "../../escape.sh"}, "Invalid AgentX launcher path"), + ({"launch_overrides_version": None}, "does not declare"), + ], +) +def test_launcher_manifest_cannot_silently_change_identity(tmp_path, changes, match): + root = _fake_inferencex(tmp_path) + _declare_launcher(root, **changes) + with pytest.raises(ValueError, match=match): + resolve_agentx_recipe(_config(), str(root), runner_type="mi355x") + + +def test_explicit_launcher_cannot_disagree_with_declared_recipe(tmp_path): + root = _fake_inferencex(tmp_path) + _declare_launcher(root) + with pytest.raises(ValueError, match="declares launcher"): + resolve_agentx_recipe( + _minimal_config(benchmark_script="wrong.sh"), + str(root), + runner_type="mi355x", + ) + + +def test_config_roundtrip_and_saved_request_have_identical_normalized_hash(tmp_path): + config = _config( + append_args=["--json", '{"x": "a b"}'], env={"SGLANG_USE_AITER": "0"} + ) + serialized = config.to_dict() + restored = BenchmarkConfig.from_dict(serialized) + assert restored.agentx.launch_overrides == config.agentx.launch_overrides + environment = launch_environment(config, tmp_path) + request = json.loads((tmp_path / "agentx_launch_overrides.json").read_text()) + assert request == serialized["agentx"]["launch_overrides"] + assert environment["AGENTX_LAUNCH_OVERRIDES_SHA256"] == digest(request) + assert request["replace_args"] is False + assert request["executable"] is None + assert request["absent_source_files"] == [] + + +def test_docker_paths_refer_to_mounted_workspace_and_remove_stale_receipt(tmp_path): + (tmp_path / "agentx_server_launch.json").write_text("stale") + env = launch_environment(_config(), tmp_path, docker=True) + assert ( + env["AGENTX_LAUNCH_OVERRIDES_FILE"] == "/workspace/agentx_launch_overrides.json" + ) + assert env["AGENTX_SERVER_LAUNCH_FILE"] == "/workspace/agentx_server_launch.json" + assert not (tmp_path / "agentx_server_launch.json").exists() + + +def test_no_extension_clears_ambient_protocol_paths_without_creating_request(tmp_path): + env = launch_environment(_minimal_config(), tmp_path) + assert set(env.values()) == {""} + assert not list(tmp_path.iterdir()) + + +def test_valid_evidence_matches_exact_request_environment_and_sources(tmp_path): + config = _config( + env={"SGLANG_USE_AITER": "0"}, + unset_env=["OMP_NUM_THREADS"], + source_files={"/candidate/kernel.py": "a" * 64}, + absent_source_files=["/candidate/deleted.py"], + ) + launch_environment(config, tmp_path) + evidence = _evidence(config, tmp_path) + assert read_launch_evidence(config, tmp_path) == evidence + + +@pytest.mark.parametrize( + "field,value", + [ + ("overrides_sha256", "wrong"), + ("source_files", {}), + ("absent_source_files", []), + ("effective_env", {}), + ("base_env", {}), + ("framework", "vllm"), + ("version", 2), + ("effective_argv", []), + ("base_argv", "not-an-array"), + ], +) +def test_rehashed_but_mismatched_receipt_is_rejected(tmp_path, field, value): + config = _config( + env={"SGLANG_USE_AITER": "0"}, + source_files={"/x.py": "a" * 64}, + absent_source_files=["/gone.py"], + ) + launch_environment(config, tmp_path) + evidence = _evidence(config, tmp_path) + evidence[field] = value + _write_evidence(tmp_path, evidence) + with pytest.raises(ValueError): + read_launch_evidence(config, tmp_path) + + +def test_missing_or_tampered_launch_receipt_cannot_pass(tmp_path): + config = _config() + launch_environment(config, tmp_path) + with pytest.raises(FileNotFoundError): + read_launch_evidence(config, tmp_path) + evidence = _evidence(config, tmp_path) + evidence["evidence_sha256"] = "bad" + (tmp_path / "agentx_server_launch.json").write_text(json.dumps(evidence)) + with pytest.raises(ValueError, match="hash mismatch"): + read_launch_evidence(config, tmp_path) + + +@pytest.mark.parametrize( + "value", [{}, {"version": True}, {"version": 2}, {"version": 1, "unknown": True}] +) +def test_invalid_extension_versions_and_fields_rejected(value): + with pytest.raises(ValueError): + AgentXConfig(launch_overrides=value) + + +def test_source_and_argument_validation_is_applied_at_config_boundary(): + with pytest.raises(ValueError, match="absolute paths"): + _config(source_files={"relative": "a" * 64}) + with pytest.raises(ValueError, match="protocol option"): + _config(remove_args=["--tp"]) + assert validate_overrides({"version": 1})["remove_args"] == [] + + +@pytest.mark.parametrize( + "overrides", + [ + {"append_args": "--x"}, + {"remove_args": ["--x", "--x"]}, + {"remove_args": ["-x"]}, + {"env": []}, + {"env": {"A": 2}}, + {"env": {"A-B": "x"}}, + {"env": {"PORT": "1"}}, + {"env": {"A": "x"}, "unset_env": ["A"]}, + {"replace_args": 1}, + {"executable": "python"}, + {"absent_source_files": ["relative.py"]}, + ], +) +def test_malformed_requests_are_rejected_before_benchmark(overrides): + with pytest.raises(ValueError): + _config(**overrides) + + +def test_changed_config_or_persisted_request_cannot_rebind_evidence(tmp_path): + config = _config() + config.agentx.launch_overrides.pop("executable") + with pytest.raises(ValueError, match="normalization"): + launch_environment(config, tmp_path) + config = _config() + launch_environment(config, tmp_path) + (tmp_path / "agentx_launch_overrides.json").write_text("{}") + with pytest.raises(ValueError, match="differs"): + read_launch_evidence(config, tmp_path) + + +@pytest.mark.parametrize( + "field,value", + [ + ("resolved_executable", None), + ("runtime_environment", None), + ("runtime_environment", {"PATH": 123}), + ("effective_argv", ["wrong-executable"]), + ], +) +def test_incomplete_runtime_and_executable_evidence_rejected(tmp_path, field, value): + config = _config(executable="/candidate/bin/python") + launch_environment(config, tmp_path) + evidence = _evidence(config, tmp_path) + evidence[field] = value + _write_evidence(tmp_path, evidence) + with pytest.raises(ValueError): + read_launch_evidence(config, tmp_path) + + +@pytest.mark.parametrize( + "manifest", ["not json", "[]", '{"version": 2, "recipes": {}}'] +) +def test_invalid_launcher_manifest_is_never_silently_ignored(tmp_path, manifest): + root = _fake_inferencex(tmp_path) + (root / "configs/agentx-launchers.json").write_text(manifest) + with pytest.raises(ValueError): + resolve_agentx_recipe(_minimal_config(), str(root), runner_type="mi355x") + + +def test_declared_launcher_must_exist(tmp_path): + root = _fake_inferencex(tmp_path) + _declare_launcher(root) + (root / "benchmarks" / SCRIPT).unlink() + with pytest.raises(ValueError, match="missing or escapes"): + resolve_agentx_recipe(_config(), str(root), runner_type="mi355x") + + +@pytest.mark.parametrize("receipt_mode", ["valid", "missing", "wrong-request"]) +def test_benchmark_result_requires_this_runs_launch_evidence( + monkeypatch, tmp_path, receipt_mode +): + root = _fake_inferencex(tmp_path) + _declare_launcher(root) + config = _minimal_config( + inferencex_path=str(root), + run_mode="local", + runner_type="mi355x", + agentx={"launch_overrides": {"version": 1}}, + ) + mode = BenchmarkMode(config, output_dir=str(tmp_path / "results")) + monkeypatch.setattr(benchmarker, "ensure_inferencex_available", lambda path: path) + monkeypatch.setattr(benchmarker, "ensure_agentx_dependencies", lambda path: None) + monkeypatch.setattr(mode, "_apply_gpu_selection", lambda: None) + monkeypatch.setattr(mode, "_prepare_benchmark_scripts", lambda: None) + monkeypatch.setattr(mode, "_cleanup_server_processes", lambda framework: None) + + def execute(_cmd, env, workspace): + assert env["AGENTX_LAUNCH_OVERRIDES_FILE"] == str( + workspace / "agentx_launch_overrides.json" + ) + aggregate = { + "scenario_type": "agentic-coding", + "recipe_fingerprint": "a" * 64, + "num_requests_total": 10, + "num_requests_successful": 10, + "request_accounting": { + "records_total": 10, + "records_profiled": 10, + "records_error_dropped": 0, + }, + "request_metrics": { + "qps": {"mean": 1}, + "throughput": { + "input": {"tokens_per_second": 100}, + "output": {"tokens_per_second": 20}, + "total": {"tokens_per_second": 120}, + "duration_seconds": 3600, + }, + "latency": {"ttft": {"mean": 1}, "tpot": {"mean": 0.01}}, + }, + } + (workspace / "inferencex_result.json").write_text(json.dumps(aggregate)) + if receipt_mode != "missing": + evidence = _evidence(config, workspace) + if receipt_mode == "wrong-request": + evidence["overrides_sha256"] = "other-run" + _write_evidence(workspace, evidence) + return BenchmarkResult(success=True), "", "" + + monkeypatch.setattr(mode, "_execute_local_benchmark", execute) + result = mode.run("launch-extension-test") + assert result.success is (receipt_mode == "valid") + if receipt_mode == "valid": + assert result.agentx_metrics["server_launch"]["overrides_sha256"] == digest( + config.agentx.launch_overrides + ) + else: + assert result.benchmark_valid is False + assert result.publishable is False + assert any("launch evidence rejected" in error for error in result.errors) From d72965776df5416dad063c00237f6e389b841162 Mon Sep 17 00:00:00 2001 From: Hao Li <18546749+haofrank@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:45:45 -0700 Subject: [PATCH 03/16] feat(agentx): resolve custom native models with verified context Resolve unregistered AMD single-node SGLang/vLLM models through the declared InferenceX generic capability. Bind model config metadata and context to recipe identity, preserve registered recipe ambiguity checks, and expose verified custom result metadata. --- Magpie/modes/benchmark/agentx.py | 64 +++++-- Magpie/modes/benchmark/agentx_custom.py | 205 +++++++++++++++++++++ Magpie/modes/benchmark/agentx_launch.py | 3 + Magpie/modes/benchmark/result.py | 24 +++ docs/reference/benchmark-config.md | 38 ++++ tests/unit/modes/test_agentx_custom.py | 226 ++++++++++++++++++++++++ 6 files changed, 545 insertions(+), 15 deletions(-) create mode 100644 Magpie/modes/benchmark/agentx_custom.py create mode 100644 tests/unit/modes/test_agentx_custom.py diff --git a/Magpie/modes/benchmark/agentx.py b/Magpie/modes/benchmark/agentx.py index 87b5a4c..f779f21 100644 --- a/Magpie/modes/benchmark/agentx.py +++ b/Magpie/modes/benchmark/agentx.py @@ -18,6 +18,7 @@ import yaml +from .agentx_custom import resolve_custom_entry from .config import AgentXConfig, BenchmarkConfig logger = logging.getLogger(__name__) @@ -28,6 +29,10 @@ _DEFAULT_AGENTIC_DURATION_SECONDS = 3600 +class NoMatchingAgentXRecipe(ValueError): + """The registered matrix has no recipe matching the requested identity.""" + + @dataclass(frozen=True) class AgentXLaunchSpec: """One concrete single-node AgentX point resolved from InferenceX.""" @@ -148,7 +153,7 @@ def _infer_recipe( if not matches: locations = ", ".join(checked) or "" - raise ValueError( + raise NoMatchingAgentXRecipe( "InferenceX has no single-node AgentX recipe matching " f"model={config.model!r}, framework={config.framework!r}, " f"precision={config.precision!r}, gpu={runner_type!r} in: " @@ -425,14 +430,28 @@ def resolve_agentx_recipe( inferencex_root = Path(inferencex_path).resolve() agentx = config.agentx - if not agentx.recipe: + custom_name = f"custom-{config.framework}-{runner_type}" + requested_custom = agentx.recipe == custom_name + if not agentx.recipe or requested_custom: if not runner_type: raise ValueError("runner_type is required when agentx.recipe is not set") - config_file = _infer_recipe( - config, - inferencex_root, - runner_type, - ) + try: + config_file = _infer_recipe(config, inferencex_root, runner_type) + except NoMatchingAgentXRecipe: + entry = resolve_custom_entry(config, inferencex_root, runner_type) + if entry is None: + raise + if not _selector_matches(entry, agentx.selector): + raise ValueError("Custom AgentX point does not match agentx.selector") + entry["recipe-fingerprint"] = _recipe_fingerprint(entry) + _resolve_launcher(config, inferencex_root, custom_name, entry, runner_type) + _apply_launch_entry(config, entry) + agentx.resolved = entry + return AgentXLaunchSpec(custom_name, "configs/agentx-launchers.json", entry) + if requested_custom: + raise ValueError( + "Custom AgentX identity now matches a registered recipe; select it explicitly" + ) else: config_file = _find_recipe_config(inferencex_root, agentx) recipe = agentx.recipe @@ -555,13 +574,19 @@ def _resolve_launcher( or not isinstance(data.get("recipes"), dict) ): raise ValueError(f"Unsupported AgentX launcher manifest: {manifest}") - row = data["recipes"].get(recipe) + entries = data.get("generic", {}) if entry.get("custom") else data["recipes"] + row = entries.get(recipe) if isinstance(entries, dict) else None if row is not None: if not isinstance(row, dict) or row.get("framework") != config.framework: - raise ValueError(f"AgentX launcher manifest framework mismatch for {recipe}") - for field, expected in ( - ("model", config.model), ("precision", config.precision) - ): + raise ValueError( + f"AgentX launcher manifest framework mismatch for {recipe}" + ) + identities = ( + () + if entry.get("custom") + else (("model", config.model), ("precision", config.precision)) + ) + for field, expected in identities: if row.get(field) != expected: raise ValueError( f"AgentX launcher manifest {field} mismatch for {recipe}" @@ -569,9 +594,7 @@ def _resolve_launcher( if runner_type and row.get("runner_type") != runner_type.lower(): raise ValueError(f"AgentX launcher manifest runner mismatch for {recipe}") script = row.get("benchmark_script") - if not isinstance(script, str) or not script.startswith( - "single_node/agentic/" - ): + if not isinstance(script, str) or not script.startswith("single_node/agentic/"): raise ValueError(f"Invalid AgentX launcher path for {recipe}") path = (root / "benchmarks" / script).resolve() agentic_root = (root / "benchmarks" / "single_node" / "agentic").resolve() @@ -655,6 +678,17 @@ def _apply_launch_entry(config: BenchmarkConfig, entry: Dict[str, Any]) -> None: "AIPERF_FAILED_REQUEST_THRESHOLD": (config.agentx.failed_request_threshold), } ) + if entry.get("custom"): + runtime_env.update( + { + "AGENTX_CUSTOM_RECIPE": "1", + "AGENTX_NATIVE_CONTEXT_LENGTH": entry["native-context-length"], + "AGENTX_MAX_MODEL_LEN": entry["max-model-len"], + "AGENTX_MODEL_CONFIG_SHA256": entry["model-config-sha256"], + "MAX_MODEL_LEN": entry["max-model-len"], + "WEKA_LOADER_OVERRIDE": entry["trace-loader"], + } + ) config.envs = runtime_env # Loading a large model plus the canonical one-hour profile can exceed the diff --git a/Magpie/modes/benchmark/agentx_custom.py b/Magpie/modes/benchmark/agentx_custom.py new file mode 100644 index 0000000..778d435 --- /dev/null +++ b/Magpie/modes/benchmark/agentx_custom.py @@ -0,0 +1,205 @@ +"""Resolve explicit custom-model AgentX points without modifying InferenceX.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import quote +from urllib.request import Request, urlopen + +if TYPE_CHECKING: + from .config import BenchmarkConfig + + +def native_context_length(config: dict[str, Any]) -> int: + text = config.get("text_config", config.get("language_config", config)) + if not isinstance(text, dict): + raise ValueError("Model text configuration must be an object") # noqa: TRY004 + values = [ + text.get(name) + for name in ( + "max_position_embeddings", + "max_sequence_length", + "seq_length", + "n_positions", + ) + ] + confirmed = [value for value in values if type(value) is int and value > 0] + if not confirmed: + raise ValueError( + "config.json does not declare a positive native context length" + ) + return min(confirmed) + + +def _model_metadata(config: BenchmarkConfig) -> tuple[dict[str, Any], str, str]: + envs = {key.upper(): value for key, value in config.envs.items()} + local = str(envs.get("MODEL_PATH") or "") + if local: + path = Path(local).expanduser() / "config.json" + try: + contents = path.read_bytes() + except OSError as exc: + raise ValueError( + f"Custom AgentX requires readable local model metadata: {path}" + ) from exc + source = str(path.resolve()) + elif Path(config.model).is_dir(): + path = Path(config.model) / "config.json" + contents = path.read_bytes() + source = str(path.resolve()) + else: + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", config.model): + raise ValueError( + "Custom AgentX model must be a local directory or HuggingFace model ID" + ) + source = f"https://huggingface.co/{quote(config.model, safe='/')}/resolve/main/config.json" + headers = {"Accept": "application/json"} + token = os.environ.get("HF_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + try: + with urlopen(Request(source, headers=headers), timeout=30) as response: + contents = response.read(8 * 1024 * 1024 + 1) + except OSError as exc: + raise ValueError( + f"Cannot read custom AgentX model config metadata for {config.model}" + ) from exc + if len(contents) > 8 * 1024 * 1024: + raise ValueError("Custom AgentX model config metadata exceeds 8 MiB") + data = json.loads(contents) + if not isinstance(data, dict): + raise ValueError("Custom AgentX config.json must contain an object") # noqa: TRY004 + return data, hashlib.sha256(contents).hexdigest(), source + + +def _positive_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not re.fullmatch(r"[1-9][0-9]*", str(value)): + raise ValueError(f"Custom AgentX {name} must be an explicit positive integer") + return int(value) + + +def resolve_custom_entry( + config: BenchmarkConfig, + root: Path, + runner_type: str, +) -> dict[str, Any] | None: + """Return a declared custom point, or None when no generic capability exists.""" + assert config.agentx is not None + name = f"custom-{config.framework}-{runner_type}" + manifest = root / "configs/agentx-launchers.json" + if not manifest.is_file(): + return None + data = json.loads(manifest.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("version") != 1: + raise ValueError("Unsupported AgentX launcher manifest") + generic = data.get("generic", {}) + if not isinstance(generic, dict): + raise ValueError("AgentX generic launcher manifest must be an object") # noqa: TRY004 + row = generic.get(name) + if row is None: + return None + if ( + not isinstance(row, dict) + or row.get("framework") != config.framework + or row.get("runner_type") != runner_type + ): + raise ValueError(f"Generic AgentX capability identity mismatch: {name}") + if row.get("launch_overrides_version") != 1: + raise ValueError("Generic AgentX requires the verified launch extension v1") + if not config.docker_image: + raise ValueError( + "Custom AgentX requires an explicit docker_image, including local runs" + ) + if config.framework not in {"sglang", "vllm"}: + raise ValueError("Custom AgentX supports SGLang and vLLM only") + if any(character.isspace() for character in config.model): + raise ValueError( + "Custom AgentX model identifier cannot contain whitespace; use MODEL_PATH for local files" + ) + envs = {key.upper(): value for key, value in config.envs.items()} + if envs.get("SERVED_MODEL_NAME", config.model) != config.model: + raise ValueError( + "Custom AgentX SERVED_MODEL_NAME must match the model identity" + ) + tp = _positive_int(envs.get("TP"), "TP") + ep = _positive_int(envs.get("EP_SIZE", envs.get("EP")), "EP_SIZE") + if "EP" in envs and _positive_int(envs["EP"], "EP") != ep: + raise ValueError("Custom AgentX EP and EP_SIZE disagree") + conc = _positive_int(config.agentx.concurrency or envs.get("CONC"), "CONC") + if tp > int(row.get("max_gpus", 8)) or tp % ep: + raise ValueError("Custom AgentX EP must divide TP within one declared GPU node") + if config.framework == "vllm" and ep not in {1, tp}: + raise ValueError("Custom vLLM AgentX supports EP=1 or EP=TP") + for key in ("PP", "PP_SIZE", "DCP_SIZE", "PCP_SIZE", "DP_SIZE", "NNODES"): + if str(envs.get(key, 1)) != "1": + raise ValueError(f"Custom AgentX supports {key}=1 only") + for key in ("DP_ATTENTION", "DISAGG"): + if str(envs.get(key, False)).lower() not in {"false", "0", ""}: + raise ValueError(f"Custom AgentX does not support {key}") + if str(envs.get("KV_OFFLOADING", "none")) != "none" or envs.get( + "KV_OFFLOAD_BACKEND" + ): + raise ValueError("Custom AgentX currently supports KV_OFFLOADING=none only") + if str(envs.get("SPEC_DECODING", "none")) != "none": + raise ValueError("Custom AgentX does not infer speculative decoding recipes") + precision = config.precision.lower() + if precision not in { + "bf16", + "bfloat16", + "fp16", + "float16", + "fp32", + "float32", + "auto", + "fp8", + "fp4", + "mxfp4", + }: + raise ValueError(f"Unsupported custom AgentX precision: {precision}") + metadata, checksum, source = _model_metadata(config) + if precision in {"fp8", "fp4", "mxfp4"} and not metadata.get("quantization_config"): + raise ValueError( + "Quantized custom AgentX requires quantization_config in model config.json" + ) + native = native_context_length(metadata) + maximum = _positive_int(envs.get("MAX_MODEL_LEN", native), "MAX_MODEL_LEN") + if maximum > native: + raise ValueError( + f"Custom AgentX MAX_MODEL_LEN={maximum} exceeds confirmed native context {native}" + ) + config.agentx.recipe = name + config.agentx.concurrency = conc + return { + "custom": True, + "model": config.model, + "framework": config.framework, + "precision": precision, + "image": config.docker_image, + "model-prefix": "custom-" + + re.sub(r"[^A-Za-z0-9_.-]+", "-", config.model.rsplit("/", 1)[-1]), + "runner": runner_type, + "tp": tp, + "pp": 1, + "ep": ep, + "dcp-size": 1, + "pcp-size": 1, + "dp-attn": False, + "conc": conc, + "spec-decoding": "none", + "kv-offloading": "none", + "total-cpu-dram-gb": 0, + "duration": 3600, + "run-eval": False, + "scenario-type": "agentic-coding", + "exp-name": f"{name}_tp{tp}_conc{conc}", + "native-context-length": native, + "max-model-len": maximum, + "model-config-sha256": checksum, + "model-config-source": source, + "trace-loader": "semianalysis_cc_traces_weka_062126_256k", + } diff --git a/Magpie/modes/benchmark/agentx_launch.py b/Magpie/modes/benchmark/agentx_launch.py index 1990aa2..8e8212c 100644 --- a/Magpie/modes/benchmark/agentx_launch.py +++ b/Magpie/modes/benchmark/agentx_launch.py @@ -15,6 +15,8 @@ { "--model", "--model-path", + "--max-model-len", + "--context-length", "--served-model-name", "--host", "--port", @@ -50,6 +52,7 @@ { "MODEL", "MODEL_PATH", + "MAX_MODEL_LEN", "MODEL_NAME", "SERVED_MODEL_NAME", "MODEL_PREFIX", diff --git a/Magpie/modes/benchmark/result.py b/Magpie/modes/benchmark/result.py index df5d52a..d05d8be 100644 --- a/Magpie/modes/benchmark/result.py +++ b/Magpie/modes/benchmark/result.py @@ -613,6 +613,26 @@ def _parse_agentx_data( "AgentX result is missing a positive duration or total throughput" ) + if data.get("custom_recipe"): + native_context = data.get("native_context_length") + max_context = data.get("max_model_len") + metadata_hash = data.get("model_config_sha256") + if not ( + data["custom_recipe"] is True + and type(native_context) is int + and type(max_context) is int + and 0 < max_context <= native_context + and isinstance(metadata_hash, str) + and len(metadata_hash) == 64 + and all( + character in "0123456789abcdef" for character in metadata_hash + ) + ): + valid = False + result.errors.append( + "Custom AgentX result has invalid model/context metadata" + ) + result.success = valid result.benchmark_valid = valid result.publishable = valid and mode == "canonical" and fingerprint_valid @@ -664,6 +684,10 @@ def _parse_agentx_data( "allocated_cpu_dram_gb", "router", "kv_p2p_transfer", + "custom_recipe", + "native_context_length", + "max_model_len", + "model_config_sha256", ) if key in data }, diff --git a/docs/reference/benchmark-config.md b/docs/reference/benchmark-config.md index 9abcd0a..9ac9904 100644 --- a/docs/reference/benchmark-config.md +++ b/docs/reference/benchmark-config.md @@ -234,6 +234,44 @@ publishes verified evidence under `agentx_metrics.server_launch`. An empty `{version: 1}` request records evidence without changing the canonical command. Omitting `launch_overrides` preserves the older execution contract. +When no registered recipe matches, a checkout with the manifest's `generic` +capabilities can run a custom SGLang or vLLM model on MI300X/MI325X/MI355X: + +```yaml +benchmark: + model: Qwen/Qwen3-0.6B + framework: sglang + precision: bf16 + docker_image: your-tested-sglang-image@sha256:your-digest + agentx: + enabled: true + launch_overrides: {version: 1} + envs: + MODEL_PATH: /models/Qwen3-0.6B + TP: 1 + EP_SIZE: 1 + CONC: 64 +``` + +No matching recipe triggers this path; multiple registered matches still require +an explicit selection. The resolved public name is `custom--`. +An explicit runtime image and TP/EP are required. TP must fit one eight-GPU node; +SGLang EP must divide TP, while vLLM supports EP=1 or EP=TP. PP/DCP/PCP remain 1, +with no DP attention, disaggregation, KV offload, or inferred speculative decoding. +Quantized precisions require the model's own `quantization_config`. + +Custom replay uses the same `inferencex-agentx-mvp` scenario and canonical duration. +Magpie reads local `MODEL_PATH/config.json`, or only the remote HuggingFace +`config.json` metadata when MODEL_PATH is absent. It never executes model code +or downloads weights during resolution. The native context, metadata SHA256, +fixed trace loader, and optional `MAX_MODEL_LEN` cap become recipe identity. +The cap cannot exceed confirmed native context. The launcher verifies the same +metadata bytes before server start and passes the fixed cap to both server and +replay. Launch overrides cannot change that context. Custom results carry +`custom_recipe`, `native_context_length`, `max_model_len`, and +`model_config_sha256` in raw output and `agentx_metrics.recipe`; results with +different model/context/metadata identities are different workloads. + Magpie AgentX v1 is single-node and supports Docker or local execution. Its own trace-replay loop is incompatible with Ray, persistent-server reuse, PyTorch/system profiling, TraceLens, and gap analysis. Those profilers default diff --git a/tests/unit/modes/test_agentx_custom.py b/tests/unit/modes/test_agentx_custom.py new file mode 100644 index 0000000..800c7d9 --- /dev/null +++ b/tests/unit/modes/test_agentx_custom.py @@ -0,0 +1,226 @@ +"""Custom AgentX uses declared native launchers and verified model metadata.""" + +import hashlib +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from Magpie.modes.benchmark.agentx import _recipe_fingerprint, resolve_agentx_recipe +from Magpie.modes.benchmark.agentx_custom import native_context_length +from Magpie.modes.benchmark.config import BenchmarkConfig + + +@pytest.fixture +def custom_config(tmp_path): + root = tmp_path / "InferenceX" + (root / "configs").mkdir(parents=True) + (root / "configs/amd-master.yaml").write_text("{}\n") + manifest = {"version": 1, "recipes": {}, "generic": {}} + for framework in ("sglang", "vllm"): + script = f"single_node/agentic/generic_{framework}.sh" + target = root / "benchmarks" / script + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("#!/bin/bash\n") + for runner in ("mi300x", "mi325x", "mi355x"): + manifest["generic"][f"custom-{framework}-{runner}"] = { + "framework": framework, + "runner_type": runner, + "benchmark_script": script, + "launch_overrides_version": 1, + "max_gpus": 8, + } + (root / "configs/agentx-launchers.json").write_text(json.dumps(manifest)) + model = tmp_path / "model with spaces" + model.mkdir() + (model / "config.json").write_text(json.dumps({"max_position_embeddings": 40960})) + data = { + "model": "Qwen/Qwen3-0.6B", + "framework": "sglang", + "precision": "bf16", + "docker_image": "sglang:pinned", + "agentx": {"enabled": True, "launch_overrides": {"version": 1}}, + "envs": {"MODEL_PATH": str(model), "TP": 1, "EP_SIZE": 1, "CONC": 64}, + } + return root, data + + +@pytest.mark.parametrize("framework", ["sglang", "vllm"]) +@pytest.mark.parametrize("runner", ["mi300x", "mi325x", "mi355x"]) +def test_custom_recipe_resolves_and_roundtrips_without_registered_matrix( + custom_config, framework, runner +): + root, data = custom_config + data["framework"] = framework + config = BenchmarkConfig.from_dict(data) + spec = resolve_agentx_recipe(config, str(root), runner) + assert spec.recipe == f"custom-{framework}-{runner}" + assert config.benchmark_script == f"single_node/agentic/generic_{framework}.sh" + assert spec.entry["custom"] is True + assert spec.entry["native-context-length"] == 40960 + assert spec.entry["max-model-len"] == 40960 + assert config.envs["RECIPE_FINGERPRINT"] == spec.entry["recipe-fingerprint"] + fingerprint_entry = { + k: v + for k, v in spec.entry.items() + if k not in {"benchmark-script", "launch-overrides-version"} + } + assert _recipe_fingerprint(fingerprint_entry) == spec.entry["recipe-fingerprint"] + assert ( + config.envs["WEKA_LOADER_OVERRIDE"] == "semianalysis_cc_traces_weka_062126_256k" + ) + snapshot = config.to_dict() + reloaded = BenchmarkConfig.from_dict(deepcopy(snapshot)) + assert resolve_agentx_recipe(reloaded, str(root), runner).entry == spec.entry + assert reloaded.to_dict() == snapshot + + +def test_custom_context_and_metadata_are_fingerprinted(custom_config): + root, data = custom_config + data["envs"]["MAX_MODEL_LEN"] = 8192 + config = BenchmarkConfig.from_dict(data) + entry = resolve_agentx_recipe(config, str(root), "mi355x").entry + assert entry["max-model-len"] == 8192 + assert config.envs["AGENTX_MAX_MODEL_LEN"] == 8192 + path = Path(data["envs"]["MODEL_PATH"]) / "config.json" + assert entry["model-config-sha256"] == hashlib.sha256(path.read_bytes()).hexdigest() + path.write_text('{"max_position_embeddings":32768}') + changed = resolve_agentx_recipe(config, str(root), "mi355x").entry + assert changed["recipe-fingerprint"] != entry["recipe-fingerprint"] + + +@pytest.mark.parametrize( + "env", + [ + {"MAX_MODEL_LEN": 50000}, + {"MAX_MODEL_LEN": 0}, + {"TP": 16}, + {"TP": 4, "EP_SIZE": 3}, + {"PP_SIZE": 2}, + {"NNODES": 2}, + {"DP_SIZE": 2}, + {"DISAGG": True}, + {"DP_ATTENTION": True}, + {"KV_OFFLOADING": "dram"}, + {"SPEC_DECODING": "mtp"}, + {"EP": 2}, + ], +) +def test_unsupported_custom_point_fails_closed(custom_config, env): + root, data = custom_config + data["envs"].update(env) + with pytest.raises(ValueError): + resolve_agentx_recipe(BenchmarkConfig.from_dict(data), str(root), "mi355x") + + +def test_custom_requires_explicit_image(custom_config): + root, data = custom_config + data.pop("docker_image") + with pytest.raises(ValueError, match="explicit docker_image"): + resolve_agentx_recipe(BenchmarkConfig.from_dict(data), str(root), "mi355x") + + +def test_custom_vllm_does_not_infer_partial_ep(custom_config): + root, data = custom_config + data["framework"] = "vllm" + data["envs"].update(TP=4, EP_SIZE=2) + with pytest.raises(ValueError, match="EP=1 or EP=TP"): + resolve_agentx_recipe(BenchmarkConfig.from_dict(data), str(root), "mi355x") + + +@pytest.mark.parametrize( + "metadata, expected", + [ + ({"max_position_embeddings": 40960}, 40960), + ({"text_config": {"max_position_embeddings": 32768}}, 32768), + ({"max_position_embeddings": 40960, "seq_length": 8192}, 8192), + ], +) +def test_context_is_derived_without_model_code(metadata, expected): + assert native_context_length(metadata) == expected + + +@pytest.mark.parametrize( + "metadata", + [ + {}, + {"max_position_embeddings": True}, + {"max_position_embeddings": -1}, + {"text_config": []}, + ], +) +def test_unknown_native_context_is_not_guessed(metadata): + with pytest.raises(ValueError): + native_context_length(metadata) + + +def test_hf_metadata_request_uses_only_config_and_does_not_persist_auth( + custom_config, monkeypatch +): + import io + + from Magpie.modes.benchmark import agentx_custom + + root, data = custom_config + data["envs"].pop("MODEL_PATH") + requests = [] + + def fetch(request, timeout): + requests.append((request, timeout)) + return io.BytesIO(b'{"max_position_embeddings":40960}') + + monkeypatch.setattr(agentx_custom, "urlopen", fetch) + monkeypatch.setenv("HF_TOKEN", "secret-test-value") + entry = resolve_agentx_recipe( + BenchmarkConfig.from_dict(data), str(root), "mi355x" + ).entry + assert ( + requests[0][0].full_url + == "https://huggingface.co/Qwen/Qwen3-0.6B/resolve/main/config.json" + ) + assert requests[0][0].get_header("Authorization") == "Bearer secret-test-value" + assert "secret-test-value" not in json.dumps(entry) + + +@pytest.mark.parametrize( + "changes, valid", + [ + ({}, True), + ({"max_model_len": 50000}, False), + ({"model_config_sha256": "bad"}, False), + ({"native_context_length": None}, False), + ], +) +def test_custom_result_preserves_and_checks_model_metadata(tmp_path, changes, valid): + from Magpie.modes.benchmark.result import ResultParser + + aggregate = { + "scenario_type": "agentic-coding", + "recipe_fingerprint": "a" * 64, + "num_requests_total": 2, + "num_requests_successful": 2, + "request_accounting": { + "records_total": 2, + "records_profiled": 2, + "records_error_dropped": 0, + "records_warmup_dropped": 0, + }, + "request_metrics": { + "throughput": {"total": {"tokens_per_second": 20}, "duration_seconds": 3600} + }, + "custom_recipe": True, + "native_context_length": 40960, + "max_model_len": 8192, + "model_config_sha256": "b" * 64, + **changes, + } + path = tmp_path / "aggregate.json" + path.write_text(json.dumps(aggregate)) + result = ResultParser.parse_inferencex_result(path, scenario="agentx") + assert result.benchmark_valid is valid + assert result.publishable is valid + assert result.agentx_metrics["recipe"]["custom_recipe"] is True + assert ( + result.agentx_metrics["recipe"]["max_model_len"] == aggregate["max_model_len"] + ) From f0a24ae53a2b08026aa0f6481fa760662fc23f9d Mon Sep 17 00:00:00 2001 From: Hao Li <18546749+haofrank@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:56:44 -0700 Subject: [PATCH 04/16] Use pinned InferenceX clients with Magpie-owned AgentX serving --- Magpie/config.yaml | 3 +- Magpie/main.py | 8 +- Magpie/mcp/server.py | 4 +- Magpie/modes/benchmark/agentx.py | 101 ++- Magpie/modes/benchmark/agentx_custom.py | 83 ++- Magpie/modes/benchmark/agentx_launch.py | 278 +++++++- Magpie/modes/benchmark/agentx_recipe.py | 547 +++++++++++++++ Magpie/modes/benchmark/agentx_runtime.py | 650 ++++++++++++++++++ Magpie/modes/benchmark/benchmarker.py | 89 ++- Magpie/modes/benchmark/config.py | 9 +- Magpie/modes/benchmark/inferencex.py | 190 +++-- Magpie/modes/benchmark/tracelens_inference.py | 93 +-- Magpie/scripts/agentx/client_legacy.sh | 15 + Magpie/scripts/benchmark/atom_mi300x.sh | 2 +- Magpie/scripts/benchmark/atom_mi355x.sh | 2 +- .../benchmark/magpie_bench_remote_compat.sh | 175 +++-- Magpie/scripts/benchmark/sglang_mi300x.sh | 2 +- Magpie/scripts/benchmark/sglang_mi355x.sh | 2 +- .../scripts/benchmark/sglang_radeon8060s.sh | 2 +- Magpie/scripts/benchmark/vllm_gfx12.sh | 2 +- Magpie/scripts/benchmark/vllm_mi300x.sh | 2 +- Magpie/scripts/benchmark/vllm_mi355x.sh | 2 +- Magpie/scripts/benchmark/vllm_radeon8060s.sh | 2 +- docs/conceptual/benchmarking-architecture.md | 18 + docs/how-to/benchmarking/benchmark.md | 40 +- .../benchmarking/persistent-server-reuse.md | 6 + docs/reference/benchmark-config.md | 81 ++- ...ang_deepseek_v4_pro_fp4_mi355x_agentx.yaml | 9 +- pyproject.toml | 2 +- tests/fixtures/inferencex_408c015/LICENSE | 201 ++++++ tests/fixtures/inferencex_408c015/README.md | 8 + .../configs/pip-runtime-deps.sh | 11 + .../benchmarks/runtime_settings.sh | 35 + .../dsv4/vllm/mi355x-fp4-mtp/agentic.yaml | 231 +++++++ .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 203 ++++++ .../vllm/mi325x-fp8-mtp/agentic.yaml | 145 ++++ .../sglang/mi300x-fp8-mtp/agentic.yaml | 175 +++++ .../configs/amd-master.yaml | 137 ++++ .../inferencex_408c015/configs/runners.yaml | 34 + .../inferencex_408c015/infx/__init__.py | 1 + .../infx/golden_al_distribution/__init__.py | 28 + .../infx/golden_al_distribution/curves.py | 90 +++ .../dsv4-pro-0813-dspark.yaml | 19 + .../dsv41flash_dspark.yaml | 23 + .../infx/golden_al_distribution/dsv4_mtp.yaml | 27 + .../golden_al_distribution/glm5.2_mtp.yaml | 19 + .../golden_al_distribution/glm5.3_mtp.yaml | 19 + .../golden_al_distribution/kimik3_dspark.yaml | 18 + ..._method_block_rejection_sample_method.yaml | 19 + .../minimaxm3_eagle3.yaml | 28 + .../minimaxm3_eagle3_gqa.yaml | 28 + .../golden_al_distribution/qwen3.5_mtp.yaml | 28 + .../qwen3.8next_mtp.yaml | 17 + .../infx/srt_slurm/__init__.py | 1 + .../infx/srt_slurm/single_node.py | 236 +++++++ .../infx/srt_slurm/synthetic_acceptance.py | 266 +++++++ tests/test_benchmark_support.py | 1 + tests/test_benchmark_unit_coverage.py | 45 +- tests/test_eval_concurrency_scripts.py | 5 +- tests/test_eval_source_paths.py | 2 +- tests/test_gfx1151_container_compatibility.py | 5 +- tests/test_gfx12_mm_benchmark.py | 1 + tests/unit/modes/test_agentx.py | 19 +- tests/unit/modes/test_agentx_attestation.py | 249 +++++++ tests/unit/modes/test_agentx_custom.py | 9 +- tests/unit/modes/test_agentx_managed.py | 168 +++++ tests/unit/modes/test_agentx_recipe.py | 632 +++++++++++++++++ tests/unit/modes/test_agentx_runtime.py | 641 +++++++++++++++++ tests/unit/modes/test_inferencex_paths.py | 280 ++++++++ 69 files changed, 6221 insertions(+), 302 deletions(-) create mode 100644 Magpie/modes/benchmark/agentx_recipe.py create mode 100644 Magpie/modes/benchmark/agentx_runtime.py create mode 100644 Magpie/scripts/agentx/client_legacy.sh create mode 100644 tests/fixtures/inferencex_408c015/LICENSE create mode 100644 tests/fixtures/inferencex_408c015/README.md create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/runtime_settings.sh create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml create mode 100644 tests/fixtures/inferencex_408c015/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml create mode 100644 tests/fixtures/inferencex_408c015/configs/amd-master.yaml create mode 100644 tests/fixtures/inferencex_408c015/configs/runners.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/__init__.py create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/__init__.py create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/curves.py create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/dsv4-pro-0813-dspark.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/dsv41flash_dspark.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/dsv4_mtp.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/glm5.2_mtp.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/glm5.3_mtp.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/kimik3_dspark.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/minimaxm3_eagle3.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/minimaxm3_eagle3_gqa.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/qwen3.5_mtp.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/golden_al_distribution/qwen3.8next_mtp.yaml create mode 100644 tests/fixtures/inferencex_408c015/infx/srt_slurm/__init__.py create mode 100644 tests/fixtures/inferencex_408c015/infx/srt_slurm/single_node.py create mode 100644 tests/fixtures/inferencex_408c015/infx/srt_slurm/synthetic_acceptance.py create mode 100644 tests/unit/modes/test_agentx_attestation.py create mode 100644 tests/unit/modes/test_agentx_managed.py create mode 100644 tests/unit/modes/test_agentx_recipe.py create mode 100644 tests/unit/modes/test_agentx_runtime.py create mode 100644 tests/unit/modes/test_inferencex_paths.py diff --git a/Magpie/config.yaml b/Magpie/config.yaml index 21f6f90..947700e 100644 --- a/Magpie/config.yaml +++ b/Magpie/config.yaml @@ -194,9 +194,10 @@ benchmark: # Path to separate image mapping configuration file image_config: "./benchmark_images.yaml" - # Path to InferenceX installation + # InferenceX repository root or its inferencex-e2e project directory # If not configured or path doesn't exist, InferenceX will be auto-cloned from: # https://github.com/SemiAnalysisAI/InferenceX.git + # New clones pin commit 408c015be4b22d14c69518643609669405507077; existing checkouts are preserved. inferencex_path: "" # Default timeout for benchmark tasks diff --git a/Magpie/main.py b/Magpie/main.py index 2ce7173..6784b34 100644 --- a/Magpie/main.py +++ b/Magpie/main.py @@ -1081,8 +1081,8 @@ def run_benchmark(args, config: Dict[str, Any]) -> int: if run_mode: benchmark_cfg["run_mode"] = run_mode - # AgentX remains an InferenceX-owned workload. This flag only asks Magpie - # to select and execute the matching InferenceX AgentX launcher/recipe. + # AgentX preserves the InferenceX workload while Magpie owns serving for + # the packaged client layout; old checkouts retain their launcher path. if getattr(args, "agentx", False): benchmark_cfg["agentx"] = { "enabled": True, @@ -1304,7 +1304,7 @@ def create_parser() -> argparse.ArgumentParser: benchmark_parser.add_argument( "--agentx", action="store_true", - help="Run the matching InferenceX AgentX trace-replay launcher", + help="Run AgentX trace replay with the matching InferenceX recipe", ) benchmark_parser.add_argument( "--agentx-mode", @@ -1324,7 +1324,7 @@ def create_parser() -> argparse.ArgumentParser: benchmark_parser.add_argument( "--inferencex-path", type=str, default="", - help="Path to InferenceX installation (auto-cloned if not specified)" + help="InferenceX repository or inferencex-e2e directory (auto-cloned at a pinned commit)" ) benchmark_parser.add_argument( "--benchmark-script", type=str, diff --git a/Magpie/mcp/server.py b/Magpie/mcp/server.py index 124996e..bc3ac89 100644 --- a/Magpie/mcp/server.py +++ b/Magpie/mcp/server.py @@ -1238,7 +1238,7 @@ async def benchmark( Default: ["gpu_user_annotation"] docker_image: Override automatic Docker image selection (optional) gpu_arch: Force GPU architecture, e.g. "gfx942" (auto-detected if omitted) - inferencex_path: Path to InferenceX installation (auto-cloned if empty) + inferencex_path: InferenceX repository or inferencex-e2e project directory (auto-cloned at a pinned commit if empty) hf_cache_path: HuggingFace cache directory (default: ~/.cache/huggingface) benchmark_script: Override benchmark script name (e.g., "dsr1_fp8_mi300x.sh") runner_type: Hardware runner type (e.g., "mi300x", "h100") - auto-detected if omitted @@ -1255,7 +1255,7 @@ async def benchmark( ray_multi_node: Whether the benchmark needs multiple nodes (default: False) ray_total_num_gpus: Total GPUs across nodes for multi-node (default: 8) ray_num_nodes: Number of nodes for multi-node (default: 1) - agentx: Run the matching InferenceX AgentX launcher and recipe + agentx: Run trace replay with the matching InferenceX AgentX recipe agentx_mode: "canonical" (publishable) or "fast" (validation only) Returns: diff --git a/Magpie/modes/benchmark/agentx.py b/Magpie/modes/benchmark/agentx.py index f779f21..01a4a88 100644 --- a/Magpie/modes/benchmark/agentx.py +++ b/Magpie/modes/benchmark/agentx.py @@ -19,7 +19,9 @@ import yaml from .agentx_custom import resolve_custom_entry +from .agentx_recipe import CLIENT_SCRIPT, custom_server_spec, native_server_spec from .config import AgentXConfig, BenchmarkConfig +from .inferencex import resolve_inferencex_root logger = logging.getLogger(__name__) @@ -40,6 +42,7 @@ class AgentXLaunchSpec: recipe: str config_file: str entry: Dict[str, Any] + server: Optional[Dict[str, Any]] = None def _recipe_fingerprint(entry: Dict[str, Any]) -> str: @@ -186,6 +189,24 @@ def _selector_matches(entry: Dict[str, Any], selector: Dict[str, Any]) -> bool: return True +def _requested_topology_matches(entry: Dict[str, Any], config: BenchmarkConfig) -> bool: + """A fresh native request's explicit topology narrows the matrix as well.""" + if not entry.get("srt-recipe"): + return True + names = { + "TP": "tp", + "EP": "ep", + "EP_SIZE": "ep", + "PP_SIZE": "pp", + "DCP_SIZE": "dcp-size", + "PCP_SIZE": "pcp-size", + } + for name, field in names.items(): + if name in config.envs and str(config.envs[name]) != str(entry.get(field, 1)): + return False + return True + + def _describe_point(entry: Dict[str, Any]) -> str: fields = ( "tp", @@ -414,6 +435,8 @@ def _expand_single_node_agentx_entries( component = arm.get(key, value.get(key)) if component is not None: entry[key] = component + if arm.get("srt-recipe"): + entry["srt-recipe"] = arm["srt-recipe"] entries.append(entry) return entries @@ -428,7 +451,23 @@ def resolve_agentx_recipe( if config.agentx is None: raise ValueError("AgentX configuration is missing") - inferencex_root = Path(inferencex_path).resolve() + # Resolution runs in the caller's directory, while the server and client + # run in InferenceX. Bind local paths once before reading metadata or + # constructing argv so both phases (and Docker mounts) use the same files. + model_path_keys = [key for key in config.envs if key.upper() == "MODEL_PATH"] + if len(model_path_keys) > 1: + raise ValueError("AgentX has duplicate MODEL_PATH environment settings") + if model_path_keys: + key = model_path_keys[0] + value = config.envs.pop(key) + if not isinstance(value, str) or not value.strip(): + raise ValueError("AgentX MODEL_PATH must be a nonempty local path") + config.envs["MODEL_PATH"] = str(Path(value).expanduser().resolve()) + local_model = Path(config.model).expanduser() + if local_model.is_dir(): + config.model = str(local_model.resolve()) + + inferencex_root = Path(resolve_inferencex_root(inferencex_path)) agentx = config.agentx custom_name = f"custom-{config.framework}-{runner_type}" requested_custom = agentx.recipe == custom_name @@ -447,7 +486,16 @@ def resolve_agentx_recipe( _resolve_launcher(config, inferencex_root, custom_name, entry, runner_type) _apply_launch_entry(config, entry) agentx.resolved = entry - return AgentXLaunchSpec(custom_name, "configs/agentx-launchers.json", entry) + return AgentXLaunchSpec( + custom_name, + ( + "magpie:custom" + if entry.get("server-launch-spec") + else "configs/agentx-launchers.json" + ), + entry, + entry.get("server-launch-spec"), + ) if requested_custom: raise ValueError( "Custom AgentX identity now matches a registered recipe; select it explicitly" @@ -475,6 +523,7 @@ def resolve_agentx_recipe( if isinstance(entry, dict) and entry.get("scenario-type") == "agentic-coding" and _selector_matches(entry, agentx.selector) + and _requested_topology_matches(entry, config) ] if not entries: selector = json.dumps(agentx.selector, sort_keys=True) @@ -537,6 +586,12 @@ def resolve_agentx_recipe( entry["recipe-fingerprint"] = _recipe_fingerprint(entry) _resolve_launcher(config, inferencex_root, recipe, entry, runner_type) + if entry.get("server-launch-spec"): + for source in (config_file, inferencex_root / "configs/runners.yaml"): + entry["server-launch-spec"]["source_files"][str(source.resolve())] = ( + hashlib.sha256(source.read_bytes()).hexdigest() + ) + entry["recipe-fingerprint"] = _recipe_fingerprint(entry) _apply_launch_entry(config, entry) try: @@ -547,6 +602,7 @@ def resolve_agentx_recipe( recipe=recipe, config_file=config_file_label, entry=entry, + server=entry.get("server-launch-spec"), ) agentx.resolved = entry logger.info("Resolved AgentX recipe %s: %s", recipe, _describe_point(entry)) @@ -561,6 +617,30 @@ def _resolve_launcher( runner_type: Optional[str], ) -> None: """Use InferenceX's declared recipe/launcher mapping, never filename guesses.""" + client = root / "benchmarks" / CLIENT_SCRIPT + if entry.get("srt-recipe") or (entry.get("custom") and client.is_file()): + if not client.is_file(): + raise ValueError( + "Native AgentX requires the official srt_agentic.sh client" + ) + if config.benchmark_script not in {None, "", CLIENT_SCRIPT}: + raise ValueError( + "Magpie-owned AgentX uses benchmark_script=srt_agentic.sh; " + "legacy server launcher paths cannot be used with a native YAML recipe" + ) + server = ( + custom_server_spec(config, root, entry) + if entry.get("custom") + else native_server_spec(config, root, entry) + ) + config.benchmark_script = CLIENT_SCRIPT + entry["benchmark-script"] = CLIENT_SCRIPT + entry["launch-overrides-version"] = 1 + entry["server-launch-spec"] = server + # Native YAML source, variant, golden acceptance, and actual server + # configuration are part of the materialized Magpie recipe identity. + entry["recipe-fingerprint"] = _recipe_fingerprint(entry) + return manifest = root / "configs" / "agentx-launchers.json" row = None if manifest.is_file(): @@ -689,6 +769,11 @@ def _apply_launch_entry(config: BenchmarkConfig, entry: Dict[str, Any]) -> None: "WEKA_LOADER_OVERRIDE": entry["trace-loader"], } ) + if entry.get("model-revision"): + runtime_env["AGENTX_MODEL_REVISION"] = entry["model-revision"] + server = entry.get("server-launch-spec") + if server: + runtime_env.update(server["client_env"]) config.envs = runtime_env # Loading a large model plus the canonical one-hour profile can exceed the @@ -700,10 +785,11 @@ def _apply_launch_entry(config: BenchmarkConfig, entry: Dict[str, Any]) -> None: def ensure_agentx_dependencies(inferencex_path: str) -> None: """Initialize the pinned AIPerf submodule required by AgentX.""" - root = Path(inferencex_path).resolve() + root = Path(resolve_inferencex_root(inferencex_path)) requirements = root / "utils" / "agentic-benchmark" / "requirements.txt" aiperf_project = root / "utils" / "aiperf" / "pyproject.toml" - if not requirements.is_file(): + native_client = root / "benchmarks" / CLIENT_SCRIPT + if not requirements.is_file() and not native_client.is_file(): raise RuntimeError( "InferenceX checkout does not contain AgentX support: " f"missing {requirements}" @@ -711,16 +797,19 @@ def ensure_agentx_dependencies(inferencex_path: str) -> None: if aiperf_project.is_file(): return + repo_root = root.parent if root.name == "inferencex-e2e" else root + submodule = str((root / "utils" / "aiperf").relative_to(repo_root)) completed = subprocess.run( [ "git", "-C", - str(root), + str(repo_root), "submodule", "update", "--init", "--recursive", - "utils/aiperf", + "--", + submodule, ], capture_output=True, text=True, diff --git a/Magpie/modes/benchmark/agentx_custom.py b/Magpie/modes/benchmark/agentx_custom.py index 778d435..8438ffd 100644 --- a/Magpie/modes/benchmark/agentx_custom.py +++ b/Magpie/modes/benchmark/agentx_custom.py @@ -4,7 +4,6 @@ import hashlib import json -import os import re from pathlib import Path from typing import TYPE_CHECKING, Any @@ -36,9 +35,12 @@ def native_context_length(config: dict[str, Any]) -> int: return min(confirmed) -def _model_metadata(config: BenchmarkConfig) -> tuple[dict[str, Any], str, str]: +def _model_metadata( + config: BenchmarkConfig, *, pin_remote: bool = False +) -> tuple[dict[str, Any], str, str, str | None]: envs = {key.upper(): value for key, value in config.envs.items()} local = str(envs.get("MODEL_PATH") or "") + revision = None if local: path = Path(local).expanduser() / "config.json" try: @@ -57,24 +59,43 @@ def _model_metadata(config: BenchmarkConfig) -> tuple[dict[str, Any], str, str]: raise ValueError( "Custom AgentX model must be a local directory or HuggingFace model ID" ) - source = f"https://huggingface.co/{quote(config.model, safe='/')}/resolve/main/config.json" + saved = (config.agentx.resolved if config.agentx else None) or {} + revision = ( + saved.get("model-revision") + if saved.get("custom") + else envs.get("AGENTX_MODEL_REVISION") + ) + if revision is not None and not re.fullmatch(r"[0-9a-f]{40}", str(revision)): + raise ValueError("Custom AgentX model revision must be an immutable commit") + source = f"https://huggingface.co/{quote(config.model, safe='/')}/resolve/{revision or 'main'}/config.json" headers = {"Accept": "application/json"} - token = os.environ.get("HF_TOKEN") - if token: - headers["Authorization"] = f"Bearer {token}" try: with urlopen(Request(source, headers=headers), timeout=30) as response: contents = response.read(8 * 1024 * 1024 + 1) + observed = getattr(response, "headers", {}).get("X-Repo-Commit") + if observed: + if revision and revision != observed: + raise ValueError("Custom model metadata revision changed") + revision = observed except OSError as exc: raise ValueError( - f"Cannot read custom AgentX model config metadata for {config.model}" + f"Cannot read public model config metadata for {config.model}; " + "gated/private models require a local MODEL_PATH" ) from exc if len(contents) > 8 * 1024 * 1024: raise ValueError("Custom AgentX model config metadata exceeds 8 MiB") + if pin_remote and not re.fullmatch(r"[0-9a-f]{40}", str(revision or "")): + raise ValueError( + "Custom AgentX remote config must identify its immutable HF revision" + ) + if revision: + source = f"https://huggingface.co/{quote(config.model, safe='/')}/resolve/{revision}/config.json" data = json.loads(contents) if not isinstance(data, dict): - raise ValueError("Custom AgentX config.json must contain an object") # noqa: TRY004 - return data, hashlib.sha256(contents).hexdigest(), source + raise ValueError( + "Custom AgentX config.json must contain an object" + ) # noqa: TRY004 + return data, hashlib.sha256(contents).hexdigest(), source, revision def _positive_int(value: Any, name: str) -> int: @@ -92,17 +113,32 @@ def resolve_custom_entry( assert config.agentx is not None name = f"custom-{config.framework}-{runner_type}" manifest = root / "configs/agentx-launchers.json" - if not manifest.is_file(): - return None - data = json.loads(manifest.read_text(encoding="utf-8")) - if not isinstance(data, dict) or data.get("version") != 1: - raise ValueError("Unsupported AgentX launcher manifest") - generic = data.get("generic", {}) - if not isinstance(generic, dict): - raise ValueError("AgentX generic launcher manifest must be an object") # noqa: TRY004 - row = generic.get(name) - if row is None: - return None + owned = (root / "benchmarks/srt_agentic.sh").is_file() + if owned: + if runner_type not in {"mi300x", "mi325x", "mi355x"}: + raise ValueError( + "Custom AgentX supports AMD MI300X, MI325X, and MI355X only" + ) + row = { + "framework": config.framework, + "runner_type": runner_type, + "launch_overrides_version": 1, + "max_gpus": 8, + } + else: + if not manifest.is_file(): + return None + data = json.loads(manifest.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("version") != 1: + raise ValueError("Unsupported AgentX launcher manifest") + generic = data.get("generic", {}) + if not isinstance(generic, dict): + raise ValueError( + "AgentX generic launcher manifest must be an object" + ) # noqa: TRY004 + row = generic.get(name) + if row is None: + return None if ( not isinstance(row, dict) or row.get("framework") != config.framework @@ -161,7 +197,7 @@ def resolve_custom_entry( "mxfp4", }: raise ValueError(f"Unsupported custom AgentX precision: {precision}") - metadata, checksum, source = _model_metadata(config) + metadata, checksum, source, revision = _model_metadata(config, pin_remote=owned) if precision in {"fp8", "fp4", "mxfp4"} and not metadata.get("quantization_config"): raise ValueError( "Quantized custom AgentX requires quantization_config in model config.json" @@ -174,7 +210,7 @@ def resolve_custom_entry( ) config.agentx.recipe = name config.agentx.concurrency = conc - return { + result = { "custom": True, "model": config.model, "framework": config.framework, @@ -203,3 +239,6 @@ def resolve_custom_entry( "model-config-source": source, "trace-loader": "semianalysis_cc_traces_weka_062126_256k", } + if revision: + result["model-revision"] = revision + return result diff --git a/Magpie/modes/benchmark/agentx_launch.py b/Magpie/modes/benchmark/agentx_launch.py index 8e8212c..d3e2546 100644 --- a/Magpie/modes/benchmark/agentx_launch.py +++ b/Magpie/modes/benchmark/agentx_launch.py @@ -4,7 +4,9 @@ import hashlib import json +import os import re +import shutil from pathlib import Path from typing import TYPE_CHECKING, Any @@ -15,6 +17,9 @@ { "--model", "--model-path", + "--revision", + "--code-revision", + "--tokenizer-revision", "--max-model-len", "--context-length", "--served-model-name", @@ -171,6 +176,131 @@ def validate_overrides(value: Any) -> dict[str, Any]: return result +def option_groups(argv: list[str]) -> tuple[list[str], list[list[str]]]: + """Separate fixed executable/positionals and unambiguous long options.""" + prefix: list[str] = [] + groups: list[list[str]] = [] + for token in argv: + if token.startswith("--"): + name = token.partition("=")[0] + if not re.fullmatch(r"--[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise ValueError(f"Ambiguous server option: {token!r}") + groups.append([token]) + elif groups: + # Short options are ambiguous with values and cannot safely be + # removed as part of the preceding long option. Negative numbers + # are ordinary values. Equals-style options already own a value. + if "=" in groups[-1][0] or ( + token.startswith("-") + and not re.fullmatch(r"-\d+(?:\.\d+)?(?:[eE][+-]?\d+)?", token) + ): + raise ValueError(f"Ambiguous server option value: {token!r}") + groups[-1].append(token) + else: + prefix.append(token) + return prefix, groups + + +def apply_launch_args( + argv: list[str], + overrides: dict[str, Any], + framework: str, +) -> list[str]: + """Apply the v1 argv contract without reading files or the process environment.""" + overrides = validate_overrides(overrides) + if framework not in {"sglang", "vllm"} or not argv: + raise ValueError( + "AgentX launch extension supports nonempty SGLang/vLLM commands" + ) + effective = list(argv) + # Empty requests do not parse/rewrite an existing command. This preserves + # every canonical launcher token, including options unfamiliar to v1. + if ( + overrides["append_args"] + or overrides["remove_args"] + or overrides["replace_args"] + ): + prefix, groups = option_groups(argv) + extra_prefix, extra = option_groups(overrides["append_args"]) + if extra_prefix: + raise ValueError( + "append_args cannot inject an executable or positional arguments" + ) + if any(group[0].partition("=")[0] in PROTECTED_ARGS for group in extra): + raise ValueError("append_args cannot change model, topology, host or port") + for name in overrides["remove_args"]: + matches = [group for group in groups if group[0].partition("=")[0] == name] + if len(matches) != 1: + raise ValueError( + f"remove_args must match exactly one existing option: {name}" + ) + groups.remove(matches[0]) + if overrides["replace_args"]: + # Keep the executable/model positionals and all protocol options; + # only the optional serving configuration is replaced. + groups = [ + group + for group in groups + if group[0].partition("=")[0] in PROTECTED_ARGS + ] + effective = prefix + [token for group in groups + extra for token in group] + if overrides["executable"] is not None: + effective[0] = overrides["executable"] + return effective + + +def _validate_golden_launch(spec: dict[str, Any], effective: list[str]) -> None: + """A candidate cannot change the acceptance curve without re-resolution.""" + if not spec.get("golden_acceptance_overrides"): + return + + def values(argv: list[str]) -> dict[str, list[list[str]]]: + result: dict[str, list[list[str]]] = {} + for group in option_groups(argv)[1]: + key, equal, inline = group[0].partition("=") + result.setdefault(key, []).append(([inline] if equal else []) + group[1:]) + return result + + base, changed = values(spec["argv"]), values(effective) + if spec["framework"] == "sglang": + keys = { + "--speculative-algorithm", + "--speculative-algo", + "--speculative-num-steps", + "--speculative-dspark-block-size", + "--speculative-draft-model-path", + } + if any(base.get(key) != changed.get(key) for key in keys): + raise ValueError( + "Changing AgentX golden acceptance inputs requires a new resolved recipe" + ) + elif spec["framework"] == "vllm": + + def parameters(options): + groups = options.get("--speculative-config", []) + if not groups: + return {} + if len(groups) != 1 or len(groups[0]) != 1: + raise ValueError( + "AgentX requires exactly one speculative-config JSON object" + ) + value = json.loads(groups[0][0]) + if not isinstance(value, dict): + raise ValueError("AgentX speculative-config must be a JSON object") + return value + + old, new = parameters(base), parameters(changed) + keys = { + "method", + "model", + "num_speculative_tokens", + "rejection_sample_method", + "synthetic_acceptance_length", + } + if any(old.get(key) != new.get(key) for key in keys): + raise ValueError("AgentX golden acceptance parameters cannot be overridden") + + def launch_environment( config: BenchmarkConfig, workspace: Path, @@ -208,8 +338,8 @@ def launch_environment( def read_launch_evidence(config: BenchmarkConfig, workspace: Path) -> dict[str, Any]: """Verify the receipt against the immutable request persisted for this run.""" - assert config.agentx is not None and config.agentx.launch_overrides is not None - overrides = config.agentx.launch_overrides + assert config.agentx is not None + overrides = validate_overrides(config.agentx.launch_overrides or {"version": 1}) request = json.loads( (workspace / "agentx_launch_overrides.json").read_text(encoding="utf-8") ) @@ -221,7 +351,9 @@ def read_launch_evidence(config: BenchmarkConfig, workspace: Path) -> dict[str, (workspace / "agentx_server_launch.json").read_text(encoding="utf-8") ) if not isinstance(evidence, dict): - raise ValueError("AgentX server launch evidence must be an object") # noqa: TRY004 + raise ValueError( + "AgentX server launch evidence must be an object" + ) # noqa: TRY004 payload = { key: value for key, value in evidence.items() if key != "evidence_sha256" } @@ -269,4 +401,144 @@ def read_launch_evidence(config: BenchmarkConfig, workspace: Path) -> dict[str, for name, value in runtime.items() ): raise ValueError("AgentX runtime environment evidence is incomplete") + spec = (config.agentx.resolved or {}).get("server-launch-spec") + if spec is not None: + if evidence.get("owner") != "magpie": + raise ValueError("AgentX server launch evidence is not Magpie-owned") + if evidence.get("server_spec_sha256") != digest(spec): + raise ValueError("AgentX server launch specification identity mismatch") + if evidence.get("recipe_source_files") != spec.get("source_files", {}): + raise ValueError("AgentX recipe source evidence is incomplete") + if evidence["effective_argv"] != apply_launch_args( + evidence["base_argv"], overrides, config.framework + ): + raise ValueError("AgentX effective argv does not match this candidate") + _validate_golden_launch(spec, evidence["effective_argv"]) + if evidence["base_argv"] != spec.get("argv"): + raise ValueError("AgentX base argv does not match the resolved recipe") return evidence + + +def prepare_server_launch( + argv: list[str], + environment: dict[str, str], + overrides: dict[str, Any] | None, + framework: str, + workspace: Path, + *, + server_spec: dict[str, Any] | None = None, +) -> tuple[list[str], dict[str, str], dict[str, Any]]: + """Attest a Magpie-owned server in its actual runtime, after setup. + + The caller immediately spawns the returned argv/environment. This function + must run inside the serving container for Docker jobs. The client never + invokes it, so client setup cannot overwrite the server's receipt. + """ + workspace.mkdir(parents=True, exist_ok=True) + receipt = workspace / "agentx_server_launch.json" + receipt.unlink(missing_ok=True) + request = validate_overrides(overrides or {"version": 1}) + if not isinstance(argv, list) or any( + not isinstance(token, str) or "\0" in token for token in argv + ): + raise ValueError("AgentX server argv must contain literal string tokens") + if any( + not isinstance(name, str) + or not isinstance(value, str) + or "\0" in name + or "\0" in value + for name, value in environment.items() + ): + raise ValueError("AgentX server environment must contain strings without NUL") + if server_spec is not None and server_spec.get("argv") != argv: + raise ValueError("AgentX server argv differs from its resolved specification") + effective = apply_launch_args(argv, request, framework) + if server_spec is not None: + _validate_golden_launch(server_spec, effective) + child_env = dict(environment) + touched = sorted(set(request["env"]) | set(request["unset_env"])) + base_env = {name: environment.get(name) for name in touched} + for name in request["unset_env"]: + child_env.pop(name, None) + child_env.update(request["env"]) + + sources = {} + for name, expected in request["source_files"].items(): + actual = hashlib.sha256(Path(name).read_bytes()).hexdigest() + if actual != expected: + raise ValueError(f"AgentX source changed before server launch: {name}") + sources[name] = actual + recipe_sources = {} + for name, expected in (server_spec or {}).get("source_files", {}).items(): + if not Path(name).is_absolute() or not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ValueError("AgentX recipe sources require absolute paths and SHA256") + actual = hashlib.sha256(Path(name).read_bytes()).hexdigest() + if actual != expected: + raise ValueError( + f"AgentX recipe input changed before server launch: {name}" + ) + recipe_sources[name] = actual + for name in request["absent_source_files"]: + if os.path.lexists(name): + raise ValueError(f"AgentX deleted source reappeared before launch: {name}") + resolved_executable = shutil.which(effective[0], path=child_env.get("PATH", "")) + if resolved_executable is None: + raise ValueError(f"AgentX server executable not found: {effective[0]}") + evidence: dict[str, Any] = { + "version": 1, + "owner": "magpie", + "framework": framework, + "base_argv": list(argv), + "effective_argv": effective, + "base_env": base_env, + "effective_env": {name: child_env.get(name) for name in touched}, + "overrides_sha256": digest(request), + "source_files": sources, + "absent_source_files": request["absent_source_files"], + "resolved_executable": str(Path(resolved_executable).resolve()), + "runtime_environment": { + name: value + for name, value in sorted(child_env.items()) + if ( + name + in { + "PATH", + "PYTHONPATH", + "LD_LIBRARY_PATH", + "LIBRARY_PATH", + "GPU_ARCHS", + } + or name.startswith( + ( + "SGLANG_", + "VLLM_", + "AITER_", + "TRITON_", + "TORCH_", + "PYTORCH_", + "HIP_", + "ROCR_", + "HSA_", + "NCCL_", + "RCCL_", + "OMP_", + ) + ) + ) + and not any( + word in name.upper() + for word in ("TOKEN", "PASSWORD", "SECRET", "API_KEY") + ) + }, + } + if server_spec is not None: + evidence["server_spec_sha256"] = digest(server_spec) + evidence["recipe_source_files"] = recipe_sources + evidence["evidence_sha256"] = digest(evidence) + (workspace / "agentx_launch_overrides.json").write_text( + json.dumps(request, indent=2) + "\n", encoding="utf-8" + ) + temporary = receipt.with_name(receipt.name + ".tmp") + temporary.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") + temporary.replace(receipt) + return effective, child_env, evidence diff --git a/Magpie/modes/benchmark/agentx_recipe.py b/Magpie/modes/benchmark/agentx_recipe.py new file mode 100644 index 0000000..831df38 --- /dev/null +++ b/Magpie/modes/benchmark/agentx_recipe.py @@ -0,0 +1,547 @@ +"""Materialize single-node native recipes without a Slurm deployment. + +InferenceX still owns point validation and golden acceptance. Magpie owns the +literal server command and its lifecycle; unsupported deployment semantics are +rejected instead of silently discarded. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import yaml + +if TYPE_CHECKING: + from .config import BenchmarkConfig + +CLIENT_SCRIPT = "srt_agentic.sh" + +# These upstream APIs perform no submission and import srtctl only in other, +# unused functions. A separate interpreter avoids altering Magpie's sys.path or +# reusing a module cached from a different checkout. +_VALIDATE = """ +import json, sys +sys.path.insert(0, sys.argv[1]) +from infx.srt_slurm.single_node import validate_recipe +from infx.srt_slurm.synthetic_acceptance import build_overrides +data = json.load(sys.stdin) +matches, errors = [], [] +for name, recipe in data['variants']: + try: + validate_recipe(recipe, data['environment']) + except (ValueError, KeyError, TypeError) as exc: + errors.append(f'{name}: {exc}') + else: + matches.append((name, recipe)) +if len(matches) != 1: + raise ValueError('Expected exactly one matching native recipe: ' + + ('; '.join(errors) if not matches else str([x[0] for x in matches]))) +name, recipe = matches[0] +overrides = build_overrides(recipe, data['environment']['FRAMEWORK'], data['environment']) +print(json.dumps({'variant': name, 'recipe': recipe, 'overrides': overrides})) +""" + + +def _merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = _merge(result[key], value) + else: + result[key] = copy.deepcopy(value) + return result + + +def _variants(raw: dict[str, Any], selector: str) -> list[tuple[str, dict[str, Any]]]: + if "base" not in raw: + if selector: + raise ValueError("Native recipe selector requires a base/override recipe") + return [("", raw)] + if any(key.startswith("zip_override_") for key in raw): + raise ValueError("Magpie does not support zipped native recipe sweeps") + if not isinstance(raw["base"], dict): + raise ValueError("Native recipe base must be a mapping") + names = sorted(key for key in raw if key.startswith("override_")) + if selector == "base" or not names: + return [("base", raw["base"])] + if selector: + if selector not in names: + raise ValueError(f"Unknown native recipe variant: {selector}") + names = [selector] + if any(not isinstance(raw[name], dict) for name in names): + raise ValueError("Native recipe overrides must be mappings") + return [(name, _merge(raw["base"], raw[name])) for name in names] + + +def _strings(value: Any, description: str) -> dict[str, str]: + if not isinstance(value, dict) or any( + not isinstance(key, str) + or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) + or not isinstance(item, (str, int, float, bool)) + for key, item in value.items() + ): + raise ValueError(f"{description} must map environment names to scalar values") + return { + key: str(item).lower() if isinstance(item, bool) else str(item) + for key, item in value.items() + } + + +def _apply_golden(recipe: dict[str, Any], overrides: list[str]) -> None: + if len(overrides) % 2: + raise ValueError("Malformed upstream golden acceptance overrides") + for operation, raw in zip(overrides[::2], overrides[1::2]): + path, _, value = raw.partition("=") + keys = path.split(".") + if keys[0] not in {"roles", "environment"}: + raise ValueError(f"Unexpected upstream golden acceptance target: {path}") + target = recipe + for key in keys[:-1]: + if not isinstance(target, dict): + raise ValueError("Native recipe roles/environment must be mappings") + target = target.setdefault(key, {}) + if not isinstance(target, dict): + raise ValueError("Native recipe roles/environment must be mappings") + if operation == "--set": + target[keys[-1]] = json.loads(value) + elif operation == "--unset": + target.pop(keys[-1], None) + else: + raise ValueError( + f"Unexpected upstream golden acceptance operation: {operation}" + ) + + +def _flags(arguments: dict[str, Any]) -> list[str]: + tokens: list[str] = [] + for name, value in arguments.items(): + if not isinstance(name, str) or not re.fullmatch(r"[a-zA-Z][\w-]*", name): + raise ValueError(f"Invalid native server option: {name!r}") + if value is None or value is False: + continue + tokens.append("--" + name) + if value is True: + continue + if isinstance(value, dict): + tokens.append(json.dumps(value, separators=(",", ":"))) + elif isinstance(value, list): + if any(not isinstance(item, (str, int, float)) for item in value): + raise ValueError(f"Unsupported native server list argument: {name}") + tokens.extend(str(item) for item in value) + elif isinstance(value, (str, int, float)): + tokens.append(str(value)) + else: + raise ValueError(f"Unsupported native server argument: {name}") + return tokens + + +def _port(config: BenchmarkConfig) -> int: + raw = config.envs.get("PORT", 8888) + if isinstance(raw, bool) or not str(raw).isdigit() or not 0 < int(raw) < 65536: + raise ValueError("AgentX PORT must be an integer between 1 and 65535") + return int(raw) + + +def _file_hashes(paths: list[Path]) -> dict[str, str]: + try: + return { + str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest() + for path in paths + } + except OSError as exc: + raise ValueError( + f"Required AgentX protocol/recipe source is unreadable: {exc}" + ) from exc + + +def resolve_client_sources(root: Path) -> dict[str, str]: + """Bind the official client, shared shell, aggregation, and acceptance gate.""" + files = [ + root / "benchmarks/srt_agentic.sh", + root / "benchmarks/benchmark_lib.sh", + root / "benchmarks/runtime_settings.sh", + root / "runners/srt-slurm/hooks/common.sh", + root / "infx/results/agentic/process_agentic_result.py", + root / "infx/results/agentic/validate_agentic_result.py", + root / "infx/__init__.py", + root / "infx/results/__init__.py", + root / "infx/results/metadata.py", + root / "infx/results/topology.py", + ] + files += sorted((root / "infx/results/agentic").rglob("*.py")) + files += sorted((root / "infx/results/power").rglob("*.py")) + return _file_hashes(files) + + +def _client_revisions(root: Path) -> dict[str, str]: + """Record checkout and harness pins when resolving an actual Git checkout.""" + head = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=10, + ) + if head.returncode or not re.fullmatch(r"[0-9a-f]{40}", head.stdout.strip()): + return {} + result = {"client_revision": head.stdout.strip()} + harness = subprocess.run( + ["git", "-C", str(root), "ls-tree", "HEAD", "--", "utils/aiperf"], + capture_output=True, + text=True, + timeout=10, + ) + match = re.fullmatch( + r"160000 commit ([0-9a-f]{40})\tutils/aiperf", harness.stdout.strip() + ) + if harness.returncode == 0 and match: + result["aiperf_revision"] = match[1] + return result + + +def _client_defaults(root: Path, config: BenchmarkConfig) -> dict[str, str]: + result = {} + for line in (root / "benchmarks/runtime_settings.sh").read_text().splitlines(): + match = re.fullmatch(r"export ([A-Z_]+)='([^']*)'", line) + if match and ( + match[1].startswith(("AIPERF_", "AGENTIC_")) + or match[1] == "ENABLE_AGENTX_POWER" + ): + result[match[1]] = match[2] + required = { + "AIPERF_PYTHON_VERSION", + "AIPERF_LIVE_FAILED_REQUEST_THRESHOLD", + "AIPERF_TRACE_IDLE_GAP_CAP_SECONDS", + "AIPERF_WARMUP_REQUESTS_PER_LANE", + "AGENTIC_WARMUP_GRACE_PERIOD", + "AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES", + "AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS", + "AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID", + "AIPERF_UNSAFE_OVERRIDE", + "AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING", + "ENABLE_AGENTX_POWER", + } + if any(not result.get(name) for name in required): + raise ValueError( + "InferenceX client runtime settings are missing literal AgentX defaults" + ) + assert config.agentx is not None + result["AIPERF_FAILED_REQUEST_THRESHOLD"] = str( + config.agentx.failed_request_threshold + ) + result["AIPERF_EXPERIMENTAL_FAST"] = "1" if config.agentx.mode == "fast" else "0" + return result + + +def _server_environment(config: BenchmarkConfig) -> dict[str, str]: + # Materialization writes replay metadata back to config.envs. Do not feed + # those fields into the server or change spec identity on snapshot reload. + if any(key.upper().startswith("SGLANG_SIMULATE_ACC_") for key in config.envs): + raise ValueError("AgentX golden acceptance environment is owned by InferenceX") + client_names = { + "MODEL", + "MODEL_PATH", + "MODEL_PREFIX", + "FRAMEWORK", + "IMAGE", + "PRECISION", + "EXP_NAME", + "TP", + "PP", + "PP_SIZE", + "EP", + "EP_SIZE", + "DCP_SIZE", + "PCP_SIZE", + "DP_ATTENTION", + "CONC", + "SPEC_DECODING", + "DISAGG", + "IS_AGENTIC", + "IS_MULTINODE", + "KV_OFFLOADING", + "KV_OFFLOAD_BACKEND", + "KV_OFFLOAD_BACKEND_METADATA", + "ROUTER_METADATA", + "KV_P2P_TRANSFER", + "TOTAL_CPU_DRAM_GB", + "DURATION", + "RUN_EVAL", + "EVAL_ONLY", + "RECIPE_FINGERPRINT", + "MAX_MODEL_LEN", + "WEKA_LOADER_OVERRIDE", + "PORT", + "GPU_COUNT", + "THINKING_MODE", + "ENABLE_AGENTX_POWER", + "REQUIRE_POWER", + } + return _strings( + { + key: value + for key, value in config.envs.items() + if key.upper() not in client_names + and not (key.upper().startswith("EXTRA_") and key.upper().endswith("_ARGS")) + and not key.upper().startswith( + ("AIPERF_", "AGENTIC_", "AGENTX_", "SCENARIO_") + ) + }, + "Explicit server environment", + ) + + +def native_server_spec( + config: BenchmarkConfig, root: Path, entry: dict[str, Any] +) -> dict[str, Any]: + """Resolve one YAML variant, preserving upstream validation and acceptance.""" + filename, _, selector = str(entry["srt-recipe"]).partition(":") + path = (root / filename).resolve() + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError( + f"Native AgentX recipe is missing or outside InferenceX: {filename}" + ) + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("Native AgentX recipe must be a mapping") + environment = { + "MODEL": entry["model"], + "MODEL_PREFIX": entry["model-prefix"], + "FRAMEWORK": entry["framework"], + "PRECISION": entry["precision"], + "IMAGE": config.docker_image or entry["image"], + "TP": str(entry["tp"]), + "EP_SIZE": str(entry["ep"]), + "PP_SIZE": str(entry.get("pp", 1)), + "DCP_SIZE": str(entry.get("dcp-size", 1)), + "PCP_SIZE": str(entry.get("pcp-size", 1)), + "GPU_COUNT": str(entry["tp"] * entry.get("pp", 1) * entry.get("pcp-size", 1)), + "DP_ATTENTION": str(entry.get("dp-attn", False)).lower(), + "CONC": str(entry["conc"]), + "KV_OFFLOADING": entry["kv-offloading"], + "TOTAL_CPU_DRAM_GB": str(entry["total-cpu-dram-gb"]), + "SPEC_DECODING": entry["spec-decoding"], + "EVAL_ONLY": "false", + "IS_AGENTIC": "1", + # This is the official single-node workflow default, not an inferred AL. + "THINKING_MODE": str(config.envs.get("THINKING_MODE", "thinking_on")), + } + completed = subprocess.run( + [sys.executable, "-c", _VALIDATE, str(root)], + input=json.dumps( + {"variants": _variants(raw, selector), "environment": environment} + ), + capture_output=True, + text=True, + timeout=30, + ) + if completed.returncode: + raise ValueError( + "InferenceX native recipe validation failed: " + completed.stderr.strip() + ) + selected = json.loads(completed.stdout) + recipe = selected["recipe"] + _apply_golden(recipe, selected["overrides"]) + allowed = { + "schema", + "name", + "model", + "resources", + "frontend", + "observability", + "engine", + "health_check", + "roles", + "benchmark", + "environment", + "setup_script", + } + if set(recipe) - allowed or recipe.get("schema") != 2: + raise ValueError("Unsupported native AgentX recipe deployment fields or schema") + engine = recipe["engine"] + if isinstance(engine, dict): + if set(engine) - {"type", "connector"} or engine.get("connector") is not None: + raise ValueError("Magpie does not support native engine connectors") + engine = engine["type"] + if engine not in {"sglang", "vllm"}: + raise ValueError("Magpie-owned AgentX supports SGLang and vLLM only") + frontend = recipe["frontend"] + if ( + frontend.get("type") != engine + or frontend.get("enable_multiple_frontends", False) + or frontend.get("args") + or frontend.get("env") + ): + raise ValueError( + "Magpie-owned AgentX does not support router or multiple frontends" + ) + observability = recipe.get("observability", {}) + if observability.get("enabled") or observability.get("tachometer", {}).get( + "enabled" + ): + raise ValueError( + "Magpie-owned AgentX does not support recipe observability services" + ) + role = recipe["roles"]["agg"] + if set(role) - {"nodes", "workers", "gpus", "args", "env"}: + raise ValueError("Unsupported native AgentX worker deployment fields") + arguments = dict(role["args"]) + for name in ("host", "port", "model", "model-path"): + if name in arguments: + raise ValueError(f"Native recipe must leave runtime-owned {name} to Magpie") + env = _strings(role.get("env", {}), "Native worker environment") + env.update(_strings(recipe.get("environment", {}), "Native global environment")) + env.update(_server_environment(config)) + port = _port(config) + model_path = str(config.envs.get("MODEL_PATH") or config.model) + if engine == "sglang": + argv = ["python3", "-m", "sglang.launch_server", "--model-path", model_path] + else: + argv = [ + "python3", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + model_path, + ] + argv += ["--host", "0.0.0.0", "--port", str(port), *_flags(arguments)] + setup = [] + files = [ + path, + root / "infx/srt_slurm/single_node.py", + root / "infx/srt_slurm/synthetic_acceptance.py", + root / "infx/golden_al_distribution/__init__.py", + root / "infx/golden_al_distribution/curves.py", + ] + files += sorted((root / "infx/golden_al_distribution").glob("*.yaml")) + if recipe.get("setup_script"): + if recipe["setup_script"] != "pip-runtime-deps.sh": + raise ValueError( + "Unsupported native AgentX setup_script: " + str(recipe["setup_script"]) + ) + script = ( + root / "benchmarks/multi_node/srt-slurm-recipes/configs/pip-runtime-deps.sh" + ) + files.append(script) + setup = [["bash", str(script)]] + client = _client_defaults(root, config) + client.update( + _strings(recipe["benchmark"].get("env", {}), "Native client environment") + ) + client.update(environment) + client.update( + PORT=str(port), + AIPERF_SERVER_URL=f"http://127.0.0.1:{port}", + AIPERF_SERVER_METRICS_URLS=f"http://127.0.0.1:{port}/metrics", + ) + context = arguments.get("context-length" if engine == "sglang" else "max-model-len") + if context is not None: + client["AIPERF_MAX_CONTEXT_LENGTH"] = str(context) + health = recipe.get("health_check", {}) + return { + **_client_revisions(root), + "version": 1, + "framework": engine, + "argv": argv, + "env": env, + "setup_commands": setup, + "port": port, + "health_path": "/health", + "health_timeout_seconds": int(health.get("interval_seconds", 10)) + * int(health.get("max_attempts", 180)), + "model_metadata": None, + "client_env": client, + "source_files": {**_file_hashes(files), **resolve_client_sources(root)}, + "recipe_variant": selected["variant"], + "golden_acceptance_overrides": selected["overrides"], + } + + +def custom_server_spec( + config: BenchmarkConfig, root: Path, entry: dict[str, Any] +) -> dict[str, Any]: + """Explicit custom models share the official replay, without a fake recipe.""" + port = _port(config) + model = str(config.envs.get("MODEL_PATH") or config.model) + dtype = {"bf16": "bfloat16", "fp16": "float16", "fp32": "float32"}.get( + config.precision, config.precision + ) + if dtype in {"fp8", "fp4", "mxfp4"}: + dtype = "auto" + if config.framework == "sglang": + argv = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--tensor-parallel-size", + str(entry["tp"]), + "--expert-parallel-size", + str(entry["ep"]), + "--context-length", + str(entry["max-model-len"]), + "--enable-metrics", + ] + else: + argv = [ + "python3", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + model, + "--tensor-parallel-size", + str(entry["tp"]), + "--max-model-len", + str(entry["max-model-len"]), + ] + if entry["ep"] > 1: + argv.append("--enable-expert-parallel") + argv += [ + "--served-model-name", + config.model, + "--host", + "0.0.0.0", + "--port", + str(port), + "--trust-remote-code", + "--dtype", + dtype, + ] + if entry.get("model-revision"): + argv += ["--revision", entry["model-revision"]] + return { + **_client_revisions(root), + "version": 1, + "framework": config.framework, + "argv": argv, + "env": _server_environment(config), + "setup_commands": [], + "port": port, + "health_path": "/health", + "health_timeout_seconds": 1800, + "model_metadata": { + "source": entry["model-config-source"], + "sha256": entry["model-config-sha256"], + "native_context_length": entry["native-context-length"], + "max_model_len": entry["max-model-len"], + "revision": entry.get("model-revision"), + }, + "client_env": { + **_client_defaults(root, config), + "AIPERF_MAX_CONTEXT_LENGTH": str(entry["max-model-len"]), + "WEKA_LOADER_OVERRIDE": entry["trace-loader"], + "PORT": str(port), + "AIPERF_SERVER_URL": f"http://127.0.0.1:{port}", + "AIPERF_SERVER_METRICS_URLS": f"http://127.0.0.1:{port}/metrics", + }, + "source_files": resolve_client_sources(root), + "recipe_variant": "custom", + "golden_acceptance_overrides": [], + } diff --git a/Magpie/modes/benchmark/agentx_runtime.py b/Magpie/modes/benchmark/agentx_runtime.py new file mode 100644 index 0000000..5bbcfae --- /dev/null +++ b/Magpie/modes/benchmark/agentx_runtime.py @@ -0,0 +1,650 @@ +############################################################################### +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# See LICENSE for license information. +############################################################################### +"""Own one AgentX server and its client, without invoking an upstream launcher.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import time +from typing import Any +import urllib.error +import urllib.request +from urllib.parse import urlparse +import uuid + +# Direct execution inside the serving image does not import Magpie's optional +# analysis/evaluation dependencies. Both files are mounted from the same build. +if __package__: + from .agentx_launch import prepare_server_launch + from .agentx_custom import native_context_length +else: + from agentx_launch import prepare_server_launch + from agentx_custom import native_context_length + + +class RuntimeInterrupted(RuntimeError): + """A signal interrupted this point; its owned children must be reaped.""" + + +class _PublicModelRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + parsed = urlparse(newurl) + if ( + parsed.scheme != "https" + or parsed.hostname != "huggingface.co" + or parsed.port not in {None, 443} + or parsed.username + or parsed.password + ): + raise ValueError( + "AgentX model metadata redirect left the public HuggingFace host" + ) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _signal_interrupt(signum, _frame): + raise RuntimeInterrupted(f"AgentX interrupted by signal {signum}") + + +def _stop_owned(process: subprocess.Popen) -> None: + """Stop only the process group created by this runtime, including descendants.""" + if process.pid <= 1 or process.pid == os.getpgrp(): + raise RuntimeError("Refusing to terminate an unowned process group") + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + process.wait() + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + # The root may exit before its workers, so leader exit alone is not cleanup. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + +def _spawn( + argv: list[str], *, env: dict[str, str], cwd: Path, output +) -> subprocess.Popen: + return subprocess.Popen( + argv, + env=env, + cwd=cwd, + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + +def _run_owned(argv, *, env, cwd, output, timeout, server=None) -> int: + process = _spawn(argv, env=env, cwd=cwd, output=output) + deadline = time.monotonic() + timeout + try: + while process.poll() is None: + if server is not None and server.poll() is not None: + raise RuntimeError( + f"AgentX server exited while the client was running ({server.returncode})" + ) + if time.monotonic() >= deadline: + raise TimeoutError(f"AgentX command exceeded its {timeout:g}s deadline") + time.sleep(0.1) + if server is not None and server.poll() is not None: + raise RuntimeError( + f"AgentX server exited before the client completed ({server.returncode})" + ) + return process.returncode + finally: + _stop_owned(process) + + +def _require_free_port(port: int) -> None: + # Do not reuse an unrelated healthy endpoint. A new server owns every point. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: + raise RuntimeError( + f"AgentX server port {port} is unavailable; existing servers are not reused" + ) from exc + + +def _wait_health(server, url: str, deadline: float) -> None: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + while time.monotonic() < deadline: + if server.poll() is not None: + raise RuntimeError( + f"AgentX server exited before becoming healthy ({server.returncode})" + ) + try: + with opener.open( + url, timeout=min(2.0, max(0.1, deadline - time.monotonic())) + ) as response: + healthy = response.status == 200 + except (OSError, urllib.error.URLError): + healthy = False + if healthy: + time.sleep(0.1) + if server.poll() is not None: + raise RuntimeError( + f"AgentX server exited during its health check ({server.returncode})" + ) + return + time.sleep(0.1) + raise TimeoutError( + "AgentX server did not become healthy before its startup deadline" + ) + + +def _verify_model_metadata(spec: dict[str, Any]) -> None: + metadata = spec.get("model_metadata") + if not metadata: + return + source = str(metadata.get("source") or "") + if source.startswith("https://"): + revision = str(metadata.get("revision") or "") + parsed = urlparse(source) + if ( + parsed.hostname != "huggingface.co" + or parsed.port not in {None, 443} + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or not re.fullmatch(r"[a-f0-9]{40}", revision) + or not parsed.path.endswith(f"/resolve/{revision}/config.json") + ): + raise ValueError( + "Remote AgentX metadata requires an immutable public HuggingFace revision" + ) + argv = spec["argv"] + if "--revision" not in argv or argv[argv.index("--revision") + 1 :][:1] != [ + revision + ]: + raise ValueError("AgentX server revision does not match its model metadata") + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), _PublicModelRedirect() + ) + try: + with opener.open(source, timeout=30) as response: + contents = response.read(4 * 1024 * 1024 + 1) + except urllib.error.HTTPError as exc: + if exc.code in {401, 403}: + raise ValueError( + "Gated AgentX models require an already downloaded local MODEL_PATH/config.json" + ) from exc + raise + else: + path = Path(source) + if not path.is_absolute() or not path.is_file(): + raise ValueError( + "AgentX model metadata requires its resolved local config.json" + ) + contents = path.read_bytes() + if len(contents) > 4 * 1024 * 1024: + raise ValueError("AgentX model configuration exceeds the 4 MiB metadata limit") + if hashlib.sha256(contents).hexdigest() != metadata.get("sha256"): + raise ValueError("AgentX model metadata changed after recipe resolution") + native = native_context_length(json.loads(contents)) + maximum = metadata.get("max_model_len") + if ( + native != metadata.get("native_context_length") + or type(maximum) is not int + or not 0 < maximum <= native + ): + raise ValueError( + "AgentX model context limits differ from the resolved metadata" + ) + + +def _verify_recipe_inputs(spec: dict[str, Any]) -> None: + for name, expected in spec.get("source_files", {}).items(): + if hashlib.sha256(Path(name).read_bytes()).hexdigest() != expected: + raise ValueError(f"AgentX recipe input changed before setup: {name}") + + +def _aiperf_git(root: Path, *arguments: str) -> str: + path = root / "utils/aiperf" + completed = subprocess.run( + ["git", "-c", f"safe.directory={path}", "-C", str(path), *arguments], + capture_output=True, + text=True, + timeout=10, + ) + if completed.returncode: + raise ValueError("AgentX requires the resolved AIPerf Git checkout") + return completed.stdout.strip() + + +def _verify_aiperf_revision(root: Path, spec: dict[str, Any]) -> None: + expected = spec.get("aiperf_revision") + if expected is None: + return + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{40}", expected): + raise ValueError("AgentX AIPerf revision must be a complete Git commit") + if ( + Path(_aiperf_git(root, "rev-parse", "--show-toplevel")).resolve() + != (root / "utils/aiperf").resolve() + ): + raise ValueError("AgentX AIPerf submodule is not initialized") + if _aiperf_git(root, "rev-parse", "HEAD") != expected: + raise ValueError("AgentX AIPerf revision changed after recipe resolution") + + +def _validate_spec(spec: Any) -> dict[str, Any]: + if ( + not isinstance(spec, dict) + or type(spec.get("version")) is not int + or spec["version"] != 1 + ): + raise ValueError( + "AgentX runtime requires a resolved server-launch-spec version 1" + ) + if spec.get("framework") not in {"sglang", "vllm"}: + raise ValueError("Magpie-owned AgentX supports SGLang and vLLM") + setup = spec.get("setup_commands", []) + if not isinstance(setup, list): + raise ValueError("AgentX setup_commands must be a list of argv token lists") + for argv in [spec.get("argv"), *setup]: + if ( + not isinstance(argv, list) + or not argv + or any(not isinstance(token, str) or "\0" in token for token in argv) + ): + raise ValueError( + "AgentX server/setup commands must be literal argv token lists" + ) + for key in ("env", "client_env"): + mapping = spec.get(key, {}) + if not isinstance(mapping, dict) or any( + not isinstance(name, str) + or not isinstance(value, str) + or "=" in name + or "\0" in name + value + for name, value in mapping.items() + ): + raise ValueError( + f"AgentX {key} must contain literal string environment values" + ) + port = spec.get("port") + if type(port) is not int or not 1 <= port <= 65535: + raise ValueError("AgentX server port must be between 1 and 65535") + health_path = spec.get("health_path", "/health") + if ( + not isinstance(health_path, str) + or not health_path.startswith("/") + or health_path.startswith("//") + ): + raise ValueError("AgentX health_path must be a local absolute URL path") + return spec + + +def _client_command(root: Path) -> list[str]: + current = root / "benchmarks" / "srt_agentic.sh" + if current.is_file(): + return ["bash", str(current)] + if (root / "benchmarks" / "benchmark_lib.sh").is_file(): + adapter = Path(__file__).parents[2] / "scripts" / "agentx" / "client_legacy.sh" + return ["bash", str(adapter)] + raise ValueError("InferenceX project has no supported AgentX client entrypoint") + + +def _client_environment(request, spec, root, workspace) -> dict[str, str]: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("AIPERF_", "AGENTIC_", "AGENTX_", "SRT_", "SRTCTL_")) + and key not in {"CONC_LIST", "BASH_ENV", "ENV", "SHELLOPTS", "BASHOPTS"} + } + server_only = set(spec.get("env", {})) - { + "ROCR_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + } + env.update( + { + key: value + for key, value in request["client_env"].items() + if key not in server_only + } + ) + env.update(spec.get("client_env", {})) + # A candidate framework overlay must not change the measurement client. + env["PYTHONPATH"] = str(root) + env.pop("PYTHONHOME", None) + env.update( + { + "INFMAX_CONTAINER_WORKSPACE": str(root), + "RESULT_DIR": str(workspace), + "AGENTIC_OUTPUT_DIR": str(workspace), + "RESULT_FILENAME": "inferencex_result", + "SERVER_LOG": str(workspace / "server.log"), + "AIPERF_RUNTIME_DIR": str(workspace / ".agentx-runtime"), + "AIPERF_SERVER_URL": f"http://127.0.0.1:{spec['port']}", + "AIPERF_SERVER_METRICS_URLS": f"http://127.0.0.1:{spec['port']}/metrics", + "PORT": str(spec["port"]), + "IS_MULTINODE": "false", + "IS_AGENTIC": "1", + "SCENARIO_TYPE": "agentic-coding", + "EVAL_ONLY": "false", + "RUN_EVAL": "false", + } + ) + # Client adapters must not alter or delete the server receipt. + for name in tuple(env): + if name.startswith(("AGENTX_LAUNCH_", "SRT_", "SRTCTL_")) or name in { + "AGENTX_SERVER_LAUNCH_FILE", + "CONC_LIST", + }: + env.pop(name) + return env + + +def _execute_local(request: dict[str, Any]) -> dict[str, Any]: + started = time.monotonic() + workspace = Path(request["workspace"]) + root = Path(request["inferencex_path"]) + workspace.mkdir(parents=True, exist_ok=True) + (workspace / "agentx_server_launch.json").unlink(missing_ok=True) + server = None + outcome = {"success": False, "errors": [], "execution_time": 0.0} + try: + spec = _validate_spec(request["server_spec"]) + _verify_recipe_inputs(spec) + _verify_aiperf_revision(root, spec) + command = _client_command(root) + _require_free_port(spec["port"]) + deadline = time.monotonic() + request["ready_timeout"] + server_env = {**os.environ, **request.get("gpu_env", {}), **spec.get("env", {})} + for key in ("BASH_ENV", "ENV", "SHELLOPTS", "BASHOPTS"): + server_env.pop(key, None) + with (workspace / "agentx_setup.log").open("w", encoding="utf-8") as setup_log: + for setup in spec.get("setup_commands", []): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "AgentX preparation exceeded its startup deadline" + ) + rc = _run_owned( + setup, env=server_env, cwd=root, output=setup_log, timeout=remaining + ) + if rc: + raise RuntimeError( + f"AgentX server preparation failed with exit code {rc}" + ) + _verify_aiperf_revision(root, spec) + _verify_model_metadata(spec) + argv, server_env, _evidence = prepare_server_launch( + spec["argv"], + server_env, + request.get("overrides"), + spec["framework"], + workspace, + server_spec=spec, + ) + with (workspace / "server.log").open("w", encoding="utf-8") as server_log: + server = _spawn(argv, env=server_env, cwd=root, output=server_log) + _wait_health( + server, + f"http://127.0.0.1:{spec['port']}{spec.get('health_path', '/health')}", + deadline, + ) + with (workspace / "agentx_client.log").open( + "w", encoding="utf-8" + ) as client_log: + rc = _run_owned( + command, + env=_client_environment(request, spec, root, workspace), + cwd=root, + output=client_log, + timeout=request["client_timeout"], + server=server, + ) + if rc: + raise RuntimeError( + f"AgentX benchmark client failed with exit code {rc}" + ) + outcome["success"] = True + except ( + OSError, + ValueError, + RuntimeError, + TimeoutError, + subprocess.SubprocessError, + ) as exc: + outcome["errors"].append(str(exc)) + finally: + if server is not None: + try: + _stop_owned(server) + except (OSError, subprocess.SubprocessError) as exc: + outcome["success"] = False + outcome["errors"].append(f"AgentX server cleanup failed: {exc}") + outcome["execution_time"] = time.monotonic() - started + return outcome + + +def _docker_command( + request, *, image: str, name: str, request_path: Path, runner_type: str +) -> list[str]: + command = [ + "docker", + "run", + "--init", + "--name", + name, + "--network=host", + "--ipc=host", + "--shm-size=16g", + ] + if runner_type.lower().startswith(("mi", "gfx", "radeon")): + command += [ + "--device=/dev/kfd", + "--device=/dev/dri", + "--group-add=video", + "--cap-add=SYS_PTRACE", + "--security-opt=seccomp=unconfined", + ] + else: + command += ["--gpus=all"] + package = Path(__file__).resolve().parents[2] + mounts = { + str(package): "ro", + request["inferencex_path"]: "rw", + request["workspace"]: "rw", + } + if request["server_spec"].get("aiperf_revision") is not None: + root = Path(request["inferencex_path"]) + _verify_aiperf_revision(root, request["server_spec"]) + # A submodule's .git file points outside the mounted project directory. + # Preserve that exact path without exposing the parent checkout. + git_directory = Path(_aiperf_git(root, "rev-parse", "--absolute-git-dir")) + mounts[str(git_directory.resolve())] = "ro" + model_path = request["client_env"].get( + "MODEL_PATH", request["client_env"].get("MODEL", "") + ) + if model_path and Path(model_path).is_dir(): + mounts[str(Path(model_path).resolve())] = "ro" + cache = request.get("hf_cache_path") + if cache and Path(cache).is_dir(): + mounts[str(Path(cache).resolve())] = "rw" + command += ["--env", f"HF_HOME={Path(cache).resolve()}"] + for source, mode in mounts.items(): + if ":" in source or "\n" in source: + raise ValueError( + "AgentX Docker mount paths cannot contain ':' or a newline" + ) + command += ["--volume", f"{source}:{source}:{mode}"] + for key, value in request.get("gpu_env", {}).items(): + command += ["--env", f"{key}={value}"] + if os.environ.get("HF_TOKEN"): + command += ["--env", "HF_TOKEN"] + command += [ + "--entrypoint", + "python3", + "--", + image, + str(Path(__file__).resolve()), + "--worker", + str(request_path), + ] + return command + + +def _execute_docker(request, *, image: str, runner_type: str) -> dict[str, Any]: + workspace = Path(request["workspace"]) + request_path = workspace / "agentx_runtime_request.json" + result_path = workspace / "agentx_runtime_result.json" + result_path.unlink(missing_ok=True) + request_path.write_text(json.dumps(request), encoding="utf-8") + name = f"magpie-agentx-{uuid.uuid4().hex}" + outcome = {"success": False, "errors": []} + started = time.monotonic() + try: + command = _docker_command( + request, + image=image, + name=name, + request_path=request_path, + runner_type=runner_type, + ) + with (workspace / "agentx_container.log").open("w", encoding="utf-8") as output: + rc = _run_owned( + command, + env=dict(os.environ), + cwd=workspace, + output=output, + timeout=request["ready_timeout"] + request["client_timeout"] + 60, + ) + if result_path.is_file(): + outcome = json.loads(result_path.read_text(encoding="utf-8")) + if rc: + outcome["success"] = False + outcome["errors"].append(f"AgentX container exited with code {rc}") + elif not result_path.is_file(): + outcome["errors"].append("AgentX container produced no runtime result") + except ( + OSError, + ValueError, + RuntimeError, + TimeoutError, + subprocess.SubprocessError, + ) as exc: + outcome["errors"].append(str(exc)) + finally: + try: + removed = subprocess.run( + ["docker", "rm", "--force", name], + capture_output=True, + text=True, + timeout=30, + ) + if removed.returncode and "No such container" not in removed.stderr: + raise RuntimeError(removed.stderr.strip()) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + outcome["success"] = False + outcome["errors"].append(f"AgentX container cleanup failed: {exc}") + outcome["execution_time"] = time.monotonic() - started + return outcome + + +def execute_agentx( + config, workspace: Path, runner_type: str, *, docker_image: str | None = None +): + """Run one fresh server/client point and return Magpie's normal execution tuple.""" + from .result import BenchmarkResult + + workspace = workspace.resolve() + workspace.mkdir(parents=True, exist_ok=True) + resolved = config.agentx.resolved if config.agentx else {} + spec = _validate_spec(resolved.get("server-launch-spec")) + if spec["framework"] != config.framework: + raise ValueError("AgentX server framework differs from the benchmark") + lifecycle = config.server_lifecycle + request = { + "workspace": str(workspace), + "inferencex_path": str(Path(config.inferencex_path).resolve()), + "server_spec": spec, + "overrides": config.agentx.launch_overrides, + "client_env": config.get_env_vars(), + "hf_cache_path": config.hf_cache_path + or os.environ.get("HF_HOME") + or str(Path.home() / ".cache/huggingface"), + "ready_timeout": float( + lifecycle.server_ready_timeout_s + if lifecycle + else spec.get("health_timeout_seconds", 2700) + ), + "client_timeout": float(config.timeout_seconds), + "gpu_env": { + key: str(value) + for key, value in config.envs.items() + if key + in {"ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"} + }, + } + if any( + not math.isfinite(request[key]) or request[key] <= 0 + for key in ("ready_timeout", "client_timeout") + ): + raise ValueError( + "AgentX startup and client timeouts must be positive finite durations" + ) + if config.run_mode == "docker": + image = docker_image or config.docker_image + if not image: + raise ValueError("AgentX Docker execution requires a pinned image") + outcome = _execute_docker(request, image=image, runner_type=runner_type) + elif config.run_mode == "local": + outcome = _execute_local(request) + else: + raise ValueError("Magpie-owned AgentX supports local and Docker execution") + result = BenchmarkResult( + success=bool(outcome["success"]), + framework=config.framework, + model=config.model, + scenario="agentx", + workspace_dir=str(workspace), + execution_time=outcome["execution_time"], + errors=outcome["errors"], + ) + log = workspace / "agentx_client.log" + stdout = ( + log.read_text(encoding="utf-8", errors="replace")[-131072:] + if log.is_file() + else "" + ) + return result, stdout, "\n".join(outcome["errors"]) + + +def _worker_main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path, required=True) + args = parser.parse_args() + request = json.loads(args.worker.read_text(encoding="utf-8")) + signal.signal(signal.SIGTERM, _signal_interrupt) + signal.signal(signal.SIGINT, _signal_interrupt) + outcome = _execute_local(request) + (Path(request["workspace"]) / "agentx_runtime_result.json").write_text( + json.dumps(outcome), encoding="utf-8" + ) + return 0 if outcome["success"] else 1 + + +if __name__ == "__main__": + raise SystemExit(_worker_main()) diff --git a/Magpie/modes/benchmark/benchmarker.py b/Magpie/modes/benchmark/benchmarker.py index 86c3525..308130b 100644 --- a/Magpie/modes/benchmark/benchmarker.py +++ b/Magpie/modes/benchmark/benchmarker.py @@ -12,6 +12,7 @@ import json import logging import os +import shlex import shutil import signal import subprocess @@ -33,7 +34,13 @@ ensure_agentx_dependencies, resolve_agentx_recipe, ) -from .agentx_launch import launch_environment, read_launch_evidence +from .agentx_launch import ( + apply_launch_args, + launch_environment, + read_launch_evidence, + validate_overrides, +) +from .agentx_runtime import execute_agentx from .config import BenchmarkConfig from .image_selector import ImageSelector from .inferencex import ensure_inferencex_available @@ -117,6 +124,49 @@ def __init__( ) self._task_id: Optional[str] = None + @property + def _managed_agentx(self) -> Optional[Dict[str, Any]]: + """The persisted recipe decides who owns the server, not script names.""" + if not self.config.is_agentx or self.config.agentx is None: + return None + return (self.config.agentx.resolved or {}).get("server-launch-spec") + + def _normalize_agentx_extra_args(self) -> None: + """Consume legacy extra-args once into the literal candidate contract.""" + if not self._managed_agentx: + return + key = f"EXTRA_{self.config.framework.upper()}_ARGS" + present = [ + name + for name in self.config.envs + if name.upper().startswith("EXTRA_") and name.upper().endswith("_ARGS") + ] + if any(name.upper() != key for name in present): + raise ValueError( + f"Magpie-owned AgentX only accepts {key} for this framework" + ) + if not present: + return + if len(present) != 1: + raise ValueError(f"Duplicate AgentX {key} environment settings") + name = present[0] + raw = self.config.envs[name] + if not isinstance(raw, str): + raise ValueError(f"{key} must be a shell-quoted argument string") + overrides = validate_overrides( + self.config.agentx.launch_overrides or {"version": 1} + ) + overrides["append_args"] = [*overrides["append_args"], *shlex.split(raw)] + # Reject protocol changes before dependency setup; the runtime repeats + # these checks immediately before starting the server. + overrides = validate_overrides(overrides) + apply_launch_args( + self._managed_agentx["argv"], overrides, self.config.framework + ) + self.config.agentx.launch_overrides = overrides + self.config.envs.pop(name) + self._managed_agentx.get("client_env", {}).pop(name, None) + def run(self, task_id: Optional[str] = None) -> BenchmarkResult: """ Run benchmark. @@ -155,8 +205,8 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: ) return result - # AgentX execution stays inside InferenceX. Magpie selects the existing - # launcher/recipe and forwards its resolved environment. + # Resolve one workload. New recipes separate Magpie-owned serving from + # the upstream client; explicit legacy checkouts retain their launcher. runner_type: Optional[str] = None if self.config.is_agentx: try: @@ -166,7 +216,13 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: self.config.inferencex_path, runner_type=runner_type, ) + self._normalize_agentx_extra_args() ensure_agentx_dependencies(self.config.inferencex_path) + if self.config.is_server_lifecycle and not self._managed_agentx: + raise ValueError( + "Legacy AgentX launchers cannot use server_lifecycle; " + "use the pinned InferenceX client layout for Magpie-managed serving" + ) self.workspace_mgr.framework = self.config.framework except (OSError, RuntimeError, ValueError) as e: result = BenchmarkResult( @@ -183,6 +239,7 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: # the already-running server's physical devices. lifecycle_server_healthy = ( self.config.is_server_lifecycle + and not self.config.is_agentx and self._reuse_http_healthy(self._reuse_benchmark_port()) ) skip_idle_gpu_for_reuse = bool(lifecycle_server_healthy) @@ -211,7 +268,8 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: return result # 1. Copy Magpie generic scripts to InferenceX/benchmarks/ - self._prepare_benchmark_scripts() + if not self._managed_agentx: + self._prepare_benchmark_scripts() # 2. Determine runner type from GPU runner_type = runner_type or self._get_runner_type() @@ -345,7 +403,14 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: gpu_monitor = None # 5-7. Build and execute (Docker or local) - if self.config.is_local: + if self._managed_agentx: + result, stdout, stderr = execute_agentx( + self.config, + workspace, + runner_type, + docker_image=(None if self.config.is_local else self._select_image()), + ) + elif self.config.is_local: symlink = self._create_workspace_symlink(workspace) try: if self.config.is_server_lifecycle: @@ -527,12 +592,19 @@ def run(self, task_id: Optional[str] = None) -> BenchmarkResult: "Benchmark produced no valid throughput/latency metrics" ) - if self.config.is_agentx and self.config.agentx.launch_overrides is not None: + if self.config.is_agentx and ( + self._managed_agentx or self.config.agentx.launch_overrides is not None + ): try: evidence = read_launch_evidence(self.config, workspace) if result.agentx_metrics is None: result.agentx_metrics = {} result.agentx_metrics["server_launch"] = evidence + if self._managed_agentx: + result.agentx_metrics["execution_owner"] = "magpie" + metadata = self._managed_agentx.get("model_metadata") + if metadata: + result.agentx_metrics["model_context"] = metadata except (OSError, ValueError) as exc: result.success = False result.benchmark_valid = False @@ -895,7 +967,6 @@ def _build_docker_command( env_vars["SGLANG_TORCH_PROFILER_DIR"] = "/workspace/torch_trace" env_vars["ATOM_TORCH_PROFILER_DIR"] = "/workspace/torch_trace" - # HuggingFace token from environment hf_token = os.environ.get("HF_TOKEN", "") if hf_token: @@ -917,7 +988,7 @@ def _build_docker_command( # With --entrypoint /bin/bash, pass -c as first arg cmd.extend([ "-c", - f"cd /opt/InferenceX && bash {benchmark_script}" + f"cd /opt/InferenceX && bash {shlex.quote(benchmark_script)}" ]) return cmd @@ -975,7 +1046,7 @@ def _build_local_command( cmd = [ "bash", "-c", - f"cd {inferencex_path} && bash {benchmark_script}", + f"cd {shlex.quote(inferencex_path)} && bash {shlex.quote(benchmark_script)}", ] env = os.environ.copy() diff --git a/Magpie/modes/benchmark/config.py b/Magpie/modes/benchmark/config.py index 799ea9f..d0d912f 100644 --- a/Magpie/modes/benchmark/config.py +++ b/Magpie/modes/benchmark/config.py @@ -925,8 +925,13 @@ def __post_init__(self): "Magpie AgentX v1 supports run_mode='local' or 'docker'; " "Ray and multi-node execution are not supported yet" ) - if self.is_server_lifecycle: - raise ValueError("AgentX cannot be combined with server_lifecycle") + if self.is_server_lifecycle and ( + not self.server_lifecycle.cleanup or self.server_lifecycle.force_reuse + ): + raise ValueError( + "AgentX cannot be combined with server_lifecycle reuse: " + "set cleanup=true and force_reuse=false for a fresh server per point" + ) incompatible = [] if self.profiler.torch_profiler.enabled: incompatible.append("torch_profiler") diff --git a/Magpie/modes/benchmark/inferencex.py b/Magpie/modes/benchmark/inferencex.py index 905b478..7608f25 100644 --- a/Magpie/modes/benchmark/inferencex.py +++ b/Magpie/modes/benchmark/inferencex.py @@ -14,12 +14,12 @@ import os import subprocess from pathlib import Path -from typing import Optional logger = logging.getLogger(__name__) # InferenceX repository configuration INFERENCEX_REPO_URL = "https://github.com/SemiAnalysisAI/InferenceX.git" +INFERENCEX_DEFAULT_REF = "408c015be4b22d14c69518643609669405507077" # Default directory resolution order (first writable candidate wins): _PROJECT_ROOT = Path(__file__).parent.parent.parent.parent INFERENCEX_DEFAULT_DIR = str(_PROJECT_ROOT / "InferenceX") @@ -31,6 +31,41 @@ } +def resolve_inferencex_root(path: str) -> str: + """Return a supported InferenceX project root without altering the checkout. + + A benchmarks directory alone is not an installation: Magpie may have copied + its own scripts there. Require the upstream library and client source too. + """ + root = Path(path).expanduser().resolve() + candidates = [] + for candidate in (root, root / "inferencex-e2e"): + if not (candidate / "benchmarks" / "benchmark_lib.sh").is_file(): + continue + legacy = candidate / "utils" / "bench_serving" / "benchmark_serving.py" + packaged = candidate / "infx" / "bench_serving" / "benchmark_serving.py" + if legacy.is_file() or ( + packaged.is_file() and (candidate / "pyproject.toml").is_file() + ): + candidates.append(candidate) + if len(candidates) != 1: + raise RuntimeError( + f"Expected one supported InferenceX project at {root}; provide the " + "repository root or its inferencex-e2e directory containing " + "benchmarks/benchmark_lib.sh and the upstream benchmark client" + ) + return str(candidates[0]) + + +def resolve_benchmark_serving(path: str) -> Path: + """Locate the supported client source for profiling and runtime adapters.""" + root = Path(resolve_inferencex_root(path)) + packaged = root / "infx" / "bench_serving" / "benchmark_serving.py" + if packaged.is_file() and (root / "pyproject.toml").is_file(): + return packaged + return root / "utils" / "bench_serving" / "benchmark_serving.py" + + def _resolve_default_inferencex_dir() -> str: """Pick the first writable default location for InferenceX. @@ -46,8 +81,11 @@ def _resolve_default_inferencex_dir() -> str: candidates = [ INFERENCEX_DEFAULT_DIR, - str(Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache"))) - / "magpie" / "InferenceX"), + str( + Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache"))) + / "magpie" + / "InferenceX" + ), "/root/workspace/InferenceX", ] for candidate in candidates: @@ -68,21 +106,21 @@ def _resolve_default_inferencex_dir() -> str: class InferenceXManager: """ Manages InferenceX repository for benchmark execution. - + Handles: - Validation of existing installation - Automatic cloning if not present - Path resolution for placeholder values """ - + def __init__( self, repo_url: str = INFERENCEX_REPO_URL, - default_dir: Optional[str] = None, + default_dir: str | None = None, ): """ Initialize InferenceX manager. - + Args: repo_url: Git repository URL for InferenceX default_dir: Default directory to clone into. If None, resolves @@ -90,114 +128,127 @@ def __init__( """ self.repo_url = repo_url self.default_dir = default_dir or _resolve_default_inferencex_dir() - - def ensure_available(self, configured_path: Optional[str] = None) -> str: + + def ensure_available(self, configured_path: str | None = None) -> str: """ Ensure InferenceX is available at the specified path. - + If the path doesn't exist or is a placeholder value, automatically clone InferenceX from the repository. - + Args: configured_path: Configured path to InferenceX (may be None or placeholder) - + Returns: Valid path to InferenceX installation - + Raises: RuntimeError: If unable to clone or validate InferenceX """ # Determine if path is a placeholder or not configured is_placeholder = self._is_placeholder(configured_path) - - # Check if configured path exists - if not is_placeholder and configured_path and os.path.exists(configured_path): - logger.debug(f"InferenceX found at: {configured_path}") - return configured_path - + # Determine target directory - target_dir = self.default_dir if is_placeholder or not configured_path else configured_path - + target_dir = ( + self.default_dir + if is_placeholder or not configured_path + else configured_path + ) + target_dir = str(Path(target_dir).expanduser().resolve()) + # Check if already exists at target directory if os.path.exists(target_dir): - if self._validate_installation(target_dir): - logger.info(f"InferenceX already exists at: {target_dir}") - return target_dir - else: - logger.warning(f"Directory exists but doesn't appear to be InferenceX: {target_dir}") - # Still return it, let downstream code handle validation - return target_dir - + project_root = resolve_inferencex_root(target_dir) + logger.info(f"InferenceX already exists at: {project_root}") + return project_root + # Clone the repository return self._clone_repository(target_dir) - - def _is_placeholder(self, path: Optional[str]) -> bool: + + def _is_placeholder(self, path: str | None) -> bool: """Check if the path is a placeholder value.""" if path is None: return True return path in PLACEHOLDER_VALUES - + def _validate_installation(self, path: str) -> bool: """ Validate that the path contains a valid InferenceX installation. - + Args: path: Path to validate - + Returns: True if valid InferenceX installation """ - required_paths = [ - "benchmarks", # InferenceX benchmark scripts directory - ] - - for required in required_paths: - if not os.path.exists(os.path.join(path, required)): - return False - + try: + resolve_inferencex_root(path) + except (OSError, RuntimeError): + return False return True - + def _clone_repository(self, target_dir: str) -> str: """ Clone InferenceX repository. - + Args: target_dir: Directory to clone into - + Returns: Path to cloned repository - + Raises: RuntimeError: If clone fails """ logger.info(f"InferenceX not found. Cloning from {self.repo_url}...") logger.info(f"Clone destination: {target_dir}") - + try: # Ensure parent directory exists parent_dir = os.path.dirname(target_dir) if parent_dir and not os.path.exists(parent_dir): os.makedirs(parent_dir, exist_ok=True) - - # Clone the repository - result = subprocess.run( - ["git", "clone", self.repo_url, target_dir], - capture_output=True, - text=True, - timeout=300, # 5 minute timeout - ) - - if result.returncode != 0: - error_msg = result.stderr.strip() if result.stderr else "Unknown error" - logger.error(f"Failed to clone InferenceX: {error_msg}") - raise RuntimeError(f"git clone failed: {error_msg}") - + + commands = [ + ["git", "clone", "--no-checkout", self.repo_url, target_dir], + [ + "git", + "-C", + target_dir, + "checkout", + "--detach", + INFERENCEX_DEFAULT_REF, + ], + [ + "git", + "-C", + target_dir, + "submodule", + "update", + "--init", + "--recursive", + "--", + "inferencex-e2e/utils/aiperf", + ], + ] + for command in commands: + result = subprocess.run( + command, capture_output=True, text=True, timeout=300, check=False + ) + if result.returncode != 0: + error_msg = ( + result.stderr.strip() if result.stderr else "Unknown error" + ) + raise RuntimeError( + f"InferenceX {' '.join(command)} failed: {error_msg}" + ) + logger.info(f"Successfully cloned InferenceX to: {target_dir}") - return target_dir - + return resolve_inferencex_root(target_dir) + except subprocess.TimeoutExpired: - logger.error("InferenceX clone timed out after 5 minutes") - raise RuntimeError("git clone timed out after 5 minutes") + logger.error("InferenceX installation command timed out after 5 minutes") + raise RuntimeError("InferenceX git command timed out after 5 minutes") except FileNotFoundError: logger.error("git command not found. Please install git.") raise RuntimeError("git is not installed. Please install git first.") @@ -207,7 +258,7 @@ def _clone_repository(self, target_dir: str) -> str: # Module-level instance for convenience -_manager: Optional[InferenceXManager] = None +_manager: InferenceXManager | None = None def get_manager() -> InferenceXManager: @@ -218,15 +269,14 @@ def get_manager() -> InferenceXManager: return _manager -def ensure_inferencex_available(configured_path: Optional[str] = None) -> str: +def ensure_inferencex_available(configured_path: str | None = None) -> str: """ Convenience function to ensure InferenceX is available. - + Args: configured_path: Configured path to InferenceX - + Returns: Valid path to InferenceX installation """ return get_manager().ensure_available(configured_path) - diff --git a/Magpie/modes/benchmark/tracelens_inference.py b/Magpie/modes/benchmark/tracelens_inference.py index d2fbe2c..80ae0fa 100644 --- a/Magpie/modes/benchmark/tracelens_inference.py +++ b/Magpie/modes/benchmark/tracelens_inference.py @@ -31,6 +31,7 @@ from ...utils.gpu import detect_gpu from .config import BenchmarkConfig +from .inferencex import resolve_benchmark_serving, resolve_inferencex_root from .tracelens import ensure_tracelens_installed from .tracelens_runtime import ATOM_DETAILED_ANNOTATION_SCRIPT, docker_image_probe @@ -386,16 +387,20 @@ def prepare( if "NUM_PROMPTS" not in envs: envs["NUM_PROMPTS"] = str(int(float(envs.get("CONC", 32))) * 10) - # ATOM ships its own runner and does not source InferenceX benchmark_lib.sh. - if self.config.framework != "atom": - self._patch_benchmark_lib(result) - - if self.config.framework == "vllm": - self._prepare_vllm_env(envs, max_iters, delay_iters) - elif self.config.framework == "sglang": - self._prepare_sglang_env(envs, max_iters, delay_iters, result) - elif self.config.framework == "atom": - self._prepare_atom_env(envs, result, runtime) + try: + # ATOM ships its own runner and does not source InferenceX benchmark_lib.sh. + if self.config.framework != "atom": + self._patch_benchmark_lib(result) + + if self.config.framework == "vllm": + self._prepare_vllm_env(envs, max_iters, delay_iters) + elif self.config.framework == "sglang": + self._prepare_sglang_env(envs, max_iters, delay_iters, result) + elif self.config.framework == "atom": + self._prepare_atom_env(envs, result, runtime) + except Exception: + self.restore() + raise result["env_updates"] = { key: envs.get(key) @@ -773,23 +778,21 @@ def _should_enable_sglang_shape_discovery(self) -> bool: ) or (self.config.is_local and host_sglang_supports_shape_discovery()) def _patch_benchmark_lib(self, result: Dict[str, Any]) -> None: - path = Path(self.config.inferencex_path) / "benchmarks" / "benchmark_lib.sh" - if not path.exists(): - warning = f"benchmark_lib.sh not found, skipping TraceLens preprocess patch: {path}" - logger.warning(warning) - result["warnings"].append(warning) - return + path = ( + Path(resolve_inferencex_root(self.config.inferencex_path)) + / "benchmarks" + / "benchmark_lib.sh" + ) text = path.read_text(encoding="utf-8") old = 'num_prompts="$max_concurrency"' new = 'num_prompts="$num_prompts"' if new in text: return - if old not in text: - warning = f"benchmark_lib.sh profile num_prompts anchor not found: {path}" - logger.warning(warning) - result["warnings"].append(warning) - return + if text.count(old) != 1: + raise RuntimeError( + f"Unsupported TraceLens benchmark_lib.sh profile anchor: {path}" + ) self._backup_file(path) path.write_text(text.replace(old, new, 1), encoding="utf-8") @@ -802,49 +805,27 @@ def _patch_sglang_benchmark_serving( delay_iters: int, result: Dict[str, Any], ) -> None: - path = ( - Path(self.config.inferencex_path) - / "utils" - / "bench_serving" - / "benchmark_serving.py" - ) - if not path.exists(): - warning = f"benchmark_serving.py not found, skipping SGLang TraceLens patch: {path}" - logger.warning(warning) - result["warnings"].append(warning) - return - + path = resolve_benchmark_serving(self.config.inferencex_path) text = path.read_text(encoding="utf-8") - common_anchor = '"num_steps": 1, "merge_profiles": True, "profile_by_stage": True' - common_repl = ( - '"shape_discovery": True, "detailed_annotations": True, ' - '"num_steps": 1, "merge_profiles": True, "profile_by_stage": True' + # Legacy clients keep this dict on one line; packaged infx clients use + # multiple lines. Match the known request fields, not source formatting. + common_anchor = ( + r'"num_steps"\s*:\s*1,\s*"merge_profiles"\s*:\s*True,' + r'\s*"profile_by_stage"\s*:\s*True' ) steady_repl = ( + '"shape_discovery": True, "detailed_annotations": True, ' f'"start_step": {delay_iters}, "num_steps": {max_iters}, ' '"merge_profiles": False, "profile_by_stage": False' ) - - if steady_repl in text and "detailed_annotations" in text: - return - - if "detailed_annotations" not in text: - if common_anchor not in text: - warning = f"SGLang benchmark_serving.py common patch anchor not found: {path}" - logger.warning(warning) - result["warnings"].append(warning) - return - self._backup_file(path) - text = text.replace(common_anchor, common_repl, 1) - - if common_anchor not in text: - warning = f"SGLang benchmark_serving.py steady-state patch anchor not found: {path}" - logger.warning(warning) - result["warnings"].append(warning) + if steady_repl in text: return - + if len(re.findall(common_anchor, text)) != 1: + raise RuntimeError( + f"Unsupported SGLang TraceLens profile request anchor: {path}" + ) self._backup_file(path) - text = text.replace(common_anchor, steady_repl, 1) + text = re.sub(common_anchor, lambda _match: steady_repl, text, count=1) path.write_text(text, encoding="utf-8") result["patched_files"].append(str(path)) logger.info("Patched SGLang benchmark_serving.py for TraceLens inference profiling") diff --git a/Magpie/scripts/agentx/client_legacy.sh b/Magpie/scripts/agentx/client_legacy.sh new file mode 100644 index 0000000..359555f --- /dev/null +++ b/Magpie/scripts/agentx/client_legacy.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# See LICENSE for license information. +# Explicit adapter for the pre-srt client library. Never launches a server. +set -eo pipefail +: "${INFMAX_CONTAINER_WORKSPACE:?}" "${AIPERF_SERVER_URL:?}" "${RESULT_DIR:?}" +source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" +for function in resolve_trace_source install_agentic_deps build_replay_cmd run_agentic_replay_and_write_outputs; do + declare -F "$function" >/dev/null || { echo "Unsupported legacy AgentX client library: $function missing" >&2; exit 2; } +done +resolve_trace_source +install_agentic_deps +mkdir -p "$RESULT_DIR" +build_replay_cmd "$RESULT_DIR" +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/Magpie/scripts/benchmark/atom_mi300x.sh b/Magpie/scripts/benchmark/atom_mi300x.sh index 9e22259..ccf5b69 100644 --- a/Magpie/scripts/benchmark/atom_mi300x.sh +++ b/Magpie/scripts/benchmark/atom_mi300x.sh @@ -126,7 +126,7 @@ if [[ "$PHASE" == "client" || "$PHASE" == "all" ]]; then SERVER_MONITOR_ARGS=() magpie_run_benchmark_serving_remote_direct trust || exit $? else - run_benchmark_serving \ + magpie_run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ --backend vllm \ diff --git a/Magpie/scripts/benchmark/atom_mi355x.sh b/Magpie/scripts/benchmark/atom_mi355x.sh index c88143c..5cb915a 100644 --- a/Magpie/scripts/benchmark/atom_mi355x.sh +++ b/Magpie/scripts/benchmark/atom_mi355x.sh @@ -122,7 +122,7 @@ if [[ "$PHASE" == "client" || "$PHASE" == "all" ]]; then SERVER_MONITOR_ARGS=() magpie_run_benchmark_serving_remote_direct trust || exit $? else - run_benchmark_serving \ + magpie_run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ --backend vllm \ diff --git a/Magpie/scripts/benchmark/magpie_bench_remote_compat.sh b/Magpie/scripts/benchmark/magpie_bench_remote_compat.sh index 9fc5816..e422568 100644 --- a/Magpie/scripts/benchmark/magpie_bench_remote_compat.sh +++ b/Magpie/scripts/benchmark/magpie_bench_remote_compat.sh @@ -1,59 +1,155 @@ #!/usr/bin/env bash ############################################################################### -# Remote benchmark compat (Magpie). -# -# InferenceX benchmarks/benchmark_lib.sh defines run_benchmark_serving() which -# parses a fixed set of flags. Older (and some current) trees reject -# --base-url at the bash layer even though utils/bench_serving/benchmark_serving.py -# accepts --base-url for OpenAI-compatible servers (SGLang/vLLM HTTP). -# -# When BENCHMARK_BASE_URL is set, Magpie *mi*.sh scripts call -# magpie_run_benchmark_serving_remote_direct() instead of passing --base-url into -# run_benchmark_serving(). -# -# Working directory must be the InferenceX repository root (Magpie benchmarker -# runs: cd && bash benchmarks/