diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 68cd412217..b7bb6ed584 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2,6 +2,109 @@ # Shared benchmarking utilities for InferenceX +_INFERENCEX_BENCHMARK_LIB_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +# Apply an explicit, versioned server-only launch extension. The Python helper +# validates edits/source identity and emits NUL-separated argv without eval. +# An unset request leaves canonical launchers completely unchanged. +agentic_apply_server_launch() { + local framework="$1" + shift + AGENTX_SERVER_COMMAND=("$@") + [[ -n "${AGENTX_LAUNCH_OVERRIDES_FILE:-}" ]] || return 0 + local output + : "${AGENTX_SERVER_LAUNCH_FILE:?AgentX launch evidence path is required}" + : "${AGENTX_LAUNCH_OVERRIDES_SHA256:?AgentX launch override hash is required}" + output="$(mktemp "${AGENTX_SERVER_LAUNCH_FILE}.argv.XXXXXX")" || return 1 + local rc=0 + python3 "${_INFERENCEX_BENCHMARK_LIB_DIR}/../utils/agentic/server_launch.py" \ + --framework "$framework" \ + --overrides "$AGENTX_LAUNCH_OVERRIDES_FILE" \ + --expected-sha256 "$AGENTX_LAUNCH_OVERRIDES_SHA256" \ + --evidence "$AGENTX_SERVER_LAUNCH_FILE" \ + --output "$output" -- "${AGENTX_SERVER_COMMAND[@]}" || rc=$? + if [[ "$rc" -eq 0 ]]; then + local token + AGENTX_SERVER_COMMAND=() + while IFS= read -r -d '' token; do + AGENTX_SERVER_COMMAND+=("$token") + done < "$output" + fi + rm -f -- "$output" + return "$rc" +} + +# Generic custom-model AgentX uses the same canonical replay/aggregation path +# as registered recipes. It never calls the fixed-length benchmark client. +run_generic_agentic_server() { + local framework="$1" + check_env_vars MODEL TP EP_SIZE CONC RESULT_DIR DURATION PRECISION \ + AGENTX_NATIVE_CONTEXT_LENGTH AGENTX_MAX_MODEL_LEN AGENTX_MODEL_CONFIG_SHA256 + require_agentic_kv_offload_none + if [[ "${AGENTX_CUSTOM_RECIPE:-0}" != "1" || "${FRAMEWORK:-}" != "$framework" ]]; then + echo "ERROR: generic AgentX requires an explicitly resolved custom recipe" >&2 + return 1 + fi + mkdir -p "$RESULT_DIR" + install_agentic_deps + local model_path="${MODEL_PATH:-$MODEL}" + local model_config + if [[ -d "$model_path" ]]; then + model_config="$model_path/config.json" + elif [[ -n "${MODEL_PATH:-}" ]]; then + echo "ERROR: custom AgentX MODEL_PATH is not an existing directory" >&2 + return 1 + else + model_config="$("$AIPERF_HF_CLI" download "$MODEL" config.json)" || return $? + fi + python3 "${_INFERENCEX_BENCHMARK_LIB_DIR}/../utils/agentic/custom_model.py" \ + --config "$model_config" --expected-sha256 "$AGENTX_MODEL_CONFIG_SHA256" \ + --native-context "$AGENTX_NATIVE_CONTEXT_LENGTH" \ + --max-context "$AGENTX_MAX_MODEL_LEN" || return $? + # benchmark_lib deliberately clears inherited MAX_MODEL_LEN. Restore only + # the independently verified, fingerprinted custom-model context here. + export MAX_MODEL_LEN="$AGENTX_MAX_MODEL_LEN" + resolve_trace_source + local dtype + case "$PRECISION" in + bf16|bfloat16) dtype=bfloat16 ;; + fp16|float16) dtype=float16 ;; + fp32|float32) dtype=float32 ;; + auto|fp8|fp4|mxfp4) dtype=auto ;; + *) echo "ERROR: unsupported custom AgentX precision $PRECISION" >&2; return 1 ;; + esac + local -a server_command + if [[ "$framework" == "sglang" ]]; then + server_command=(python3 -m sglang.launch_server + --model-path "$model_path" --served-model-name "$MODEL" + --host 0.0.0.0 --port "$PORT" --trust-remote-code + --tp "$TP" --ep-size "$EP_SIZE" --dtype "$dtype" + --context-length "$MAX_MODEL_LEN" --enable-metrics) + elif [[ "$framework" == "vllm" ]]; then + server_command=(vllm serve "$model_path" --served-model-name "$MODEL" + --host 0.0.0.0 --port "$PORT" --trust-remote-code + --tensor-parallel-size "$TP" --dtype "$dtype" + --max-model-len "$MAX_MODEL_LEN" --disable-log-requests) + if [[ "$EP_SIZE" != "1" ]]; then + server_command+=(--enable-expert-parallel) + fi + else + echo "ERROR: unsupported generic AgentX framework $framework" >&2 + return 1 + fi + agentic_apply_server_launch "$framework" "${server_command[@]}" || return $? + server_command=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/server_command.txt" "${server_command[@]}" + SERVER_LOG="$RESULT_DIR/server.log" + "${server_command[@]}" > "$SERVER_LOG" 2>&1 & + SERVER_PID=$! + trap 'kill "$SERVER_PID" 2>/dev/null || true; wait "$SERVER_PID" 2>/dev/null || true' EXIT + wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +} + # Keep Python bytecode out of the mounted workspace. Benchmark jobs often run as # root inside containers, and root-owned cache directories break future checkout # cleanup on self-hosted runners. diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 61add3eafa..9cca77167b 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -228,6 +228,9 @@ SGLANG_CMD=( --enable-metrics ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 1b9311eda0..fde5160425 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -422,6 +422,9 @@ VLLM_CMD=( # (srok), not yet #--attention_config.use_fp4_indexer_cache=True +agentic_apply_server_launch vllm "${VLLM_CMD[@]}" +VLLM_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & diff --git a/benchmarks/single_node/agentic/generic_sglang.sh b/benchmarks/single_node/agentic/generic_sglang.sh new file mode 100644 index 0000000000..03dceff112 --- /dev/null +++ b/benchmarks/single_node/agentic/generic_sglang.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" +run_generic_agentic_server sglang diff --git a/benchmarks/single_node/agentic/generic_vllm.sh b/benchmarks/single_node/agentic/generic_vllm.sh new file mode 100644 index 0000000000..423e52b206 --- /dev/null +++ b/benchmarks/single_node/agentic/generic_vllm.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" +run_generic_agentic_server vllm diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 98b4531e4b..8bab2ac49a 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -238,6 +238,9 @@ SGLANG_CMD=( --enable-metrics ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh index bdc2e348bf..42f9e05886 100755 --- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh @@ -103,6 +103,9 @@ SGLANG_CMD=( --enable-cache-report ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index ec2eaf3c36..9c2b81605a 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -206,6 +206,9 @@ VLLM_CMD=( "${SPEC_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) +agentic_apply_server_launch vllm "${VLLM_CMD[@]}" +VLLM_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh index b735220fc7..ee1e0cd393 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh @@ -224,6 +224,9 @@ VLLM_CMD=( --speculative-config "$SPEC_CONFIG" "${OFFLOAD_ARGS[@]}" ) +agentic_apply_server_launch vllm "${VLLM_CMD[@]}" +VLLM_CMD=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh index 88d9078819..59868b23d5 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh @@ -178,6 +178,9 @@ VLLM_CMD=( --speculative-config "$SPEC_CONFIG" "${OFFLOAD_ARGS[@]}" ) +agentic_apply_server_launch vllm "${VLLM_CMD[@]}" +VLLM_CMD=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh index 1851139dab..2714f1cdfb 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh @@ -89,6 +89,9 @@ VLLM_CMD=( --trust-remote-code --stream-interval 20 ) +agentic_apply_server_launch vllm "${VLLM_CMD[@]}" +VLLM_CMD=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index 804d65b08e..934aec3504 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -112,6 +112,9 @@ SGLANG_CMD=( --enable-cache-report ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh index aed9047466..8a0864281d 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh @@ -91,6 +91,9 @@ SGLANG_CMD=( --enable-cache-report ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh index 123e1c4acf..48d13d7a2b 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -136,6 +136,9 @@ SGLANG_CMD=( --enable-cache-report ) +agentic_apply_server_launch sglang "${SGLANG_CMD[@]}" +SGLANG_CMD=("${AGENTX_SERVER_COMMAND[@]}") + write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/configs/agentx-launchers.json b/configs/agentx-launchers.json new file mode 100644 index 0000000000..c4e5cdd6f2 --- /dev/null +++ b/configs/agentx-launchers.json @@ -0,0 +1,137 @@ +{ + "version": 1, + "recipes": { + "qwen3.5-fp8-mi325x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi325x", + "benchmark_script": "single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh", + "launch_overrides_version": 1, + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "precision": "fp8" + }, + "qwen3.5-fp4-mi355x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh", + "launch_overrides_version": 1, + "model": "amd/Qwen3.5-397B-A17B-MXFP4", + "precision": "fp4" + }, + "qwen3.5-fp8-mi300x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi300x", + "benchmark_script": "single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh", + "launch_overrides_version": 1, + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "precision": "fp8" + }, + "kimik3-fp4-mi355x-vllm-agentic-mtp": { + "framework": "vllm", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/kimik3_fp4_mi355x_mtp.sh", + "launch_overrides_version": 1, + "model": "moonshotai/Kimi-K3", + "precision": "fp4" + }, + "dsv4-fp4-mi355x-vllm-agentic-mtp": { + "framework": "vllm", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh", + "launch_overrides_version": 1, + "model": "deepseek-ai/DeepSeek-V4-Pro", + "precision": "fp4" + }, + "minimaxm3-fp8-mi300x-vllm-agentic-mtp": { + "framework": "vllm", + "runner_type": "mi300x", + "benchmark_script": "single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh", + "launch_overrides_version": 1, + "model": "MiniMaxAI/MiniMax-M3-MXFP8", + "precision": "fp8" + }, + "glm5.2-fp8-mi325x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi325x", + "benchmark_script": "single_node/agentic/glm5.2_fp8_mi325x_mtp.sh", + "launch_overrides_version": 1, + "model": "zai-org/GLM-5.2-FP8", + "precision": "fp8" + }, + "minimaxm3-fp8-mi325x-vllm-agentic-mtp": { + "framework": "vllm", + "runner_type": "mi325x", + "benchmark_script": "single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh", + "launch_overrides_version": 1, + "model": "MiniMaxAI/MiniMax-M3-MXFP8", + "precision": "fp8" + }, + "minimaxm3-fp4-mi355x-vllm-agentic-mtp": { + "framework": "vllm", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh", + "launch_overrides_version": 1, + "model": "amd/MiniMax-M3-MXFP4", + "precision": "fp4" + }, + "glm5.2-fp4-mi355x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh", + "launch_overrides_version": 1, + "model": "amd/GLM-5.2-MXFP4", + "precision": "fp4" + }, + "dsv4-fp4-mi355x-sglang-agentic-mtp": { + "framework": "sglang", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh", + "launch_overrides_version": 1, + "model": "deepseek-ai/DeepSeek-V4-Pro", + "precision": "fp4" + } + }, + "generic": { + "custom-sglang-mi300x": { + "framework": "sglang", + "runner_type": "mi300x", + "benchmark_script": "single_node/agentic/generic_sglang.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + }, + "custom-sglang-mi325x": { + "framework": "sglang", + "runner_type": "mi325x", + "benchmark_script": "single_node/agentic/generic_sglang.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + }, + "custom-sglang-mi355x": { + "framework": "sglang", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/generic_sglang.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + }, + "custom-vllm-mi300x": { + "framework": "vllm", + "runner_type": "mi300x", + "benchmark_script": "single_node/agentic/generic_vllm.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + }, + "custom-vllm-mi325x": { + "framework": "vllm", + "runner_type": "mi325x", + "benchmark_script": "single_node/agentic/generic_vllm.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + }, + "custom-vllm-mi355x": { + "framework": "vllm", + "runner_type": "mi355x", + "benchmark_script": "single_node/agentic/generic_vllm.sh", + "launch_overrides_version": 1, + "max_gpus": 8 + } + } +} diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index a3afb3076a..042cdc6fc8 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -323,6 +323,45 @@ gh run cancel --repo SemiAnalysisAI/InferenceX Use `scancel` or process termination only with explicit approval and a concrete reason. They can bypass cleanup or strand the runner. After a recipe fix, dispatch one targeted fast e2e point, inspect it live, then reserve a canonical run/full sweep for the candidate that passed. +## 11. Apply verified server launch extensions + +`configs/agentx-launchers.json` declares the supported recipe-to-launcher +mapping and launch extension version. The 11 declared AMD SGLang/vLLM +launchers call `agentic_apply_server_launch` after their normal setup and +command assembly, immediately before recording and spawning the server. +Unset `AGENTX_LAUNCH_OVERRIDES_FILE` preserves the original command/environment. + +To request an extension, set `AGENTX_LAUNCH_OVERRIDES_FILE` to a JSON file, +`AGENTX_LAUNCH_OVERRIDES_SHA256` to its normalized canonical JSON hash, and +`AGENTX_SERVER_LAUNCH_FILE` to an output file in the existing result directory. +The v1 request contains `version: 1`, token arrays `append_args` and +`remove_args` (long option names), boolean `replace_args`, string mapping +`env`, name array `unset_env`, optional absolute `executable`, absolute-path +SHA256 mapping `source_files`, and absolute-path array `absent_source_files`. +Missing collections default to empty, `replace_args` to false, and +`executable` to null. Hash the complete normalized object using JSON with +sorted keys, separators `(',', ':')`, `ensure_ascii=True`, encoded as UTF-8. + +Removal must match exactly one base long option and includes its values up to +the next long option. Ambiguous short options fail. Replacement keeps fixed +entrypoint positionals and model/host/port/topology flags; appended arguments +cannot modify those protected controls. Server environment overrides are +applied after launcher exports and affect only the server child process. +The executable override selects a Python interpreter for SGLang or a vLLM +entrypoint for vLLM; it does not change the remaining entrypoint tokens. + +Source contents/absences are checked after recipe setup, so an installer that +overwrites a candidate is rejected before launch. The receipt records +base/effective argv, changed environment, selected runtime controls (without +credential-like names), executable resolution, request hash and source +evidence. Its `evidence_sha256` hashes the complete receipt except that field. +This is launch evidence, not proof that model loading or the replay succeeded. +Use the ordinary canonical result gate after every candidate measurement. + +GPU-free verification: `python -m pytest utils/agentic/test_server_launch.py`. +Real GPU performance for a candidate still requires the fast/canonical flow +above; an empty `{ "version": 1 }` request must preserve canonical argv/env. + ## Completion checklist - Matrix preview matches intended scenario, topology, eval mode, and concurrency. @@ -333,3 +372,38 @@ Use `scancel` or process termination only with explicit approval and a concrete - Every backend/frontend and metrics source is represented in live evidence. - Fast/smoke results are labeled diagnostic. Only the canonical candidate is used for final comparison. - Workflow and artifact collection conclude green before success is reported. + +## 12. Explicit custom models on AMD single-node runners + +`configs/agentx-launchers.json` declares `generic` capabilities named +`custom-{sglang,vllm}-{mi300x,mi325x,mi355x}`. The launchers +`single_node/agentic/generic_sglang.sh` and `generic_vllm.sh` call the same +`build_replay_cmd` and `run_agentic_replay_and_write_outputs` functions as registered +recipes. They do not use the fixed-length/random-request benchmark client. + +A resolver supplies an explicit model, precision, image, TP, EP, concurrency, +`AGENTX_CUSTOM_RECIPE=1`, `AGENTX_NATIVE_CONTEXT_LENGTH`, `AGENTX_MAX_MODEL_LEN`, +and `AGENTX_MODEL_CONFIG_SHA256`. The image is the caller's tested framework +runtime. TP fits one eight-GPU node; SGLang EP divides TP, while vLLM EP is 1 or TP. +These launchers do not infer PP, DP attention, disaggregation, KV offload, or MTP. +The framework loads the model's own quantization configuration when applicable. + +`utils/agentic/custom_model.py` checks local `MODEL_PATH/config.json`, or a +metadata-only HuggingFace download, against the resolved SHA256 and positive +native context. It never executes model configuration code. The fixed replay +context may equal or be below the confirmed native context, never above it; it +is passed to both server and replay and cannot be changed by launch overrides. +The trace loader is `semianalysis_cc_traces_weka_062126_256k`. Context, config hash, +loader, image, and topology belong in the resolved recipe fingerprint. Reusing a +result across those identities is invalid. + +Raw output adds `custom_recipe`, `native_context_length`, `max_model_len`, and +`model_config_sha256`. The canonical protocol, request validity checks, and +3600-second measurement are unchanged. Custom models are explicit runtime +points, not additions to the registered master-config sweep matrix. GPU-free +boundary tests do not establish that a particular model/image can run; validate +the actual model/image and retain launch, replay, and aggregate evidence. + +`server_launch.apply_launch_args(argv, overrides, framework)` exposes the pure v1 +argv contract for receipt verification. `prepare_launch` additionally checks +executable availability and source hashes/absences on the server host. diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 669ed290c3..577c3b2b97 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -323,6 +323,41 @@ gh run cancel --repo SemiAnalysisAI/InferenceX 只有在获得明确批准且有具体理由时才使用 `scancel` 或终止进程;否则可能绕过 cleanup 或使 runner 残留。修复 recipe 后,先分派一个目标 fast e2e 点并实时检查,只有通过检查的 candidate 才值得进行 canonical 运行/完整 sweep。 +## 11. 使用带校验的服务启动扩展 + +`configs/agentx-launchers.json` 声明支持的 recipe 与 launcher 对应关系及扩展版本。 +其中 11 个 AMD SGLang/vLLM launcher 在完成原有准备步骤和命令组装后、 +记录并启动服务前调用 `agentic_apply_server_launch`。 +不设置 `AGENTX_LAUNCH_OVERRIDES_FILE` 时,原有命令和环境保持不变。 + +启用扩展时,将 `AGENTX_LAUNCH_OVERRIDES_FILE` 设为 JSON 文件路径, +`AGENTX_LAUNCH_OVERRIDES_SHA256` 设为规范化 JSON 的哈希, +`AGENTX_SERVER_LAUNCH_FILE` 设为已有结果目录中的输出文件。 +v1 请求包含 `version: 1`、token 数组 `append_args`、长选项名称数组 +`remove_args`、布尔值 `replace_args`、字符串映射 `env`、名称数组 +`unset_env`、可选绝对路径 `executable`、绝对路径到 SHA256 的映射 +`source_files`,以及绝对路径数组 `absent_source_files`。 +未提供的集合默认为空,`replace_args` 默认为 false,`executable` 默认为 null。 +完整规范化对象使用排序键、`(',', ':')` 分隔符和 `ensure_ascii=True` +序列化为 JSON,再以 UTF-8 编码计算 SHA256。 + +删除必须恰好匹配一个原有长选项,并包含其后直到下一个长选项前的值; +存在歧义的短选项会被拒绝。替换保留固定入口位置参数以及模型、host、port、 +拓扑选项,追加参数也不能改变这些受保护字段。环境覆盖在 launcher 的 export +之后生效,仅影响服务子进程。SGLang 的 executable 指向 Python 解释器, +vLLM 的 executable 指向 vLLM 入口,不改变其余入口 token。 + +源码内容和缺失路径在 recipe 准备完成后校验,因此安装步骤覆盖候选代码时, +启动会立即被拒绝。回执记录原始和生效 argv、环境变更、选定运行时控制变量 +(排除类似凭证的名称)、可执行文件解析结果、请求哈希和源码证据。 +`evidence_sha256` 对除该字段以外的完整回执计算哈希。 +这只证明启动输入,不证明模型加载或 replay 成功;每次候选测量仍须通过 +正常的 canonical 结果校验。 + +无需 GPU 的验证命令为 `python -m pytest utils/agentic/test_server_launch.py`。 +候选的真实 GPU 性能仍须按上文执行 fast/canonical 流程; +空请求 `{ "version": 1 }` 必须保持 canonical argv/env 不变。 + ## 完成检查清单 - 矩阵预览符合预期 scenario、topology、eval mode 与 concurrency。 @@ -333,3 +368,31 @@ gh run cancel --repo SemiAnalysisAI/InferenceX - 每个 backend/frontend 与 metrics source 都在实时证据中有所体现。 - Fast/smoke 结果明确标为诊断用途;只有 canonical candidate 用于最终比较。 - 在报告成功前,工作流与 artifact collection 均已得出 green 结论。 + +## 12. AMD 单节点上的显式自定义模型 + +`configs/agentx-launchers.json` 声明名为 +`custom-{sglang,vllm}-{mi300x,mi325x,mi355x}` 的 `generic` 能力。 +`single_node/agentic/generic_sglang.sh` 和 `generic_vllm.sh` 与已注册配方调用同一套 +`build_replay_cmd`、`run_agentic_replay_and_write_outputs`,不使用固定长度或随机请求客户端。 + +解析器必须明确给出 model、precision、image、TP、EP、concurrency、 +`AGENTX_CUSTOM_RECIPE=1`、`AGENTX_NATIVE_CONTEXT_LENGTH`、`AGENTX_MAX_MODEL_LEN` +和 `AGENTX_MODEL_CONFIG_SHA256`。image 是调用方已经验证的框架运行环境。 +TP 不得超过单节点的 8 张 GPU;SGLang 的 EP 必须整除 TP,vLLM 的 EP 只能为 1 或 TP。 +这些启动器不推断 PP、DP attention、disaggregation、KV offload 或 MTP; +量化模型由框架读取模型自带的量化配置。 + +`utils/agentic/custom_model.py` 读取本地 `MODEL_PATH/config.json`,或仅下载 HuggingFace +配置元数据,核验已解析的 SHA256 和正整数原生上下文,不执行模型配置代码。 +固定重放上下文可以等于或小于已确认的原生上下文,不能超过;该值同时传给服务端和重放客户端, +启动覆盖参数不能修改它。trace loader 固定为 `semianalysis_cc_traces_weka_062126_256k`。 +上下文、配置哈希、loader、image 和拓扑都必须进入配方指纹,不能跨这些身份复用结果。 + +原始结果增加 `custom_recipe`、`native_context_length`、`max_model_len` 和 +`model_config_sha256`。canonical 协议、请求有效性检查和 3600 秒测量时长保持不变。 +自定义模型属于显式运行点,不会自动增加已注册 master-config 的 sweep 矩阵行。 +无 GPU 边界测试不能证明某个模型和镜像可运行;仍需实测并保留启动、重放及聚合证据。 + +`server_launch.apply_launch_args(argv, overrides, framework)` 提供无文件访问的 v1 参数转换接口, +可用于核对启动回执。`prepare_launch` 还会在服务端检查可执行文件及源文件的哈希或缺失状态。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 085e9cb889..601c5778a3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6049,3 +6049,39 @@ - "Add GB200 GLM-5.2 FP4 Dynamo-SGLang AgentX with EAGLE MTP, HiCache DRAM offload, and KV-aware correlation-ID affinity." - "Measure the TP8 aggregate curve and tuned 1P1D TP4 HiCache c10/c12 disaggregated frontier with every logical SGLang metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2620 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + - qwen3.5-fp8-mi300x-sglang-agentic-mtp + - kimik3-fp4-mi355x-vllm-agentic-mtp + - dsv4-fp4-mi355x-vllm-agentic-mtp + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + - glm5.2-fp8-mi325x-sglang-agentic-mtp + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + - glm5.2-fp4-mi355x-sglang-agentic-mtp + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add opt-in structured server argv/environment extensions with protected recipe topology and verified source/launch evidence; unset extensions preserve canonical commands." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + - qwen3.5-fp8-mi300x-sglang-agentic-mtp + - kimik3-fp4-mi355x-vllm-agentic-mtp + - dsv4-fp4-mi355x-vllm-agentic-mtp + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + - glm5.2-fp8-mi325x-sglang-agentic-mtp + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + - glm5.2-fp4-mi355x-sglang-agentic-mtp + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add explicit custom-model SGLang/vLLM AgentX launchers on MI300X/MI325X/MI355X using the canonical replay path and verified model context metadata; protect fixed context from server overrides." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index c0d8d70ce1..db848318b5 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -237,6 +237,14 @@ def build_agg( } agg.update(multinode_fields) + if env_bool("AGENTX_CUSTOM_RECIPE"): + agg.update({ + "custom_recipe": True, + "native_context_length": env_int("AGENTX_NATIVE_CONTEXT_LENGTH"), + "max_model_len": env_int("MAX_MODEL_LEN"), + "model_config_sha256": os.environ.get("AGENTX_MODEL_CONFIG_SHA256", ""), + }) + router = optional_component_metadata("ROUTER_METADATA") if router is not None: agg["router"] = router diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index a567e9aced..e4ba154331 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1507,3 +1507,19 @@ def test_processor_supports_per_run_subdir_layout(tmp_path: Path): output_dir = tmp_path / "out" agg = _run_processor(result_dir, output_dir) assert agg["num_requests_total"] == 1 + + +def test_processor_binds_custom_model_metadata_and_fingerprint(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={ + "AGENTX_CUSTOM_RECIPE": "1", + "AGENTX_NATIVE_CONTEXT_LENGTH": "40960", + "MAX_MODEL_LEN": "8192", + "AGENTX_MODEL_CONFIG_SHA256": "a" * 64, + "RECIPE_FINGERPRINT": "c" * 64, + }) + assert agg["custom_recipe"] is True + assert agg["native_context_length"] == 40960 + assert agg["max_model_len"] == 8192 + assert agg["model_config_sha256"] == "a" * 64 + assert agg["recipe_fingerprint"] == "c" * 64 diff --git a/utils/agentic/custom_model.py b/utils/agentic/custom_model.py new file mode 100644 index 0000000000..7b1902d3ef --- /dev/null +++ b/utils/agentic/custom_model.py @@ -0,0 +1,68 @@ +"""Verify a custom AgentX model's context metadata without executing model code.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def native_context_length(config: dict[str, Any]) -> int: + text = config.get("text_config", config.get("language_config", config)) + if not isinstance(text, dict): + raise ValueError("Model text configuration must be an object") # noqa: TRY004 + values = [ + text.get(name) + for name in ( + "max_position_embeddings", + "max_sequence_length", + "seq_length", + "n_positions", + ) + ] + confirmed = [value for value in values if type(value) is int and value > 0] + if not confirmed: + raise ValueError( + "config.json does not declare a positive native context length" + ) + # Do not guess a RoPE extension or trust tokenizer sentinel limits. When + # aliases disagree, only their common supported range is confirmed. + return min(confirmed) + + +def verify_model_config( + path: Path, expected_sha256: str, native: int, maximum: int +) -> None: + contents = path.read_bytes() + if hashlib.sha256(contents).hexdigest() != expected_sha256: + raise ValueError("Custom AgentX model config changed after recipe resolution") + config = json.loads(contents) + if not isinstance(config, dict) or native_context_length(config) != native: + raise ValueError( + "Custom AgentX native context metadata does not match its recipe" + ) + if not 0 < maximum <= native: + raise ValueError( + "Custom AgentX replay context must be within the native model context" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument("--native-context", type=int, required=True) + parser.add_argument("--max-context", type=int, required=True) + args = parser.parse_args() + try: + verify_model_config( + args.config, args.expected_sha256, args.native_context, args.max_context + ) + except (OSError, ValueError) as exc: + parser.exit(2, f"Custom AgentX model metadata rejected: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/utils/agentic/server_launch.py b/utils/agentic/server_launch.py new file mode 100644 index 0000000000..19b93ed871 --- /dev/null +++ b/utils/agentic/server_launch.py @@ -0,0 +1,369 @@ +"""Apply the explicit AgentX server launch extension before starting a server. + +Only the server command/environment is changed. Replay inputs and recipe +topology stay owned by the launcher. No shell evaluation is performed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +from pathlib import Path +from typing import Any + +PROTECTED_ARGS = frozenset( + { + "--model", + "--model-path", + "--max-model-len", + "--context-length", + "--served-model-name", + "--host", + "--port", + "--tp", + "--tp-size", + "--tensor-parallel-size", + "--pp", + "--pp-size", + "--pipeline-parallel-size", + "--ep", + "--ep-size", + "--expert-parallel-size", + "--dp", + "--dp-size", + "--data-parallel-size", + "--data-parallel-rank", + "--data-parallel-start-rank", + "--data-parallel-size-local", + "--data-parallel-address", + "--data-parallel-rpc-port", + "--enable-dp-attention", + "--enable-expert-parallel", + "--decode-context-parallel-size", + "--prefill-context-parallel-size", + "--dcp-size", + "--pcp-size", + "--nnodes", + "--node-rank", + "--dist-init-addr", + } +) +PROTECTED_ENV = frozenset( + { + "MODEL", + "MODEL_PATH", + "MAX_MODEL_LEN", + "MODEL_NAME", + "SERVED_MODEL_NAME", + "MODEL_PREFIX", + "TP", + "PP_SIZE", + "EP_SIZE", + "DP_ATTENTION", + "DCP_SIZE", + "PCP_SIZE", + "CONC", + "DURATION", + "PORT", + "RUN_EVAL", + "EVAL_ONLY", + "IS_AGENTIC", + "SCENARIO_TYPE", + "SCENARIO_SUBDIR", + "RECIPE_FINGERPRINT", + "ROCR_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + } +) +PROTECTED_ENV_PREFIXES = ("AGENTX_", "AGENTIC_", "AIPERF_", "SGLANG_SIMULATE_ACC_") +FIELDS = { + "version", + "append_args", + "remove_args", + "replace_args", + "env", + "unset_env", + "source_files", + "absent_source_files", + "executable", +} +RUNTIME_ENV_NAMES = frozenset( + { + "PATH", + "PYTHONPATH", + "LD_LIBRARY_PATH", + "LIBRARY_PATH", + "SGLANG_SRC_PYTHONPATH", + "GPU_ARCHS", + } +) +RUNTIME_ENV_PREFIXES = ( + "SGLANG_", + "VLLM_", + "AITER_", + "TRITON_", + "TORCH_", + "PYTORCH_", + "HIP_", + "ROCR_", + "HSA_", + "NCCL_", + "RCCL_", + "OMP_", +) + + +def digest(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + + +def validate_overrides(value: Any) -> dict[str, Any]: + if ( + not isinstance(value, dict) + or type(value.get("version")) is not int + or value["version"] != 1 + ): + raise ValueError("AgentX launch_overrides requires version: 1") + if unknown := value.keys() - FIELDS: + raise ValueError(f"Unknown AgentX launch override fields: {sorted(unknown)}") + result = { + "version": 1, + "append_args": [], + "remove_args": [], + "replace_args": False, + "env": {}, + "unset_env": [], + "source_files": {}, + "absent_source_files": [], + "executable": None, + **value, + } + for key in ("append_args", "remove_args", "unset_env", "absent_source_files"): + if not isinstance(result[key], list) or any( + not isinstance(item, str) or "\0" in item for item in result[key] + ): + raise ValueError(f"{key} must be a list of strings without NUL bytes") + if type(result["replace_args"]) is not bool: + raise ValueError("replace_args must be a boolean") + for key in ("env", "source_files"): + if not isinstance(result[key], dict) or any( + not isinstance(name, str) + or not isinstance(item, str) + or "\0" in name + or "\0" in item + for name, item in result[key].items() + ): + raise ValueError(f"{key} must map strings to strings without NUL bytes") + if len(set(result["remove_args"])) != len(result["remove_args"]): + raise ValueError("remove_args must not contain duplicates") + for name in result["remove_args"]: + if not re.fullmatch(r"--[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise ValueError(f"remove_args must contain long option names: {name!r}") + if name in PROTECTED_ARGS: + raise ValueError(f"Cannot remove protocol option {name}") + names = set(result["env"]) | set(result["unset_env"]) + if set(result["env"]) & set(result["unset_env"]): + raise ValueError("An environment variable cannot be both set and unset") + for name in names: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Invalid environment variable name: {name!r}") + if name in PROTECTED_ENV or name.startswith(PROTECTED_ENV_PREFIXES): + raise ValueError(f"Cannot override protocol environment variable {name}") + for name, checksum in result["source_files"].items(): + if not Path(name).is_absolute() or not re.fullmatch(r"[0-9a-f]{64}", checksum): + raise ValueError( + "source_files requires absolute paths and lowercase SHA256 values" + ) + for name in result["absent_source_files"]: + if not Path(name).is_absolute() or name in result["source_files"]: + raise ValueError("absent_source_files requires distinct absolute paths") + executable = result["executable"] + if executable is not None and ( + not isinstance(executable, str) + or "\0" in executable + or not Path(executable).is_absolute() + ): + raise ValueError("executable must be an absolute path or null") + return result + + +def option_groups(argv: list[str]) -> tuple[list[str], list[list[str]]]: + """Separate fixed executable/positionals and unambiguous long options.""" + prefix: list[str] = [] + groups: list[list[str]] = [] + for token in argv: + if token.startswith("--"): + name = token.partition("=")[0] + if not re.fullmatch(r"--[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise ValueError(f"Ambiguous server option: {token!r}") + groups.append([token]) + elif groups: + # Short options are ambiguous with values and cannot safely be + # removed as part of the preceding long option. Negative numbers + # are ordinary values. Equals-style options already own a value. + if "=" in groups[-1][0] or ( + token.startswith("-") + and not re.fullmatch(r"-\d+(?:\.\d+)?(?:[eE][+-]?\d+)?", token) + ): + raise ValueError(f"Ambiguous server option value: {token!r}") + groups[-1].append(token) + else: + prefix.append(token) + return prefix, groups + + +def apply_launch_args( + argv: list[str], + overrides: dict[str, Any], + framework: str, +) -> list[str]: + """Apply the v1 argv contract without reading files or the process environment.""" + overrides = validate_overrides(overrides) + if framework not in {"sglang", "vllm"} or not argv: + raise ValueError( + "AgentX launch extension supports nonempty SGLang/vLLM commands" + ) + effective = list(argv) + # Empty requests do not parse/rewrite an existing command. This preserves + # every canonical launcher token, including options unfamiliar to v1. + if ( + overrides["append_args"] + or overrides["remove_args"] + or overrides["replace_args"] + ): + prefix, groups = option_groups(argv) + extra_prefix, extra = option_groups(overrides["append_args"]) + if extra_prefix: + raise ValueError( + "append_args cannot inject an executable or positional arguments" + ) + if any(group[0].partition("=")[0] in PROTECTED_ARGS for group in extra): + raise ValueError("append_args cannot change model, topology, host or port") + for name in overrides["remove_args"]: + matches = [group for group in groups if group[0].partition("=")[0] == name] + if len(matches) != 1: + raise ValueError( + f"remove_args must match exactly one existing option: {name}" + ) + groups.remove(matches[0]) + if overrides["replace_args"]: + # Keep the executable/model positionals and all protocol options; + # only the optional serving configuration is replaced. + groups = [ + group + for group in groups + if group[0].partition("=")[0] in PROTECTED_ARGS + ] + effective = prefix + [token for group in groups + extra for token in group] + if overrides["executable"] is not None: + effective[0] = overrides["executable"] + return effective + + +def prepare_launch( + argv: list[str], + overrides: dict[str, Any], + framework: str, + environ: dict[str, str] | None = None, +) -> tuple[list[str], dict[str, Any]]: + overrides = validate_overrides(overrides) + environ = dict(os.environ if environ is None else environ) + effective = apply_launch_args(argv, overrides, framework) + if overrides["executable"] is not None: + executable = Path(overrides["executable"]) + if not executable.is_file() or not os.access(executable, os.X_OK): + raise ValueError( + f"Requested server executable is not executable: {executable}" + ) + sources = {} + for name, expected in overrides["source_files"].items(): + path = Path(name) + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise ValueError(f"AgentX source file changed before server launch: {name}") + sources[name] = actual + for name in overrides["absent_source_files"]: + if os.path.lexists(name): + raise ValueError( + f"AgentX deleted source file reappeared before launch: {name}" + ) + touched = sorted(set(overrides["env"]) | set(overrides["unset_env"])) + base_env = {name: environ.get(name) for name in touched} + effective_env = {name: overrides["env"].get(name) for name in touched} + child_env = dict(environ) + for name in overrides["unset_env"]: + child_env.pop(name, None) + child_env.update(overrides["env"]) + resolved_executable = shutil.which(effective[0], path=child_env.get("PATH")) + if resolved_executable is None: + raise ValueError(f"Server executable not found: {effective[0]}") + evidence = { + "version": 1, + "framework": framework, + "base_argv": argv, + "effective_argv": effective, + "base_env": base_env, + "effective_env": effective_env, + "overrides_sha256": digest(overrides), + "source_files": sources, + "absent_source_files": overrides["absent_source_files"], + "resolved_executable": str(Path(resolved_executable).resolve()), + "runtime_environment": { + name: value + for name, value in sorted(child_env.items()) + if (name in RUNTIME_ENV_NAMES or name.startswith(RUNTIME_ENV_PREFIXES)) + and not any( + secret in name.upper() + for secret in ("TOKEN", "PASSWORD", "SECRET", "API_KEY") + ) + }, + } + evidence["evidence_sha256"] = digest(evidence) + # The env utility execs the original server; overrides never leak to the + # router or replay subprocesses in the parent launcher. + command = [] + if touched: + command.append("env") + for name in overrides["unset_env"]: + command.extend(["-u", name]) + command.extend(f"{name}={value}" for name, value in overrides["env"].items()) + return command + effective, evidence + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--framework", required=True) + parser.add_argument("--overrides", required=True, type=Path) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("argv", nargs=argparse.REMAINDER) + args = parser.parse_args() + try: + request = validate_overrides(json.loads(args.overrides.read_text())) + if digest(request) != args.expected_sha256: + raise ValueError("AgentX launch override hash does not match the request") + argv = args.argv[1:] if args.argv[:1] == ["--"] else args.argv + command, evidence = prepare_launch(argv, request, args.framework) + args.output.write_bytes(b"".join(token.encode() + b"\0" for token in command)) + temporary = args.evidence.with_name(args.evidence.name + ".tmp") + temporary.write_text(json.dumps(evidence, indent=2) + "\n") + temporary.replace(args.evidence) + except (OSError, ValueError) as exc: + parser.exit(2, f"AgentX server launch rejected: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/utils/agentic/test_custom_model.py b/utils/agentic/test_custom_model.py new file mode 100644 index 0000000000..b5b1d006db --- /dev/null +++ b/utils/agentic/test_custom_model.py @@ -0,0 +1,182 @@ +"""GPU-free checks of custom metadata and the real Bash/launch/replay boundary.""" + +import hashlib +import json +import os +import shlex +import subprocess +from pathlib import Path + +import pytest + +from utils.agentic.custom_model import native_context_length, verify_model_config +from utils.agentic.server_launch import apply_launch_args, digest, validate_overrides + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.mark.parametrize( + "metadata, expected", + [ + ({"max_position_embeddings": 40960}, 40960), + ({"text_config": {"max_position_embeddings": 32768}}, 32768), + ({"max_position_embeddings": 40960, "seq_length": 8192}, 8192), + ], +) +def test_native_context(metadata, expected): + assert native_context_length(metadata) == expected + + +@pytest.mark.parametrize( + "metadata", + [ + {}, + {"max_position_embeddings": True}, + {"max_position_embeddings": -1}, + {"text_config": []}, + ], +) +def test_unknown_context_is_rejected(metadata): + with pytest.raises(ValueError): + native_context_length(metadata) + + +def test_verifier_binds_metadata_bytes_and_context(tmp_path): + path = tmp_path / "config.json" + path.write_text('{"max_position_embeddings":40960}') + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + verify_model_config(path, checksum, 40960, 8192) + with pytest.raises(ValueError, match="within"): + verify_model_config(path, checksum, 40960, 50000) + with pytest.raises(ValueError, match="does not match"): + verify_model_config(path, checksum, 32768, 8192) + path.write_text('{"max_position_embeddings":32768}') + with pytest.raises(ValueError, match="changed"): + verify_model_config(path, checksum, 40960, 8192) + + +def test_pure_argv_contract_requires_no_source_or_executable_access(): + base = [ + "python3", + "-m", + "sglang.launch_server", + "--tp", + "1", + "--context-length", + "8192", + "--foo", + "old", + ] + overrides = validate_overrides( + { + "version": 1, + "executable": "/not/installed/python", + "source_files": {"/not/present.py": "0" * 64}, + "replace_args": True, + "append_args": ["--foo", "new"], + } + ) + assert apply_launch_args(base, overrides, "sglang") == [ + "/not/installed/python" + ] + base[1:7] + ["--foo", "new"] + + +def test_generic_manifest_declares_only_supported_single_node_launchers(): + generic = json.loads((ROOT / "configs/agentx-launchers.json").read_text())[ + "generic" + ] + assert len(generic) == 6 + for framework in ("sglang", "vllm"): + for runner in ("mi300x", "mi325x", "mi355x"): + row = generic[f"custom-{framework}-{runner}"] + assert row == { + "framework": framework, + "runner_type": runner, + "benchmark_script": f"single_node/agentic/generic_{framework}.sh", + "launch_overrides_version": 1, + "max_gpus": 8, + } + path = ROOT / "benchmarks" / row["benchmark_script"] + subprocess.run(["bash", "-n", str(path)], check=True) + assert f"run_generic_agentic_server {framework}" in path.read_text() + + +@pytest.mark.parametrize("framework", ["sglang", "vllm"]) +def test_generic_bash_builds_canonical_replay_and_verified_launch(tmp_path, framework): + model = tmp_path / "model files with spaces" + model.mkdir() + metadata = model / "config.json" + metadata.write_text('{"max_position_embeddings":40960}') + output = tmp_path / "results with spaces" + output.mkdir() + fake_server = tmp_path / "fake server" + fake_server.write_text("#!/usr/bin/env python3\nimport time\ntime.sleep(30)\n") + fake_server.chmod(0o755) + overrides = validate_overrides( + { + "version": 1, + "executable": str(fake_server), + "append_args": ["--test-value", "literal space"], + "env": {"SGLANG_TEST": "verified"}, + } + ) + request = output / "request.json" + request.write_text(json.dumps(overrides)) + evidence = output / "server_launch.json" + env = dict( + os.environ, + MODEL="Qwen/Qwen3-0.6B", + MODEL_PATH=str(model), + TP="1", + EP_SIZE="1", + CONC="64", + PRECISION="bf16", + FRAMEWORK=framework, + DURATION="3600", + RESULT_DIR=str(output), + KV_OFFLOADING="none", + IS_AGENTIC="1", + AGENTX_CUSTOM_RECIPE="1", + AGENTX_NATIVE_CONTEXT_LENGTH="40960", + AGENTX_MAX_MODEL_LEN="8192", + AGENTX_MODEL_CONFIG_SHA256=hashlib.sha256(metadata.read_bytes()).hexdigest(), + AGENTX_LAUNCH_OVERRIDES_FILE=str(request), + AGENTX_LAUNCH_OVERRIDES_SHA256=digest(overrides), + AGENTX_SERVER_LAUNCH_FILE=str(evidence), + MAX_MODEL_LEN="123", + AIPERF_EXPERIMENTAL_FAST="0", + ) + # Stub external services only. The real library verifies metadata, composes + # server argv, applies launch edits, and composes the canonical replay CLI. + script = f""" +set -eo pipefail +source {shlex.quote(str(ROOT / "benchmarks/benchmark_lib.sh"))} +install_agentic_deps() {{ AIPERF_CLI=aiperf; }} +resolve_trace_source() {{ :; }} +wait_for_server_ready() {{ :; }} +run_agentic_replay_and_write_outputs() {{ printf '%s\\n' "$REPLAY_CMD" > "$RESULT_DIR/replay.txt"; }} +run_benchmark_serving() {{ echo "wrong fixed-length client" >&2; exit 91; }} +run_generic_agentic_server {framework} +""" + completed = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + timeout=15, + check=False, + ) + assert completed.returncode == 0, completed.stderr + receipt = json.loads(evidence.read_text()) + assert receipt["effective_argv"][-2:] == ["--test-value", "literal space"] + assert receipt["effective_env"] == {"SGLANG_TEST": "verified"} + assert str(model) in receipt["base_argv"] + context_arg = "--context-length" if framework == "sglang" else "--max-model-len" + assert receipt["base_argv"][receipt["base_argv"].index(context_arg) + 1] == "8192" + replay = (output / "replay.txt").read_text() + assert "--scenario inferencex-agentx-mvp" in replay + assert "--benchmark-duration 3600" in replay + assert "--max-context-length 8192" in replay + assert "--concurrency 64" in replay + assert "--random-seed 42" in replay + assert "--warmup-requests-per-lane 10" in replay diff --git a/utils/agentic/test_server_launch.py b/utils/agentic/test_server_launch.py new file mode 100644 index 0000000000..761cb3b90a --- /dev/null +++ b/utils/agentic/test_server_launch.py @@ -0,0 +1,300 @@ +"""GPU-free tests for the real AgentX server launch boundary.""" + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from utils.agentic.server_launch import ( + PROTECTED_ARGS, + digest, + prepare_launch, + validate_overrides, +) + +ROOT = Path(__file__).resolve().parents[2] +BASE = [ + sys.executable, + "-m", + "sglang.launch_server", + "--model-path", + "/models/a b", + "--host", + "0.0.0.0", + "--port", + "8888", + "--tp", + "4", + "--mem-fraction-static", + "0.85", + "--enable-metrics", +] + + +def request(**kwargs): + return validate_overrides({"version": 1, **kwargs}) + + +def test_empty_extension_preserves_every_command_token_and_environment(): + command, receipt = prepare_launch( + BASE, request(), "sglang", {"PATH": os.environ["PATH"]} + ) + assert command == BASE + assert receipt["effective_argv"] == BASE + assert receipt["effective_env"] == {} + assert receipt["evidence_sha256"] == digest( + {k: v for k, v in receipt.items() if k != "evidence_sha256"} + ) + + +def test_remove_then_append_keeps_quoted_values_and_protocol(): + command, receipt = prepare_launch( + BASE, + request( + remove_args=["--mem-fraction-static"], + append_args=["--mem-fraction-static", "0.8", "--json", '{"spaces": "a b"}'], + ), + "sglang", + ) + assert "0.85" not in command + assert command[-4:] == [ + "--mem-fraction-static", + "0.8", + "--json", + '{"spaces": "a b"}', + ] + assert receipt["base_argv"] == BASE + assert command[:11] == BASE[:11] + + +def test_remove_equals_form_and_negative_multivalue_arguments(): + command, _ = prepare_launch( + BASE + ["--test=-1"], request(remove_args=["--test"]), "sglang" + ) + assert command == BASE + command, _ = prepare_launch( + BASE + ["--test", "-1", "-2.5"], request(remove_args=["--test"]), "sglang" + ) + assert command == BASE + + +def test_replace_preserves_protocol_options_and_executable_positionals(): + command, _ = prepare_launch( + BASE, request(replace_args=True, append_args=["--foo", "bar"]), "sglang" + ) + assert command == BASE[:11] + ["--foo", "bar"] + + +@pytest.mark.parametrize("name", sorted(PROTECTED_ARGS)) +def test_protocol_options_cannot_be_added_or_removed(name): + with pytest.raises(ValueError, match="protocol option"): + request(remove_args=[name]) + with pytest.raises(ValueError, match="cannot change"): + prepare_launch(BASE, request(append_args=[f"{name}=different"]), "sglang") + + +@pytest.mark.parametrize( + "kwargs", + [ + {"remove_args": ["--missing"]}, + {"append_args": ["positional"]}, + {"append_args": ["--foo", "-p", "123"]}, + {"append_args": ["--foo=bar", "orphan"]}, + {"append_args": ["--"]}, + ], +) +def test_ambiguous_or_unmatched_edits_fail_closed(kwargs): + with pytest.raises(ValueError): + prepare_launch(BASE, request(**kwargs), "sglang") + + +def test_duplicate_base_option_cannot_be_removed_ambiguously(): + with pytest.raises(ValueError, match="exactly one"): + prepare_launch( + BASE + ["--enable-metrics"], + request(remove_args=["--enable-metrics"]), + "sglang", + ) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"env": {"PORT": "9999"}}, + {"unset_env": ["AGENTIC_TRACE_SOURCE"]}, + {"env": {"SGLANG_SIMULATE_ACC_LEN": "100"}}, + {"env": {"A-B": "x"}}, + {"env": {"SAFE": "x"}, "unset_env": ["SAFE"]}, + {"source_files": {"relative.py": "0" * 64}}, + {"absent_source_files": ["relative.py"]}, + {"executable": "python3"}, + {"replace_args": "false"}, + ], +) +def test_invalid_requests_fail_closed(kwargs): + with pytest.raises(ValueError): + request(**kwargs) + + +def test_server_env_is_scoped_and_executes_literal_values(): + code = "import json,os;print(json.dumps([os.getenv('TEST_LITERAL'),os.getenv('REMOVE_ME')]))" + original = dict(os.environ, REMOVE_ME="old") + command, evidence = prepare_launch( + [sys.executable, "-c", code], + request( + env={"TEST_LITERAL": "$(touch should-not-exist)"}, unset_env=["REMOVE_ME"] + ), + "sglang", + original, + ) + completed = subprocess.run( + command, env=original, check=True, capture_output=True, text=True + ) + assert json.loads(completed.stdout) == ["$(touch should-not-exist)", None] + assert original["REMOVE_ME"] == "old" + assert evidence["effective_env"] == { + "REMOVE_ME": None, + "TEST_LITERAL": "$(touch should-not-exist)", + } + + +def test_candidate_sources_are_verified_after_launcher_setup(tmp_path): + source = tmp_path / "framework kernel.py" + source.write_text("candidate\n") + deleted = tmp_path / "deleted.py" + overrides = request( + source_files={str(source): hashlib.sha256(source.read_bytes()).hexdigest()}, + absent_source_files=[str(deleted)], + ) + _, evidence = prepare_launch(BASE, overrides, "sglang") + assert evidence["source_files"] == overrides["source_files"] + source.write_text("installer overwrote candidate\n") + with pytest.raises(ValueError, match="changed before server launch"): + prepare_launch(BASE, overrides, "sglang") + source.write_text("candidate\n") + deleted.symlink_to(tmp_path / "nonexistent") + with pytest.raises(ValueError, match="reappeared"): + prepare_launch(BASE, overrides, "sglang") + + +def test_runtime_executable_override(tmp_path): + candidate = tmp_path / "python candidate" + candidate.symlink_to(sys.executable) + command, evidence = prepare_launch( + BASE, request(executable=str(candidate)), "sglang" + ) + assert command[0] == str(candidate) + assert evidence["resolved_executable"] == str(Path(sys.executable).resolve()) + + +def test_runtime_environment_records_final_controls_without_credentials(): + _, evidence = prepare_launch( + BASE, + request(env={"SGLANG_USE_AITER": "0"}, unset_env=["OMP_NUM_THREADS"]), + "sglang", + { + "PATH": os.environ["PATH"], + "SGLANG_USE_AITER": "1", + "OMP_NUM_THREADS": "4", + "VLLM_API_KEY": "private", + "SGLANG_SECRET": "private", + "UNRELATED": "ignored", + }, + ) + assert evidence["runtime_environment"] == { + "PATH": os.environ["PATH"], + "SGLANG_USE_AITER": "0", + } + + +def test_real_bash_hook_preserves_spaces_and_writes_verified_receipt(tmp_path): + work = tmp_path / "run with spaces" + work.mkdir() + overrides = request( + append_args=["--new", "value with spaces"], env={"CUSTOM_LITERAL": "$not_shell"} + ) + config = work / "request.json" + config.write_text(json.dumps(overrides)) + receipt = work / "receipt.json" + result = subprocess.run( + [ + "bash", + "-c", + 'set -e; source "$1"; shift; agentic_apply_server_launch sglang "$@"; printf "%s\\0" "${AGENTX_SERVER_COMMAND[@]}"', + "test", + str(ROOT / "benchmarks/benchmark_lib.sh"), + *BASE, + ], + env={ + **os.environ, + "AGENTX_LAUNCH_OVERRIDES_FILE": str(config), + "AGENTX_LAUNCH_OVERRIDES_SHA256": digest(overrides), + "AGENTX_SERVER_LAUNCH_FILE": str(receipt), + "PYTHONPYCACHEPREFIX": str(work / "pycache"), + }, + capture_output=True, + check=True, + ) + argv = result.stdout.decode().split("\0")[:-1] + assert argv == [ + "env", + "CUSTOM_LITERAL=$not_shell", + *BASE, + "--new", + "value with spaces", + ] + evidence = json.loads(receipt.read_text()) + assert evidence["effective_argv"] == BASE + ["--new", "value with spaces"] + + +def test_manifest_launchers_all_call_hook_before_recording_and_spawning(): + manifest = json.loads((ROOT / "configs/agentx-launchers.json").read_text()) + assert len(manifest["recipes"]) == 11 + for row in manifest["recipes"].values(): + path = ROOT / "benchmarks" / row["benchmark_script"] + script = path.read_text() + var = "SGLANG_CMD" if row["framework"] == "sglang" else "VLLM_CMD" + hook = f'agentic_apply_server_launch {row["framework"]} "${{{var}[@]}}"' + assert script.count(hook) == 1 + assert script.index(hook) < script.index(f'"${{{var}[@]}}" >') + assert row["launch_overrides_version"] == 1 + + +def test_every_launcher_mapping_matches_the_actual_recipe_identity(): + manifest = json.loads((ROOT / "configs/agentx-launchers.json").read_text()) + recipes = yaml.safe_load((ROOT / "configs/amd-master.yaml").read_text()) + runners = yaml.safe_load((ROOT / "configs/runners.yaml").read_text())["hardware"] + for name, row in manifest["recipes"].items(): + recipe = recipes[name] + assert row["model"] == recipe["model"] + assert row["precision"] == recipe["precision"] + assert row["framework"] == recipe["framework"] + assert recipe["runner"] in runners + assert ( + recipe["runner"].removeprefix("cluster:").split("-")[0] + == row["runner_type"] + ) + assert "agentic-coding" in recipe["scenarios"] + assert not recipe.get("multinode", False) + assert Path(row["benchmark_script"]).name.startswith( + recipe["model-prefix"] + "_" + ) + + +def test_supported_launchers_bash_syntax(): + major = subprocess.check_output( + ["bash", "-c", 'printf %s "${BASH_VERSINFO[0]}"'], text=True + ) + if int(major) < 4: + pytest.skip("The existing InferenceX launchers require Bash 4+ ([[ -v ]])") + manifest = json.loads((ROOT / "configs/agentx-launchers.json").read_text()) + for row in manifest["recipes"].values(): + subprocess.run( + ["bash", "-n", str(ROOT / "benchmarks" / row["benchmark_script"])], + check=True, + )