feat(agentx): adopt harness v1.0.6 scenario defaults (#3646)

* feat(agentx): adopt harness v1.0.6 scenario defaults

Signed-off-by: Cam Quilici <cjquilici@gmail.com>

* docs(agentx): explain default seed and override

Signed-off-by: Cam Quilici <cjquilici@gmail.com>

---------

Signed-off-by: Cam Quilici <cjquilici@gmail.com>

Author: cquil11

inferencex-e2e/benchmarks/benchmark_lib.sh

@@ -3262,15 +3262,12 @@ build_replay_cmd() {
         AIPERF_FAILED_REQUEST_THRESHOLD AIPERF_LIVE_FAILED_REQUEST_THRESHOLD \
         AIPERF_TRACE_IDLE_GAP_CAP_SECONDS
     check_env_vars \
-        AGENTIC_WARMUP_GRACE_PERIOD AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES \
+        AGENTIC_WARMUP_GRACE_PERIOD \
         AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS AIPERF_EXPERIMENTAL_FAST \
         AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID AIPERF_UNSAFE_OVERRIDE \
         AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING AIPERF_WARMUP_REQUESTS_PER_LANE
-    # Recorded assistant responses drive prompt construction by default;
-    # AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1 threads the live response
-    # back into the session instead. The scenario plugin locks --cache-bust
-    # first_turn_prefix and a 10s whole-system idle cap; the 300s per-trajectory
-    # cap below is ours. See utils/aiperf/docs/tutorials/agentx-mvp.md.
+    # The agentx preset supplies shared replay and runtime defaults. Recipe-owned
+    # duration, warmup, failure threshold, and trace cap remain explicit below.
     local result_dir="$1"
     local duration="$DURATION"
     local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE}"
@@ -3281,20 +3278,9 @@ build_replay_cmd() {
         warmup_requests_per_lane=1
     fi
 
-    export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES}"
-    # Dataset configuration takes 4-5 min on fast /tmp (B300) but reached 14 min
-    # on H200 when 14 parallel jobs hit aiperf's default 900s Configure Profiling
-    # timeout; 1800s absorbs that without touching the measurement window.
-    export AIPERF_DATASET_CONFIGURATION_TIMEOUT=1800
-    # aiperf requires SERVICE_PROFILE_CONFIGURE_TIMEOUT >= DATASET_CONFIGURATION_TIMEOUT.
-    export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800
-    # Headless realtime metrics are opt-in on AIPerf main.
-    export AIPERF_UI_REALTIME_METRICS_ENABLED=true
-    REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp"
+    REPLAY_CMD="$AIPERF_CLI profile --scenario agentx"
     REPLAY_CMD+=" --url ${AIPERF_SERVER_URL:-http://localhost:$PORT}"
     REPLAY_CMD+=" --endpoint /v1/chat/completions"
-    REPLAY_CMD+=" --endpoint-type chat"
-    REPLAY_CMD+=" --streaming"
     # SERVED_MODEL_NAME covers frontends that register the model under a wire
     # name (dynamo-trt serves "DeepSeek-V4-Pro" while $MODEL is the HF id);
     # a mismatch 404s at warmup.
@@ -3304,14 +3290,9 @@ build_replay_cmd() {
     REPLAY_CMD+=" --tokenizer $MODEL"
     REPLAY_CMD+=" --concurrency $CONC"
     REPLAY_CMD+=" --benchmark-duration $duration"
-    REPLAY_CMD+=" --stats-interval 30"
-    REPLAY_CMD+=" --random-seed 42"
     # Live abort threshold; recipes with correlated low-concurrency trajectories
     # may loosen it while AIPERF_FAILED_REQUEST_THRESHOLD stays the post-run gate.
     REPLAY_CMD+=" --failed-request-threshold $AIPERF_LIVE_FAILED_REQUEST_THRESHOLD"
-    # AIPerf clamps the start ratio so at least one profile turn follows warmup.
-    REPLAY_CMD+=" --trajectory-start-min-ratio 0.25"
-    REPLAY_CMD+=" --trajectory-start-max-ratio 0.75"
     # Extra one-token requests per lane after the t* snapshot primers; profiling
     # resumes from the resulting live state. Do not pass --burst-phase-starts:
     # the spread default preserves each lane's recorded phase-start offset.
@@ -3322,9 +3303,6 @@ build_replay_cmd() {
     # Maximum wait for warmup to drain, not a fixed sleep; saturation arms with a
     # larger in-flight set can raise AGENTIC_WARMUP_GRACE_PERIOD.
     REPLAY_CMD+=" --warmup-grace-period ${AGENTIC_WARMUP_GRACE_PERIOD}"
-    # Server usage fields for ISL/OSL instead of client-side tokenize; the
-    # per-record tokenization was pinning CPU on minimax-m2.5 at high concurrency.
-    REPLAY_CMD+=" --use-server-token-count"
     if [ -n "${AIPERF_EXTRA_INPUTS:-}" ]; then
         REPLAY_CMD+=" --extra-inputs $AIPERF_EXTRA_INPUTS"
     fi
@@ -3342,11 +3320,6 @@ build_replay_cmd() {
         # request; this is the router's inactivity lease, not an HTTP timeout.
         REPLAY_CMD+=" --dynamo-session-timeout-seconds ${AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS}"
     fi
-    # aiperf's GpuMetricTimeSeries freezes its schema on the first DCGM scrape
-    # and KeyErrors when an optional field (xid_errors, power_violation) first
-    # appears mid-run. The gpu_telemetry artifact is unused downstream; the
-    # Prometheus server-metrics path is unaffected.
-    REPLAY_CMD+=" --no-gpu-telemetry"
     # The dataset manager loads the tokenizer regardless of
     # --use-server-token-count, and Kimi checkpoints ship a custom tokenizer
     # that needs trust_remote_code. Benign for other models.
@@ -3357,13 +3330,6 @@ build_replay_cmd() {
     if [ -n "${MAX_MODEL_LEN:-}" ] && [ "$MAX_MODEL_LEN" != "0" ]; then
         REPLAY_CMD+=" --max-context-length $MAX_MODEL_LEN"
     fi
-    # Default is 100; the with-subagents corpus has 393 unique traces. The loader
-    # treats this as min(cap, available), see semianalysis_cc_traces_weka.py.
-    REPLAY_CMD+=" --num-dataset-entries 393"
-    # Per-second server-metrics slices feed the post-run plotter; matches
-    # kv-cache-tester's poll_interval=1.0 so metrics_plots.png is comparable.
-    # Without it aiperf emits only aggregates and the panels are flat lines.
-    REPLAY_CMD+=" --slice-duration 1.0"
     # Multi-node launchers pass every worker's Prometheus endpoint; AIPerf takes
     # several values after one --server-metrics flag and keeps endpoint_url per series.
     if [ -n "${AIPERF_SERVER_METRICS_URLS:-}" ]; then

inferencex-e2e/docs/agentx-standalone.md

@@ -11,12 +11,12 @@ Run the AgentX client against an already-running OpenAI-compatible server.
 ## Install
 
 With Git and [uv](https://docs.astral.sh/uv/getting-started/installation/) installed,
-clone the harness at the pinned commit. Its CLI is named `aiperf`.
+clone the harness at the pinned `agentx-v1.0.6` release commit. Its CLI is named `aiperf`.
 
 ```bash
 git clone --filter=blob:none --no-checkout https://github.com/SemiAnalysisAI/agentx-harness.git
 cd agentx-harness
-git checkout --detach 754356e9a39acc6cc6afb242d123bb57c3fb6f75
+git checkout --detach 89b21867872a5bbc4b0676bf5005c404da5e9f94
 uv venv --python 3.11 .venv-agentx
 uv pip install --python .venv-agentx/bin/python -e . 'datasets>=4.7.0'
 source .venv-agentx/bin/activate
@@ -33,33 +33,28 @@ DATASET="semianalysis_cc_traces_weka_062126"
 CONC=8
 OUTPUT_DIR="$PWD/results/agentx-c${CONC}-$(date -u +%Y%m%dT%H%M%SZ)"
 
-export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0
-export AIPERF_DATASET_CONFIGURATION_TIMEOUT=1800
-export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800
-export AIPERF_UI_REALTIME_METRICS_ENABLED=true
-export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
-
 mkdir -p "$OUTPUT_DIR"
 aiperf profile \
-  --scenario inferencex-agentx-mvp \
+  --scenario agentx \
   --url "$SERVER_URL" --endpoint /v1/chat/completions \
-  --endpoint-type chat --streaming \
   --model "$SERVED_MODEL_NAME" --tokenizer "$TOKENIZER" \
   --tokenizer-trust-remote-code \
-  --concurrency "$CONC" --benchmark-duration 3600 \
-  --stats-interval 30 --random-seed 42 \
-  --failed-request-threshold 0.10 \
-  --trajectory-start-min-ratio 0.25 --trajectory-start-max-ratio 0.75 \
-  --warmup-requests-per-lane 10 --warmup-grace-period 1800 \
-  --trace-idle-gap-cap-seconds 300 \
-  --use-server-token-count --no-gpu-telemetry \
-  --num-dataset-entries 393 --slice-duration 1.0 \
+  --concurrency "$CONC" \
   --public-dataset "$DATASET" \
   --output-artifact-dir "$OUTPUT_DIR/aiperf_artifacts" \
   2>&1 | tee "$OUTPUT_DIR/aiperf.log"
 ```
 
-This profiles for one hour, plus dataset preparation, warmup, and drain time.
+The `agentx` preset supplies a one-hour profile, ten warmup requests per lane,
+and shared reporting and runtime settings. Explicit CLI arguments and `AIPERF_*`
+environment variables override preset defaults. Model, tokenizer, dataset,
+concurrency, and output remain caller inputs; `--tokenizer-trust-remote-code` is
+needed only for tokenizers with custom code.
+
+The default random seed is **42**. Keep this seed when reproducing AgentX results,
+or pass `--random-seed 123` to use a different seed.
+
+Allow additional time for dataset preparation, warmup, and draining requests.
 To sweep, restart the server for every `CONC` value and use a fresh output
 directory. Do not flush caches or reuse a live server across concurrency points. Match the target
 recipe's duration and warmup settings when reproducing a result.

inferencex-e2e/docs/agentx-standalone_zh.md

@@ -11,12 +11,12 @@
 ## 安装
 
 安装 Git 和 [uv](https://docs.astral.sh/uv/getting-started/installation/) 后，
-克隆测试工具并检出锁定的提交。其 CLI 名为 `aiperf`。
+克隆测试工具并检出锁定的 `agentx-v1.0.6` 发布提交。其 CLI 名为 `aiperf`。
 
 ```bash
 git clone --filter=blob:none --no-checkout https://github.com/SemiAnalysisAI/agentx-harness.git
 cd agentx-harness
-git checkout --detach 754356e9a39acc6cc6afb242d123bb57c3fb6f75
+git checkout --detach 89b21867872a5bbc4b0676bf5005c404da5e9f94
 uv venv --python 3.11 .venv-agentx
 uv pip install --python .venv-agentx/bin/python -e . 'datasets>=4.7.0'
 source .venv-agentx/bin/activate
@@ -33,33 +33,25 @@ DATASET="semianalysis_cc_traces_weka_062126"
 CONC=8
 OUTPUT_DIR="$PWD/results/agentx-c${CONC}-$(date -u +%Y%m%dT%H%M%SZ)"
 
-export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0
-export AIPERF_DATASET_CONFIGURATION_TIMEOUT=1800
-export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800
-export AIPERF_UI_REALTIME_METRICS_ENABLED=true
-export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
-
 mkdir -p "$OUTPUT_DIR"
 aiperf profile \
-  --scenario inferencex-agentx-mvp \
+  --scenario agentx \
   --url "$SERVER_URL" --endpoint /v1/chat/completions \
-  --endpoint-type chat --streaming \
   --model "$SERVED_MODEL_NAME" --tokenizer "$TOKENIZER" \
   --tokenizer-trust-remote-code \
-  --concurrency "$CONC" --benchmark-duration 3600 \
-  --stats-interval 30 --random-seed 42 \
-  --failed-request-threshold 0.10 \
-  --trajectory-start-min-ratio 0.25 --trajectory-start-max-ratio 0.75 \
-  --warmup-requests-per-lane 10 --warmup-grace-period 1800 \
-  --trace-idle-gap-cap-seconds 300 \
-  --use-server-token-count --no-gpu-telemetry \
-  --num-dataset-entries 393 --slice-duration 1.0 \
+  --concurrency "$CONC" \
   --public-dataset "$DATASET" \
   --output-artifact-dir "$OUTPUT_DIR/aiperf_artifacts" \
   2>&1 | tee "$OUTPUT_DIR/aiperf.log"
 ```
 
-测量窗口为一小时，另需数据集准备、预热和请求排空时间。
+`agentx` 预设提供一小时测量窗口、每条 lane 十个预热请求，以及共享的报告和运行时设置。
+显式 CLI 参数和 `AIPERF_*` 环境变量优先于预设默认值。
+模型、tokenizer、数据集、并发和输出路径仍由调用方指定；只有 tokenizer 需要自定义代码时，才需使用 `--tokenizer-trust-remote-code`。
+
+默认随机种子为 **42**。复现 AgentX 结果时请保留该种子；也可以传入 `--random-seed 123` 使用其他种子。
+
+测量之外还需数据集准备、预热和请求排空时间。
 扫描并发时，每个 `CONC` 值都必须重启服务并使用新的输出目录。
 不要清空缓存或在不同并发点之间复用正在运行的服务。复现结果时，使用目标 recipe 的测量时长和预热设置。
 

inferencex-e2e/docs/eval-agentx-procedures.md

@@ -197,7 +197,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench
 
 [`install_agentic_deps()`](../benchmarks/benchmark_lib.sh) declares the AgentX client dependencies directly alongside the editable `utils/aiperf` install. It installs them into the isolated `AIPERF_RUNTIME_DIR` environment with the caller-supplied `AIPERF_PYTHON_VERSION`.
 
-AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L3255-L3259)).
+AgentX is AIPerf `agentx` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L3255-L3259)).
 
 Every AgentX throughput concurrency runs against a fresh server deployment. The matrix creates a separate job per point; replay clients reject multiple concurrency values. AgentX does not flush caches or reuse a running server for another point. Warmup and profiling for the same point share the deployment. This does not change fixed-sequence sweeps or graded-eval batching.
 
@@ -240,7 +240,7 @@ Treat fast results as bring-up evidence, never as a replacement for the canonica
 
 ## 8. Preserve trace and run provenance
 
-AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L3165-L3234), [replay semantics](../benchmarks/benchmark_lib.sh#L3236-L3366)).
+AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L3165-L3234), [replay semantics](../benchmarks/benchmark_lib.sh#L3236-L3366)).
 
 Capture orchestration provenance immediately:
 

inferencex-e2e/docs/eval-agentx-procedures_zh.md

@@ -194,7 +194,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \
 
 [`install_agentic_deps()`](../benchmarks/benchmark_lib.sh) 在安装可编辑模式的 `utils/aiperf` 时直接声明 AgentX client 所需的依赖，并使用调用方提供的 `AIPERF_PYTHON_VERSION` 将它们安装到隔离的 `AIPERF_RUNTIME_DIR` 环境中。
 
-AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L3255-L3259)）。
+AgentX 是 AIPerf `agentx` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L3255-L3259)）。
 
 每个 AgentX 吞吐量并发点都必须使用新启动的服务。矩阵为每个点生成独立作业，replay client 会拒绝多个并发值。AgentX 不清空缓存，也不复用正在运行的服务来测试另一个并发点。同一测试点的预热和正式测量共用服务。此规则不改变定长序列 sweep 或评分 eval 的批量执行行为。
 
@@ -237,7 +237,7 @@ Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。
 
 ## 8. 保留 trace 与运行 provenance
 
-AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L3165-L3234)、[replay 语义](../benchmarks/benchmark_lib.sh#L3236-L3366)）。
+AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L3165-L3234)、[replay 语义](../benchmarks/benchmark_lib.sh#L3236-L3366)）。
 
 立即记录 orchestration provenance：
 

inferencex-e2e/perf-changelog.yaml

@@ -9193,3 +9193,12 @@
   description:
     - "Retire the mi325x-amds runner pool and route MI325X workloads to the Barite mi325x-amd pool in MI325X-UBUNTU. Keep model recipes unchanged; generated host-DRAM budgets now use the replacement cluster's declared memory capacity."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3644
+
+- config-keys:
+    - "*"
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Bump AgentX Harness from agentx-v1.0.5 (754356e9a39acc6cc6afb242d123bb57c3fb6f75) to agentx-v1.0.6 (89b21867872a5bbc4b0676bf5005c404da5e9f94) and use --scenario agentx for shared replay and runtime defaults. Preserve recipe duration, warmup, failure threshold, trace cap, routing and corpus selection as explicit inputs."
+    - "The preset supplies a 900000 ms TCP user timeout when unset; explicit AIPERF environment settings take precedence over preset defaults."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3646