@@ -91,16 +91,20 @@ base:
       WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
 
-# One variant per point (c128 excluded for both TP2 and TP4). Engram tables
-# move to pinned host memory at TP2, where resident tables leave too little
-# KV. Graph capture covers 2x CONC x (1 + 5 drafts). From c32 up the prefill
-# chunk is capped at 8192: a chunk scheduled alongside a decode step stalls
-# every decoding sequence for the length of that chunk, so the cap bounds the
-# stall rather than buying KV room, and it stays above 1024 x 6 slots, so the
-# scheduler is never capped at the captured shape. c1 through c16 keep 16384.
-# Chunked-prefill steps are 0 of 596 engine steps at c1 and 5 of 1183 at c16,
-# against 38 of 111 at c64, so below c32 the cap only splits one prefill into
-# more chunks and costs TTFT without a decode tail to protect.
+# One variant per point, concurrency 1 through 192 on both TP arms. Engram
+# tables move to pinned host memory at TP2 always and at TP4 from c128, where
+# resident tables leave too little KV. Graph capture covers 2x CONC x (1 + 5
+# drafts), which is what the benchmark admits.
+#
+# The prefill chunk follows two separate rules. From c32 through c64 it is
+# capped at 8192 for decode latency: a chunk scheduled alongside a decode step
+# stalls every decoding sequence for the length of that chunk, so the cap
+# bounds the stall. c1 through c16 keep 16384, where chunked-prefill steps are
+# 0 of 596 engine steps at c1 and 5 of 1183 at c16, against 38 of 111 at c64,
+# so the cap would only split one prefill into more chunks and cost TTFT with
+# no decode tail to protect. At c128 and c192 the chunk instead buys KV room,
+# staying at 8192 on TP4 and dropping to 4096 on TP2. Only those TP2 points
+# fall below 1024 x 6 token slots, so only they set max-num-seqs explicitly.
 override_tp4_c1:
   roles:
     agg:
@@ -192,6 +196,35 @@ override_tp4_c64:
     env:
       CONC: '64'
 
+# c128 and c192 offload the Engram tables even at TP=4: resident leaves
+# 116K KV tokens per concurrent request at c128, below the 122K at which
+# TP=2 c64 collapsed. Offloaded, c128 measured 388K.
+override_tp4_c128:
+  roles:
+    agg:
+      gpus: 4
+      args:
+        tensor-parallel-size: 4
+        engram-config: '{"cpu_offload":true}'
+        max-cudagraph-capture-size: 2048
+        max-num-batched-tokens: 8192
+  benchmark:
+    env:
+      CONC: '128'
+
+override_tp4_c192:
+  roles:
+    agg:
+      gpus: 4
+      args:
+        tensor-parallel-size: 4
+        engram-config: '{"cpu_offload":true}'
+        max-cudagraph-capture-size: 2304
+        max-num-batched-tokens: 8192
+  benchmark:
+    env:
+      CONC: '192'
+
 override_tp2_c1:
   roles:
     agg:
@@ -282,3 +315,35 @@ override_tp2_c64:
   benchmark:
     env:
       CONC: '64'
+
+# TP=2 buys KV room at c128 and c192 with a 4096 chunk, which measured 260K KV
+# tokens per concurrent request at c128 against 23.6K at the upstream 16384.
+# 4096 is below 1024 x 6 slots, so max-num-seqs drops to the 2x CONC the
+# benchmark actually admits.
+override_tp2_c128:
+  roles:
+    agg:
+      gpus: 2
+      args:
+        tensor-parallel-size: 2
+        engram-config: '{"cpu_offload":true}'
+        max-cudagraph-capture-size: 2048
+        max-num-batched-tokens: 4096
+        max-num-seqs: 256
+  benchmark:
+    env:
+      CONC: '128'
+
+override_tp2_c192:
+  roles:
+    agg:
+      gpus: 2
+      args:
+        tensor-parallel-size: 2
+        engram-config: '{"cpu_offload":true}'
+        max-cudagraph-capture-size: 2304
+        max-num-batched-tokens: 4096
+        max-num-seqs: 384
+  benchmark:
+    env:
+      CONC: '192'
@@ -1508,16 +1508,16 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark:
     agentic-coding:
     - dram-utilization: 0.60
       search-space:
-      # The prefill chunk is capped at 8192 from c32 up, bounding the decode
+      # The prefill chunk is capped at 8192 at c32 and c64, bounding the decode
       # stall a chunk causes when it shares an engine step with a decode batch;
       # c1-c16 keep 16384, where such mixed steps are too rare to pay for the
-      # longer prefill. The recipe otherwise changes the memory split only where
-      # a point ran short of KV. c128 is excluded for both TP=2 and TP=4. Every
-      # point is measured here rather than combined from the earlier run: the
-      # image move leaves concurrency 1-32 measured only on the superseded
-      # nightly-eed1f3d0 pin, and the chunk change invalidates c32 and c64.
-      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml }
-      - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml }
+      # longer prefill. c128 and c192 size the chunk for KV room instead, and
+      # offload the Engram tables on TP=4 as well. Every point is measured here
+      # rather than combined from the earlier run: the image move leaves
+      # concurrency 1-32 measured only on the superseded nightly-eed1f3d0 pin,
+      # the chunk change invalidates c32 and c64, and c128 and c192 are new.
+      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 192], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml }
+      - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 192], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml }
 
 # The golden acceptance curve remains the provisional GLM-5.2-derived curve
 # documented in infx/golden_al_distribution/glm5.3_mtp.yaml.
 
@@ -695,9 +695,13 @@ A configuration is ready for sweep only when the executable files agree, the exa
 
 ## DeepSeek-V4.1-Flash on MI355X
 
-The `dsv41flash-fp4-mi355x-vllm-agentic-dspark` recipe extends [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) to MI355X AgentX: TP4 and TP2, concurrency 1–64, native five-token DSpark. Throughput uses the [committed golden AL](../infx/golden_al_distribution/dsv41flash_dspark.yaml) of 3.51 for thinking on and five draft tokens, with synthetic rejection sampling and adaptive verification disabled. Accuracy evals retain real block rejection but, unlike the CUDA arms, also keep adaptive verification disabled: it trims verification requests on device, which the ROCm `DeepseekV4IndexerBackend` does not support, and the engine refused to start with it enabled ([run 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)). FP4 describes the MXFP4 experts; the checkpoint also contains MXFP8 weights.
+The `dsv41flash-fp4-mi355x-vllm-agentic-dspark` recipe extends [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) to MI355X AgentX: TP4 and TP2, concurrency 1–192, native five-token DSpark. Throughput uses the [committed golden AL](../infx/golden_al_distribution/dsv41flash_dspark.yaml) of 3.51 for thinking on and five draft tokens, with synthetic rejection sampling and adaptive verification disabled. Accuracy evals retain real block rejection but, unlike the CUDA arms, also keep adaptive verification disabled: it trims verification requests on device, which the ROCm `DeepseekV4IndexerBackend` does not support, and the engine refused to start with it enabled ([run 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)). FP4 describes the MXFP4 experts; the checkpoint also contains MXFP8 weights.
 
-Follow the AMD overrides in the merged [upstream recipe #968](https://github.com/vllm-project/recipes/pull/968): `VLLM_ROCM_USE_AITER=1`, `VLLM_ROCM_USE_AITER_MOE=1`, and `--moe-backend aiter`. The generic AITER selector uses CK experts; `VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1` opts into the DeepSeek-V4.1 a4w4 path from [vllm-project/vllm#58819](https://github.com/vllm-project/vllm/pull/58819). This is a process-wide opt-in for MXFP4 MoE experts, including DSpark layers that use the same implementation. `VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4` additionally routes the tensor-parallel all-reduce through the INT4 quick-reduce codec, as in [upstream recipe #1061](https://github.com/vllm-project/recipes/pull/1061) and the merged MI355X DSV4-Pro and Kimi-K3 vLLM recipes. The recipe pins `semianalysis_cc_traces_weka_062126` (the unfiltered corpus) via `WEKA_LOADER_OVERRIDE`. KV stays GPU-resident. Engram stayed on GPU under the upstream AMD defaults until [vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491) widened the two `is_cuda()` gates to `is_cuda_alike()`. From that commit on, ROCm resolves an `EngramConfig` and `cpu_offload` defaults to on through `VLLM_PLE_CPU_OFFLOAD`, so the recipe sets `--engram-config` explicitly rather than leaning on that default. TP=2 always offloads, since the tables need 94.4 GiB per rank there; TP=4 keeps them resident, since its KV pool is not the constraint through concurrency 64. `--max-num-batched-tokens` is 8192 from concurrency 32 up and 16384 below that. The cap is for decode latency, not memory: a prefill chunk scheduled into the same engine step as a decode batch stalls every decoding sequence for the length of that chunk, so the stall scales with the chunk. It therefore only pays where such mixed steps exist, which is a function of concurrency — Torch profiles of the TP2 arm count 0 chunked-prefill steps in 596 engine steps at c1 and 5 in 1183 at c16, against 38 in 111 at c64. A 14-point A/B that capped every point ([run 37375212446](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37375212446) against the 16384 baseline [run 37080504692](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37080504692)) measured that split: TP2 c32 ITL p90 fell 13.19 → 11.29 ms and TP4 c64 fell 19.76 → 14.26 ms, while concurrency 1–16 gained no interactivity and repeatedly lost TTFT (TP4 c4 p90 364 → 471 ms, TP2 c4 505 → 600 ms). Memory is a second reason to keep the cap at the top of the ladder, since the sparse-attention indexer and its companion per-rank buffers grow at roughly 4.4 MiB per batched token. Do not drop it below six times the API-server default of 1024 sequences, or `--max-num-seqs` is capped at the graph-capture shape: DSpark verifies 1+5 tokens per sequence, and at 4096 against 1024 sequences the engram projection faults during profiling. Engram placement follows the same rule of spending device memory on KV only at the concurrencies that ran short of it. Images built before that merge still reject the option on ROCm. The MI355X launcher uses the shared HF cache and mounts this model's repository at `/ix`, and exports `INFMAX_CONTAINER_WORKSPACE=/ix` so AgentX dependencies and outputs resolve inside that mount.
+Follow the AMD overrides in the merged [upstream recipe #968](https://github.com/vllm-project/recipes/pull/968): `VLLM_ROCM_USE_AITER=1`, `VLLM_ROCM_USE_AITER_MOE=1`, and `--moe-backend aiter`. The generic AITER selector uses CK experts; `VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1` opts into the DeepSeek-V4.1 a4w4 path from [vllm-project/vllm#58819](https://github.com/vllm-project/vllm/pull/58819). This is a process-wide opt-in for MXFP4 MoE experts, including DSpark layers that use the same implementation. `VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4` additionally routes the tensor-parallel all-reduce through the INT4 quick-reduce codec, as in [upstream recipe #1061](https://github.com/vllm-project/recipes/pull/1061) and the merged MI355X DSV4-Pro and Kimi-K3 vLLM recipes. The recipe pins `semianalysis_cc_traces_weka_062126` (the unfiltered corpus) via `WEKA_LOADER_OVERRIDE`. KV stays GPU-resident. Engram stayed on GPU under the upstream AMD defaults until [vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491) widened the two `is_cuda()` gates to `is_cuda_alike()`. From that commit on, ROCm resolves an `EngramConfig` and `cpu_offload` defaults to on through `VLLM_PLE_CPU_OFFLOAD`, so the recipe sets `--engram-config` explicitly rather than leaning on that default. TP=2 always offloads, since the tables need 94.4 GiB per rank there; TP=4 keeps them resident through concurrency 64, since its KV pool is not the constraint there, and offloads at c128 and c192, where resident tables leave 116K KV tokens per concurrent request, below the 122K at which TP=2 c64 collapsed.
+
+`--max-num-batched-tokens` follows two separate rules. At concurrency 32 and 64 it is 8192 and below that 16384, sized for decode latency. At c128 and c192 it is sized for KV room instead: 8192 on TP=4, which measured 388K KV tokens per concurrent request at c128, and 4096 on TP=2, which measured 260K against 23.6K at the upstream 16384. Only those two TP=2 points fall below 1024 × 6 token slots, so only they set `--max-num-seqs` explicitly, to the 2 × CONC the benchmark admits. Graph capture covers that same 2 × CONC × (1 + 5 drafts) batch, rising from 1024 through c64 to 2048 at c128 and 2304 at c192. c192 is unmeasured on this SKU; scaling the c128 totals gives roughly 259K KV tokens per request at TP=4 and 173K at TP=2, the latter only 1.4× above the collapse line.
+
+The latency rule is the one the A/B below measures: a prefill chunk scheduled into the same engine step as a decode batch stalls every decoding sequence for the length of that chunk, so the stall scales with the chunk. It therefore only pays where such mixed steps exist, which is a function of concurrency — Torch profiles of the TP2 arm count 0 chunked-prefill steps in 596 engine steps at c1 and 5 in 1183 at c16, against 38 in 111 at c64. A 14-point A/B that capped every point ([run 37375212446](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37375212446) against the 16384 baseline [run 37080504692](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37080504692)) measured that split: TP2 c32 ITL p90 fell 13.19 → 11.29 ms and TP4 c64 fell 19.76 → 14.26 ms, while concurrency 1–16 gained no interactivity and repeatedly lost TTFT (TP4 c4 p90 364 → 471 ms, TP2 c4 505 → 600 ms). Memory is a second reason to keep the cap at the top of the ladder, since the sparse-attention indexer and its companion per-rank buffers grow at roughly 4.4 MiB per batched token. Dropping the chunk below six times the API-server default of 1024 sequences also forces `--max-num-seqs` down with it, because DSpark verifies 1+5 tokens per sequence and at 4096 against 1024 sequences the engram projection faults during profiling; the TP=2 c128 and c192 points above are the only ones in that regime. Engram placement follows the same rule of spending device memory on KV only at the concurrencies that ran short of it. Images built before that merge still reject the option on ROCm. The MI355X launcher uses the shared HF cache and mounts this model's repository at `/ix`, and exports `INFMAX_CONTAINER_WORKSPACE=/ix` so AgentX dependencies and outputs resolve inside that mount.
 
 The current candidate pins `vllm/vllm-openai-rocm:nightly-rocm100-21d93d0d8c0e9627900020382bfce4730e61cab7`, the ROCm 10.0 nightly published 2026-10-06, with a4w4 and INT4 quick reduce enabled. It sits 151 commits past `nightly-rocm100-0cbac6cd1305f710e12193596b27488397bcb205` and adds [vllm#56720](https://github.com/vllm-project/vllm/pull/56720) (one-pass dequantize and a gather-sized grid for the compressed K cache), [vllm#57947](https://github.com/vllm-project/vllm/pull/57947) (fused QSA pre-indexer on the AMD path), [vllm#58560](https://github.com/vllm-project/vllm/pull/58560) (compressor ring kept out of the null block) and [vllm#59794](https://github.com/vllm-project/vllm/pull/59794) (AITER 0.1.24.post1, which supplies the a4w4 and quick-reduce kernels). Its GPU sweep and accuracy evaluation must qualify this image/flag combination; the evidence below describes the preceding a8w4, unquantized-collective baseline.