diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml index 47d2f7616e..49eb44b0fd 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml @@ -1,13 +1,14 @@ # DeepSeek-V4-Pro-0813 AgentX on MI355X with vLLM DSpark K6 # (https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x). Pure # TP8 through c64; the wide band runs DP attention (DEP8) behind the vLLM -# Router. The KV cache is GPU-resident. +# Router, both with the AITER MoE (EP1) and with AITER MegaMoE (EP8). The KV +# cache is GPU-resident. base: schema: 2 name: dsv4-fp4-mi355x-vllm-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 + container: vllm/vllm-openai-rocm:nightly-fe4df3df0eb703862c686b256e9fe44fa94c8945 precision: fp4 resources: gpu_type: mi355x @@ -229,3 +230,143 @@ override_dep8_c192: env: CONC: '192' AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +# DEP8 with AITER MegaMoE: fused EP8 dispatch/GEMM/combine on the MoRI +# symmetric heap, which vLLM sizes itself. MegaMoE requires TP1, expert +# parallelism, a MoRI all2all backend and no EPLB. Its shared-expert fusion +# would requantize the FP8 shared experts of the DSpark draft layers to +# MXFP4, so it stays off to serve the draft as shipped. +override_dep8_megamoe_c32: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: mori_high_throughput + moe-backend: aiter_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 32 + env: + VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: '0' + benchmark: + env: + CONC: '32' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_megamoe_c64: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: mori_high_throughput + moe-backend: aiter_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 64 + env: + VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: '0' + benchmark: + env: + CONC: '64' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_megamoe_c128: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: mori_high_throughput + moe-backend: aiter_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 128 + env: + VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: '0' + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_megamoe_c192: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: mori_high_throughput + moe-backend: aiter_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 192 + env: + VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: '0' + benchmark: + env: + CONC: '192' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_dep8_megamoe_c256: + frontend: + type: vllm-router + args: + policy: consistent_hash + request-timeout-secs: 14400 + disable-retries: true + env: + SETUP_PIP_PACKAGES: 'vllm-router==0.1.14' + roles: + agg: + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: mori_high_throughput + moe-backend: aiter_mega_moe + prefill-schedule-interval: 8 + long-prefill-token-threshold: 16384 + max-num-seqs: 256 + env: + VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS: '0' + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 48a89330eb..7c9bff1d94 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -924,7 +924,7 @@ qwen3.5-fp8-mi325x-sglang-mtp: # DeepSeek-V4-Pro-0813 FP4 AgentX on one MI355X node using vLLM DSpark K6. # Keep the historical config key for changelog compatibility. dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 + image: vllm/vllm-openai-rocm:nightly-fe4df3df0eb703862c686b256e9fe44fa94c8945 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:mi355x-amds @@ -939,6 +939,9 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # Single-node validation found c128 to be the DPA throughput peak among # c64/c128/c256. Add c192 to locate the knee before the c256 regression. - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } + # DEP8 with AITER MegaMoE (EP8) beside the EP1 DPA band; the Pareto + # frontier keeps whichever MoE wins at each concurrency. + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [32, 64, 128, 192, 256], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml } # LMCache invalid-block recovery currently assumes one KV-cache group, # while speculative decoding creates two. Restore these points after the # upstream hybrid KV recovery fix lands: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 17ff00f039..9661a17d7a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9646,3 +9646,14 @@ - "ATOM's mono decode path (on by default) now takes any decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, not only DSpark verify steps. Steps of 12 or more rows that are a multiple of TP run its sublayers token-parallel, with reduce-scatters instead of all-reduces. The draft runner never runs token-parallel." - "No data-type or precision change to the DSpark draft. The checkpoint's bundled mtp.0-2 stages are stored like the target's layers: FP8 E4M3 weights with 32x32 UE8M0 block scales for the attention projections, shared experts and main_proj; MXFP4 routed experts; BF16 norms, router gate, Markov head and confidence head; F32 attention sinks and hc mixers. ATOM builds the draft with the target's native_quant_config() and make_v4_quant_config(), with no online quantization in this recipe, so it runs as shipped: FP8 activations on the FP8 GEMMs and the target's BF16 KV cache. Between 73defd9b8 and a0527c9db only the rows the draft reads under bounded replay change." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3928 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 to vllm/vllm-openai-rocm:nightly-fe4df3df0eb703862c686b256e9fe44fa94c8945, the latest published ROCm nightly, which includes the AITER MegaMoEV2 integration for DeepSeek-V4 (vllm-project/vllm#59685), and update model.container to match." + - "Add a DEP8 band with AITER MegaMoE at concurrency 32, 64, 128, 192 and 256: data-parallel-size 8, tensor-parallel-size 1, --enable-expert-parallel, --all2all-backend mori_high_throughput and --moe-backend aiter_mega_moe, behind the same vLLM Router consistent-hash routing, prefill interleaving and max-num-seqs = concurrency as the existing DEP8 band. vLLM sizes the MoRI symmetric heap for MegaMoE (8G)." + - "The TP8 band (concurrency 1-64) and the EP1 DEP8 band (concurrency 64, 128, 192) keep their settings on the new image, so the frontier can pick the better MoE at each overlapping concurrency." + - "No weight-precision change to the DSpark draft: VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS stays 0, because the shared-expert fusion requantizes the FP8 shared experts of the draft layers to MXFP4. Without it, the routed experts of the target and the bundled DeepSeek-V4-Pro-0813 draft keep their shipped MXFP4 weights and the FP8 shared experts run on their regular path. Like the DeepGEMM MegaMoE backends of the B200/B300 vLLM AgentX configs, the MegaMoE band runs the routed experts with FP8 activations (a8w4) where the default AITER MoE uses BF16; the TP8 and EP1 DEP8 bands keep the default." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3968