From bcc71ef67ee28a55cb539eb0a98906edeffd7ad0 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Wed, 30 Sep 2026 00:57:30 -0700 Subject: [PATCH 1/2] perf: update B300 DSV4 DSpark configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新 B300 DSV4 DSpark 配置，启用 DP 推测解码与预填充协调，保留 TP4/TP8 低延迟测试点。 --- .../dsv4/sglang/b300-fp4-mtp/agentic.yaml | 142 ++++++++++++++---- inferencex-e2e/configs/nvidia-master.yaml | 17 +-- inferencex-e2e/perf-changelog.yaml | 8 + 3 files changed, 124 insertions(+), 43 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml index 3beb2ef09c..157c7013ab 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -1,12 +1,10 @@ -# DeepSeek-V4-Pro-0813 AgentX on B300 with SGLang DSpark (block 6). Two arms: -# TP8 with the flashinfer MXFP4 MoE and GPU-resident KV, and DEP8 (attention -# DP + Mega-MoE + FP4 indexer) behind the SGLang router with a HiCache DRAM tier. +# Update B300 DSV4 performance with DSpark. base: schema: 2 name: dsv4-fp4-b300-sglang-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + container: lmsysorg/sglang:dev-cu13-nightly-0929@sha256:497d6ce4887830aad5382242b79c5cfc15faefa0bbb970b4580041d8b131775b precision: fp4 resources: gpu_type: b300 @@ -43,7 +41,7 @@ base: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - attention-backend: compressed + attention-backend: dsv4 page-size: 256 disable-shared-experts-fusion: true enable-metrics: true @@ -72,25 +70,112 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -# One variant per point. TP8: admission is 2x CONC and the graph batch 4x -# CONC, capped at 64. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 4 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + override_tp8_c1: roles: agg: args: max-running-requests: 2 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 benchmark: env: CONC: '1' KV_OFFLOADING: none +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + override_tp8_c4: roles: agg: args: max-running-requests: 8 - cuda-graph-max-bs: 16 + cuda-graph-max-bs-decode: 16 benchmark: env: CONC: '4' @@ -101,7 +186,7 @@ override_tp8_c8: agg: args: max-running-requests: 16 - cuda-graph-max-bs: 32 + cuda-graph-max-bs-decode: 32 benchmark: env: CONC: '8' @@ -112,7 +197,7 @@ override_tp8_c16: agg: args: max-running-requests: 32 - cuda-graph-max-bs: 64 + cuda-graph-max-bs-decode: 64 benchmark: env: CONC: '16' @@ -123,18 +208,12 @@ override_tp8_c32: agg: args: max-running-requests: 64 - cuda-graph-max-bs: 64 + cuda-graph-max-bs-decode: 64 benchmark: env: CONC: '32' KV_OFFLOADING: none -# DEP8 replaces the TP8 MoE path: the router keeps each session on the DP rank -# holding its radix prefix, the global prefill chunk is 8192 per rank, decode -# graphs cover the padded speculative batch across ranks, and AIPerf owns the -# warmup. HiCache capacity is a host/device ratio; 3 keeps the tier near 2 TB. -# Mega-MoE's transient workspace sits outside the static pool, so the static -# fraction steps down with CONC: 0.90 to c256, 0.88 at c384, 0.86 from c512. override_dep8_c32: frontend: &dep8_frontend type: sglang-router @@ -145,6 +224,7 @@ override_dep8_c32: connect-timeout-secs: 900 request-timeout-secs: 14400 disable-health-check: true + disable-circuit-breaker: true # A transient router-to-engine send failure would otherwise abort the run. retry-max-retries: 8 retry-initial-backoff-ms: 500 @@ -158,7 +238,9 @@ override_dep8_c32: data-parallel-size: 8 tokenizer-worker-num: 8 enable-prefill-delayer: true - prefill-decode-interval: 20 + prefill-decode-interval: 32 + enable-mixed-chunk: true + schedule-policy: shortest-prefill-first enable-dp-attention: true enable-dp-lm-head: true enable-dp-attention-local-control-broadcast: true @@ -172,14 +254,16 @@ override_dep8_c32: cuda-graph-max-bs-decode: 544 swa-full-tokens-ratio: 0.075 enable-hierarchical-cache: true - hicache-ratio: 3 + hicache-ratio: 2 hicache-write-policy: write_back hicache-io-backend: direct hicache-mem-layout: page_first_direct skip-server-warmup: true max-running-requests: 64 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: &dep8_env + SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION: '1' + SGLANG_RAGGED_VERIFY_MODE: static # Covers the 8192-token per-rank prefill budget. SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' benchmark: @@ -196,7 +280,7 @@ override_dep8_c64: args: <<: *dep8_args max-running-requests: 128 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -210,7 +294,7 @@ override_dep8_c128: args: <<: *dep8_args max-running-requests: 256 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -224,7 +308,7 @@ override_dep8_c256: args: <<: *dep8_args max-running-requests: 512 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -238,7 +322,7 @@ override_dep8_c384: args: <<: *dep8_args max-running-requests: 768 - mem-fraction-static: 0.88 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -251,8 +335,8 @@ override_dep8_c512: agg: args: <<: *dep8_args - max-running-requests: 1024 - mem-fraction-static: 0.86 + max-running-requests: 768 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -265,8 +349,8 @@ override_dep8_c576: agg: args: <<: *dep8_args - max-running-requests: 1152 - mem-fraction-static: 0.86 + max-running-requests: 768 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 07c47e6e40..0ee3fbc78b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -1119,7 +1119,7 @@ dsr1-fp8-b300-sglang: srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp8/8k1k.yaml dsv4-fp4-b300-sglang-agentic-hicache-mtp: - image: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + image: lmsysorg/sglang:dev-cu13-nightly-0929@sha256:497d6ce4887830aad5382242b79c5cfc15faefa0bbb970b4580041d8b131775b model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b300-dsxe @@ -1130,21 +1130,10 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } - # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is - # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by - # DP_ATTENTION: - # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 - # + mem-fraction 0.88 + swa-full-tokens-ratio 0.1 - # dp-attn: true -> DP-attn + megamoe + fp4 indexer - # + chunked-prefill 65536 + mem-fraction 0.90 - # + swa-full-tokens-ratio 0.075 - # + prefill-decode-interval 20 - # Both paths share DSpark K=6 (1 step, 7 draft tokens) and - # max-running-requests 2*CONC. - dsv4-fp4-b300-sglang-agentic-hicache-eagle: image: lmsysorg/sglang:dev-cu13-nightly-09242@sha256:c47bbe7448050608b4e08669c742df4835a45a95bf6b19f2c4e51a7f6609e9d5 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2b184929a0..27b59b1e96 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9127,3 +9127,11 @@ - "Add Kimi-K3 MXFP4 vLLM agentic-coding on MI355X (TP8, DSpark): dcp1 c1/c4 GPU-resident and c8-c14 with SimpleCPUOffload DRAM offload, plus a new dcp8 c44/c48/c70 throughput band (mtp synthetic acceptance, no draft); vLLM ROCm image vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89" - "The Inferact/Kimi-K3-DSpark draft (K3DSparkModel, model_type k3_dspark, 5 layers, hidden 7168, torch_dtype bfloat16, no quantization_config) keeps every layer at its pristine dtype. vLLM builds all its modules with quant_config from get_draft_quant_config() (vllm/models/kimi_k3/nvidia/dspark_mla.py), which returns None because K3DSparkModel is explicitly excluded from the DeepSeek-V4 branch that would set draft.quantization = target.quantization (vllm/config/speculative.py:1431); so context_proj (ReplicatedLinear), context_kv_proj (MergedColumnParallelLinear) and each decoder layer's MLA q/kv projections and dense KimiMLP (gate/up/down) load unquantized in BF16 -- no ptpc_fp8, no mxfp4, no INT4 weights. The draft has no FusedMoE/block_sparse_moe, so VLLM_ROCM_USE_AITER_MOE_SITUV2 (A8W4) never touches it, and its KV cache stays fp8 (kv_cache_dtype). This PR only changes draft depth (num_speculative_tokens 4->7) and enables INT4 custom quick all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, vllm/distributed/device_communicators/quick_all_reduce.py); because the draft shares the target's TP8 group (vllm/v1/spec_decode/draft_model.py), that INT4 applies to its tensor-parallel reductions too -- a collective-reduction transport precision, not any draft weight, activation, or KV dtype." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3561 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update B300 DSV4 DSpark performance with DP spec/prefill coordination, mixed 64K chunks and interval 32; retain TP4/TP8 low-latency coverage." + - "更新 B300 DSV4 DSpark 性能，启用 DP 推测解码与预填充协调、64K 混合分块和间隔 32；保留 TP4/TP8 低延迟测试点。" From a7921562b15ee317142f44805c996738736349d8 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Wed, 30 Sep 2026 01:04:00 -0700 Subject: [PATCH 2/2] fix: add DSpark submission changelog PR link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 DSpark 提交的变更日志 PR 链接，修复矩阵规划校验失败。 --- inferencex-e2e/perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 27b59b1e96..46ff3d2323 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9135,3 +9135,4 @@ description: - "Update B300 DSV4 DSpark performance with DP spec/prefill coordination, mixed 64K chunks and interval 32; retain TP4/TP8 low-latency coverage." - "更新 B300 DSV4 DSpark 性能，启用 DP 推测解码与预填充协调、64K 混合分块和间隔 32；保留 TP4/TP8 低延迟测试点。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3606