diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml index 3beb2ef09c..684a200445 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -1,12 +1,10 @@ -# DeepSeek-V4-Pro-0813 AgentX on B300 with SGLang DSpark (block 6). Two arms: -# TP8 with the flashinfer MXFP4 MoE and GPU-resident KV, and DEP8 (attention -# DP + Mega-MoE + FP4 indexer) behind the SGLang router with a HiCache DRAM tier. +# Update B300 DSV4 performance with DSpark. base: schema: 2 name: dsv4-fp4-b300-sglang-agentic model: path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + container: lmsysorg/sglang:nightly-dev-cu13-20261008-7760d526@sha256:f991a9648fc0a00410bab27d96bf96a0263be9d9b178fe01ee95cffe622fd283 precision: fp4 resources: gpu_type: b300 @@ -43,7 +41,7 @@ base: speculative-num-steps: 1 speculative-eagle-topk: 1 speculative-num-draft-tokens: 7 - attention-backend: compressed + attention-backend: dsv4 page-size: 256 disable-shared-experts-fusion: true enable-metrics: true @@ -72,25 +70,112 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -# One variant per point. TP8: admission is 2x CONC and the graph batch 4x -# CONC, capped at 64. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 2 + cuda-graph-max-bs-decode: 4 + benchmark: + env: + CONC: '1' + KV_OFFLOADING: none + +override_tp4_c2: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 4 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + +override_tp4_c4: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 8 + cuda-graph-max-bs-decode: 16 + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none + +override_tp4_c8: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 16 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '8' + KV_OFFLOADING: none + +override_tp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-running-requests: 64 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '32' + KV_OFFLOADING: none + override_tp8_c1: roles: agg: args: max-running-requests: 2 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 benchmark: env: CONC: '1' KV_OFFLOADING: none +override_tp8_c2: + roles: + agg: + args: + max-running-requests: 4 + cuda-graph-max-bs-decode: 8 + benchmark: + env: + CONC: '2' + KV_OFFLOADING: none + override_tp8_c4: roles: agg: args: max-running-requests: 8 - cuda-graph-max-bs: 16 + cuda-graph-max-bs-decode: 16 benchmark: env: CONC: '4' @@ -101,7 +186,7 @@ override_tp8_c8: agg: args: max-running-requests: 16 - cuda-graph-max-bs: 32 + cuda-graph-max-bs-decode: 32 benchmark: env: CONC: '8' @@ -112,7 +197,7 @@ override_tp8_c16: agg: args: max-running-requests: 32 - cuda-graph-max-bs: 64 + cuda-graph-max-bs-decode: 64 benchmark: env: CONC: '16' @@ -123,18 +208,12 @@ override_tp8_c32: agg: args: max-running-requests: 64 - cuda-graph-max-bs: 64 + cuda-graph-max-bs-decode: 64 benchmark: env: CONC: '32' KV_OFFLOADING: none -# DEP8 replaces the TP8 MoE path: the router keeps each session on the DP rank -# holding its radix prefix, the global prefill chunk is 8192 per rank, decode -# graphs cover the padded speculative batch across ranks, and AIPerf owns the -# warmup. HiCache capacity is a host/device ratio; 3 keeps the tier near 2 TB. -# Mega-MoE's transient workspace sits outside the static pool, so the static -# fraction steps down with CONC: 0.90 to c256, 0.88 at c384, 0.86 from c512. override_dep8_c32: frontend: &dep8_frontend type: sglang-router @@ -145,6 +224,7 @@ override_dep8_c32: connect-timeout-secs: 900 request-timeout-secs: 14400 disable-health-check: true + disable-circuit-breaker: true # A transient router-to-engine send failure would otherwise abort the run. retry-max-retries: 8 retry-initial-backoff-ms: 500 @@ -158,7 +238,9 @@ override_dep8_c32: data-parallel-size: 8 tokenizer-worker-num: 8 enable-prefill-delayer: true - prefill-decode-interval: 20 + prefill-decode-interval: 32 + enable-mixed-chunk: true + schedule-policy: shortest-prefill-first enable-dp-attention: true enable-dp-lm-head: true enable-dp-attention-local-control-broadcast: true @@ -172,14 +254,16 @@ override_dep8_c32: cuda-graph-max-bs-decode: 544 swa-full-tokens-ratio: 0.075 enable-hierarchical-cache: true - hicache-ratio: 3 + hicache-ratio: 2 hicache-write-policy: write_back hicache-io-backend: direct hicache-mem-layout: page_first_direct skip-server-warmup: true max-running-requests: 64 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: &dep8_env + SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION: '1' + SGLANG_RAGGED_VERIFY_MODE: static # Covers the 8192-token per-rank prefill budget. SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8320' benchmark: @@ -196,7 +280,7 @@ override_dep8_c64: args: <<: *dep8_args max-running-requests: 128 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -210,7 +294,7 @@ override_dep8_c128: args: <<: *dep8_args max-running-requests: 256 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -224,7 +308,7 @@ override_dep8_c256: args: <<: *dep8_args max-running-requests: 512 - mem-fraction-static: 0.9 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -238,7 +322,7 @@ override_dep8_c384: args: <<: *dep8_args max-running-requests: 768 - mem-fraction-static: 0.88 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -251,8 +335,8 @@ override_dep8_c512: agg: args: <<: *dep8_args - max-running-requests: 1024 - mem-fraction-static: 0.86 + max-running-requests: 768 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: @@ -265,8 +349,8 @@ override_dep8_c576: agg: args: <<: *dep8_args - max-running-requests: 1152 - mem-fraction-static: 0.86 + max-running-requests: 768 + mem-fraction-static: 0.84 env: *dep8_env benchmark: env: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d8eaafffb9..7b1e32f43d 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -1119,7 +1119,7 @@ dsr1-fp8-b300-sglang: srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/sglang/b300-fp8/8k1k.yaml dsv4-fp4-b300-sglang-agentic-hicache-mtp: - image: lmsysorg/sglang:nightly-dev-20260901-07c8f729 + image: lmsysorg/sglang:nightly-dev-cu13-20261008-7760d526@sha256:f991a9648fc0a00410bab27d96bf96a0263be9d9b178fe01ee95cffe622fd283 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b300-dsxe @@ -1130,21 +1130,10 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.95 search-space: - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [32, 64, 128, 256, 384, 512, 576], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml } - # DeepSeek-V4-Pro-0813 on B300 with DSpark speculative decoding. Recipe is - # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by - # DP_ATTENTION: - # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 - # + mem-fraction 0.88 + swa-full-tokens-ratio 0.1 - # dp-attn: true -> DP-attn + megamoe + fp4 indexer - # + chunked-prefill 65536 + mem-fraction 0.90 - # + swa-full-tokens-ratio 0.075 - # + prefill-decode-interval 20 - # Both paths share DSpark K=6 (1 step, 7 draft tokens) and - # max-running-requests 2*CONC. - dsv4-fp4-b300-sglang-agentic-hicache-eagle: image: lmsysorg/sglang:dev-cu13-nightly-09242@sha256:c47bbe7448050608b4e08669c742df4835a45a95bf6b19f2c4e51a7f6609e9d5 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3cb75805e5..8953c0a81b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9475,3 +9475,11 @@ - "Refresh the aggregated curve with TP4/EP4 points at concurrency 1, 4, 8, 16, 24 and 32, plus TP4/EP4/DP4 points at concurrency 48 and 128 behind sglang-router." - "Apply the point-specific prefill chunking, scheduling, admission, memory and SWA prefix-tail settings from the single-node recipe." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3745 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update B300 DSV4 DSpark performance with DP spec/prefill coordination, mixed 64K chunks and interval 32; retain TP4/TP8 low-latency coverage. Use the CUDA 13 development nightly from 2026-10-08 (7760d526)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3606