From 078f128a0f57abc9a53b46d0580ac07707c20547 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:44:04 +0000 Subject: [PATCH 1/3] chore(qwen3.5-fp4-gb200-dynamo-sglang-mtp): update SGLang image to v0.5.21-cu130 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin lmsysorg/sglang:v0.5.21-cu130 by digest and rename options removed in v0.5.21: mamba-scheduler-strategy -> mamba-radix-cache-strategy, cuda-graph-max-bs -> cuda-graph-max-bs-decode, and SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 -> SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0. The old image accepted the old names as deprecated aliases for the same settings. 将 SGLang 镜像更新为 v0.5.21-cu130（按 digest 固定），并重命名 v0.5.21 中已移除的选项：mamba-scheduler-strategy 改为 mamba-radix-cache-strategy， cuda-graph-max-bs 改为 cuda-graph-max-bs-decode， SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 改为 SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0。旧镜像将旧名称作为同一设置的 弃用别名接受。 Co-Authored-By: Claude Opus 5.5 --- .../gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml | 12 ++++++------ .../gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml | 12 ++++++------ inferencex-e2e/configs/nvidia-master.yaml | 2 +- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml index 20609e1a05..bbefccfca3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml @@ -38,7 +38,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -54,7 +54,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -69,7 +69,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 @@ -106,7 +106,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -122,7 +122,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -142,7 +142,7 @@ roles: max-total-tokens: 1500000 max-mamba-cache-size: 256 max-running-requests: 128 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml index 398b96f45c..af64b3b715 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml @@ -38,7 +38,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -54,7 +54,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -69,7 +69,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 1 @@ -106,7 +106,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -122,7 +122,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -142,7 +142,7 @@ roles: max-total-tokens: 1500000 max-mamba-cache-size: 256 max-running-requests: 128 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml index 236b708130..4091ef198a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml @@ -38,7 +38,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -54,7 +54,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -69,7 +69,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 @@ -106,7 +106,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -122,7 +122,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -142,7 +142,7 @@ roles: max-total-tokens: 1500000 max-mamba-cache-size: 256 max-running-requests: 128 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml index 8f5268d60e..9bcd860f2a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml @@ -42,7 +42,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -58,7 +58,7 @@ roles: tensor-parallel-size: 2 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -72,7 +72,7 @@ roles: context-length: 9236 max-total-tokens: 64000 max-running-requests: 64 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 16384 max-prefill-tokens: 16384 scheduler-recv-interval: 10 @@ -110,7 +110,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -126,7 +126,7 @@ roles: tensor-parallel-size: 2 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -145,7 +145,7 @@ roles: max-total-tokens: 750000 max-mamba-cache-size: 128 max-running-requests: 84 - cuda-graph-max-bs: 88 + cuda-graph-max-bs-decode: 88 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml index 947d18fc6d..2228005300 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml @@ -39,7 +39,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -55,7 +55,7 @@ roles: tensor-parallel-size: 2 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -69,7 +69,7 @@ roles: context-length: 9236 max-total-tokens: 64000 max-running-requests: 64 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 16384 max-prefill-tokens: 16384 scheduler-recv-interval: 10 @@ -106,7 +106,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -122,7 +122,7 @@ roles: tensor-parallel-size: 2 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -141,7 +141,7 @@ roles: max-total-tokens: 750000 max-mamba-cache-size: 128 max-running-requests: 86 - cuda-graph-max-bs: 88 + cuda-graph-max-bs-decode: 88 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml index f3771aa7c2..84ea3cb774 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml @@ -38,7 +38,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -54,7 +54,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -69,7 +69,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 @@ -106,7 +106,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -122,7 +122,7 @@ roles: tensor-parallel-size: 4 data-parallel-size: 1 expert-parallel-size: 1 - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -142,7 +142,7 @@ roles: max-total-tokens: 1500000 max-mamba-cache-size: 256 max-running-requests: 128 - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml index 7ad4de2f07..fade98945b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml @@ -40,7 +40,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -58,7 +58,7 @@ roles: expert-parallel-size: 4 enable-dp-attention: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -72,7 +72,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 65536 max-prefill-tokens: 65536 scheduler-recv-interval: 1 @@ -111,7 +111,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -129,7 +129,7 @@ roles: expert-parallel-size: 16 enable-dp-attention: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -147,7 +147,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake disable-radix-cache: true - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 mem-fraction-static: 0.8 context-length: 9236 max-total-tokens: 2200000 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml index dd8964559d..489daf6f0c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml @@ -40,7 +40,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -58,7 +58,7 @@ roles: expert-parallel-size: 4 enable-dp-attention: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -72,7 +72,7 @@ roles: context-length: 9236 max-total-tokens: 128000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 65536 max-prefill-tokens: 65536 scheduler-recv-interval: 1 @@ -111,7 +111,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -129,7 +129,7 @@ roles: expert-parallel-size: 16 enable-dp-attention: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -147,7 +147,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake disable-radix-cache: true - cuda-graph-max-bs: 256 + cuda-graph-max-bs-decode: 256 mem-fraction-static: 0.8 context-length: 9236 max-total-tokens: 2200000 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml index 3d4979c22e..be96d56436 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml @@ -41,7 +41,7 @@ roles: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -60,7 +60,7 @@ roles: enable-dp-attention: true enable-dp-attention-local-control-broadcast: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 2048 attention-backend: trtllm_mha @@ -74,7 +74,7 @@ roles: context-length: 9236 max-total-tokens: 256000 max-running-requests: 128 - cuda-graph-max-bs: 4 + cuda-graph-max-bs-decode: 4 chunked-prefill-size: 262144 max-prefill-tokens: 131072 scheduler-recv-interval: 1 @@ -116,7 +116,7 @@ roles: SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '3600' SGLANG_HEALTH_STARTING_OK: '1' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' @@ -139,7 +139,7 @@ roles: expert-parallel-size: 4 enable-dp-attention: true enable-dp-lm-head: true - mamba-scheduler-strategy: no_buffer + mamba-radix-cache-strategy: no_buffer mamba-ssm-dtype: bfloat16 mamba-track-interval: 128 attention-backend: trtllm_mha @@ -158,7 +158,7 @@ roles: disaggregation-mode: decode disaggregation-transfer-backend: mooncake disable-radix-cache: true - cuda-graph-max-bs: 512 + cuda-graph-max-bs-decode: 512 mem-fraction-static: 0.94 context-length: 9236 max-total-tokens: 2052736 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index b50c15d71b..4e5b7c30e3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7830,7 +7830,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true qwen3.5-fp4-gb200-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469 model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb200 From f182efb62daed7e095f1124038bdf1fc164008d2 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:19:52 +0000 Subject: [PATCH 2/3] fix(qwen3.5-fp4-gb200-dynamo-sglang-mtp): use the image's Dynamo for c12288 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The c12288 recipe pinned the ai-dynamo 1.2.1 wheel over the image's bundled ai-dynamo. dynamo.sglang 1.2.1 imports sglang.srt.server_args_config_parser, which SGLang v0.5.21 moved under sglang.srt.utils, so workers failed at import. Use the bundled Dynamo like the family's other recipes. c12288 配方用 ai-dynamo 1.2.1 wheel 覆盖了镜像自带的 ai-dynamo。 dynamo.sglang 1.2.1 导入 sglang.srt.server_args_config_parser，而 SGLang v0.5.21 已将其移至 sglang.srt.utils，导致 worker 导入失败。 改为与本配置族其他配方一样使用镜像自带的 Dynamo。 Co-Authored-By: Claude Opus 5.5 --- .../sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml index be96d56436..b5292b4d43 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml @@ -1,9 +1,7 @@ schema: 2 name: qwen35-v2-gb200-c12288-7p4d-dep4-mtp1-cap1056 dynamo: - install: true - source: - wheel: "1.2.1" + install: false request_plane: "nats" frontend: type: dynamo From 57441f3ecad68a86fcbc465fe2579b67916599a3 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:52:32 +0000 Subject: [PATCH 3/3] fix(qwen3.5-fp4-gb200-dynamo-sglang-mtp): keep prefill CUDA graphs disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SGLang 074bb928 auto-disabled breakable prefill CUDA graphs for multimodal models and DP attention, so these Qwen3.5 prefill workers ran without them. v0.5.21 marks Qwen3_5MoeForConditionalGeneration as breakable-supported and drops the DP-attention rule; the DEP4 prefill workers then OOM capturing a 65536-token graph. Set cuda-graph-backend-prefill: disabled on every prefill role to keep the old effective setting. SGLang 074bb928 会对多模态模型和 DP attention 自动禁用 breakable prefill CUDA graph，因此这些 Qwen3.5 prefill worker 此前不使用它。 v0.5.21 将 Qwen3_5MoeForConditionalGeneration 标记为支持 breakable， 并移除了 DP attention 规则；DEP4 prefill worker 在捕获 65536 token 的 图时显存不足。在每个 prefill 角色上设置 cuda-graph-backend-prefill: disabled，以保持旧的实际设置。 Co-Authored-By: Claude Opus 5.5 --- .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml | 1 + .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml | 1 + .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml | 1 + .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml | 1 + .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml | 1 + .../qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml | 1 + .../sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml | 1 + .../sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml | 1 + .../sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml | 1 + 9 files changed, 9 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml index bbefccfca3..5130aa393f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml @@ -70,6 +70,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml index af64b3b715..4298b290b3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml @@ -70,6 +70,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 1 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml index 4091ef198a..f493b3e024 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c8.yaml @@ -70,6 +70,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml index 9bcd860f2a..8d0bd5428b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p2d-tp2-tp2-c512.yaml @@ -73,6 +73,7 @@ roles: max-total-tokens: 64000 max-running-requests: 64 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 16384 max-prefill-tokens: 16384 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml index 2228005300..0bf4f4a288 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/2p3d-tp2-tp2-c768.yaml @@ -70,6 +70,7 @@ roles: max-total-tokens: 64000 max-running-requests: 64 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 16384 max-prefill-tokens: 16384 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml index 84ea3cb774..275037c04e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/3p5d-tp4-tp4-c128.yaml @@ -70,6 +70,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 32768 max-prefill-tokens: 32768 scheduler-recv-interval: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml index fade98945b..e6135be62b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c5120.yaml @@ -73,6 +73,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 65536 max-prefill-tokens: 65536 scheduler-recv-interval: 1 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml index 489daf6f0c..e035ec8b9e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p1d-dep4-dep16-c8192.yaml @@ -73,6 +73,7 @@ roles: max-total-tokens: 128000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 65536 max-prefill-tokens: 65536 scheduler-recv-interval: 1 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml index b5292b4d43..7019ee5c05 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/7p4d-dep4-dep4-c12288.yaml @@ -73,6 +73,7 @@ roles: max-total-tokens: 256000 max-running-requests: 128 cuda-graph-max-bs-decode: 4 + cuda-graph-backend-prefill: disabled chunked-prefill-size: 262144 max-prefill-tokens: 131072 scheduler-recv-interval: 1