Skip to content

Commit 07086dc

Browse files
RohitNagrajhshrivastava-droidclaudeAnkur-singh
authored
Add Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP / 新增 Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang MTP 配置 (#2594)
* feat(config): add Qwen3.5 FP4 GB200 Dynamo-SGLang MTP 新增 Qwen3.5 FP4 GB200 Dynamo-SGLang MTP 配置。 * chore(changelog): link PR #2594 在性能变更日志中补充 PR #2594 链接。 * fix(changelog): link PR #2594 to Qwen3.5 entry 将 PR #2594 链接补充到对应的 Qwen3.5 变更日志条目。 * Migrate Qwen3.5 GB200 recipes to srt-slurm schema 2 Applied srtctl migrate from the srt-slurm commit pinned by main; resolved prefill/decode nodes, workers, env, engine args, frontend, model and benchmark settings are unchanged. --------- Co-authored-by: hshrivastava-droid <hshrivastava@nvidia.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: Ankur-singh <ankusingh@nvidia.com>
1 parent 5d7bdb6 commit 07086dc

11 files changed

Lines changed: 1738 additions & 0 deletions

File tree

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c1.yaml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# Qwen3.5 NVFP4 V2 benchmark operating point.
2+
# Topology and serving knobs are pinned for reproducibility.
3+
schema: 2
4+
name: qwen35-v2-gb200-c1-1p1d-tp4-mtp3-baseline
5+
dynamo:
6+
install: false
7+
frontend:
8+
type: dynamo
9+
enable_multiple_frontends: false
10+
args:
11+
enforce-disagg: true
12+
model:
13+
path: qwen3.5-fp4
14+
container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928
15+
precision: fp4
16+
resources:
17+
gpu_type: gb200
18+
gpus_per_node: 4
19+
engine: sglang
20+
roles:
21+
prefill:
22+
nodes: 1
23+
workers: 1
24+
env:
25+
NO_COLOR: '1'
26+
PYTHONUNBUFFERED: '1'
27+
HF_HUB_OFFLINE: '1'
28+
TRANSFORMERS_OFFLINE: '1'
29+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
30+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
31+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
32+
NCCL_MNNVL_ENABLE: '1'
33+
NCCL_NVLS_ENABLE: '0'
34+
NCCL_CUMEM_ENABLE: '1'
35+
MC_FORCE_MNNVL: '1'
36+
SGLANG_ENABLE_SPEC_V2: '1'
37+
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
38+
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
39+
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
40+
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
41+
FLASHINFER_DISABLE_VERSION_CHECK: '1'
42+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
43+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
44+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
45+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
46+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
47+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
48+
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
49+
SGLANG_HEALTH_STARTING_OK: '1'
50+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
51+
args:
52+
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
53+
model-path: /model
54+
trust-remote-code: true
55+
reasoning-parser: qwen3
56+
tool-call-parser: qwen3_coder
57+
quantization: modelopt_mixed
58+
fp4-gemm-backend: flashinfer_cutlass
59+
kv-cache-dtype: fp8_e4m3
60+
tensor-parallel-size: 4
61+
data-parallel-size: 1
62+
expert-parallel-size: 1
63+
mamba-scheduler-strategy: no_buffer
64+
mamba-ssm-dtype: bfloat16
65+
mamba-track-interval: 2048
66+
attention-backend: trtllm_mha
67+
mm-attention-backend: triton_attn
68+
moe-runner-backend: flashinfer_trtllm
69+
linear-attn-decode-backend: flashinfer
70+
disaggregation-mode: prefill
71+
disaggregation-transfer-backend: mooncake
72+
disaggregation-bootstrap-port: 31000
73+
disable-radix-cache: true
74+
mem-fraction-static: 0.8
75+
context-length: 9236
76+
max-total-tokens: 128000
77+
max-running-requests: 128
78+
cuda-graph-max-bs: 4
79+
chunked-prefill-size: 32768
80+
max-prefill-tokens: 32768
81+
scheduler-recv-interval: 10
82+
stream-interval: 30
83+
load-balance-method: round_robin
84+
page-size: 64
85+
watchdog-timeout: 1000000
86+
log-level: info
87+
decode:
88+
nodes: 1
89+
workers: 1
90+
env:
91+
NO_COLOR: '1'
92+
PYTHONUNBUFFERED: '1'
93+
HF_HUB_OFFLINE: '1'
94+
TRANSFORMERS_OFFLINE: '1'
95+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
96+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
97+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
98+
NCCL_MNNVL_ENABLE: '1'
99+
NCCL_NVLS_ENABLE: '0'
100+
NCCL_CUMEM_ENABLE: '1'
101+
MC_FORCE_MNNVL: '1'
102+
MC_TE_METRIC: 'true'
103+
SGLANG_ENABLE_SPEC_V2: '1'
104+
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
105+
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
106+
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
107+
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
108+
FLASHINFER_DISABLE_VERSION_CHECK: '1'
109+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
110+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
111+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
112+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
113+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
114+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
115+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
116+
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
117+
SGLANG_HEALTH_STARTING_OK: '1'
118+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
119+
args:
120+
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
121+
model-path: /model
122+
trust-remote-code: true
123+
reasoning-parser: qwen3
124+
tool-call-parser: qwen3_coder
125+
quantization: modelopt_mixed
126+
fp4-gemm-backend: flashinfer_cutlass
127+
kv-cache-dtype: fp8_e4m3
128+
tensor-parallel-size: 4
129+
data-parallel-size: 1
130+
expert-parallel-size: 1
131+
mamba-scheduler-strategy: no_buffer
132+
mamba-ssm-dtype: bfloat16
133+
mamba-track-interval: 128
134+
attention-backend: trtllm_mha
135+
mm-attention-backend: triton_attn
136+
moe-runner-backend: flashinfer_trtllm
137+
linear-attn-decode-backend: flashinfer
138+
speculative-algorithm: NEXTN
139+
speculative-num-steps: 3
140+
speculative-eagle-topk: 1
141+
speculative-num-draft-tokens: 4
142+
disaggregation-mode: decode
143+
disaggregation-transfer-backend: mooncake
144+
disaggregation-bootstrap-port: 31000
145+
disable-radix-cache: true
146+
mem-fraction-static: 0.8
147+
context-length: 9236
148+
max-total-tokens: 1500000
149+
max-mamba-cache-size: 256
150+
max-running-requests: 128
151+
cuda-graph-max-bs: 256
152+
chunked-prefill-size: 32768
153+
max-prefill-tokens: 32768
154+
scheduler-recv-interval: 10
155+
stream-interval: 30
156+
page-size: 64
157+
watchdog-timeout: 1000000
158+
decode-log-interval: 1
159+
health_check:
160+
max_attempts: 240
161+
interval_seconds: 10
162+
benchmark:
163+
type: sa-bench
164+
isl: 8192
165+
osl: 1024
166+
concurrencies: '1'
167+
req_rate: inf
168+
random_range_ratio: 0.8
169+
num_prompts_mult: 10
170+
num_warmup_mult: 2
171+
use_chat_template: true

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4-v2/8k1k/mtp/1p1d-tp4-tp4-c4.yaml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# Qwen3.5 NVFP4 V2 benchmark operating point.
2+
# Topology and serving knobs are pinned for reproducibility.
3+
schema: 2
4+
name: qwen35-v2-gb200-c4-1p1d-tp4-mtp3-prefill-recv1
5+
dynamo:
6+
install: false
7+
frontend:
8+
type: dynamo
9+
enable_multiple_frontends: false
10+
args:
11+
enforce-disagg: true
12+
model:
13+
path: qwen3.5-fp4
14+
container: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928
15+
precision: fp4
16+
resources:
17+
gpu_type: gb200
18+
gpus_per_node: 4
19+
engine: sglang
20+
roles:
21+
prefill:
22+
nodes: 1
23+
workers: 1
24+
env:
25+
NO_COLOR: '1'
26+
PYTHONUNBUFFERED: '1'
27+
HF_HUB_OFFLINE: '1'
28+
TRANSFORMERS_OFFLINE: '1'
29+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
30+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
31+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
32+
NCCL_MNNVL_ENABLE: '1'
33+
NCCL_NVLS_ENABLE: '0'
34+
NCCL_CUMEM_ENABLE: '1'
35+
MC_FORCE_MNNVL: '1'
36+
SGLANG_ENABLE_SPEC_V2: '1'
37+
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
38+
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
39+
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
40+
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
41+
FLASHINFER_DISABLE_VERSION_CHECK: '1'
42+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
43+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
44+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
45+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
46+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
47+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
48+
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
49+
SGLANG_HEALTH_STARTING_OK: '1'
50+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
51+
args:
52+
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
53+
model-path: /model
54+
trust-remote-code: true
55+
reasoning-parser: qwen3
56+
tool-call-parser: qwen3_coder
57+
quantization: modelopt_mixed
58+
fp4-gemm-backend: flashinfer_cutlass
59+
kv-cache-dtype: fp8_e4m3
60+
tensor-parallel-size: 4
61+
data-parallel-size: 1
62+
expert-parallel-size: 1
63+
mamba-scheduler-strategy: no_buffer
64+
mamba-ssm-dtype: bfloat16
65+
mamba-track-interval: 2048
66+
attention-backend: trtllm_mha
67+
mm-attention-backend: triton_attn
68+
moe-runner-backend: flashinfer_trtllm
69+
linear-attn-decode-backend: flashinfer
70+
disaggregation-mode: prefill
71+
disaggregation-transfer-backend: mooncake
72+
disaggregation-bootstrap-port: 31000
73+
disable-radix-cache: true
74+
mem-fraction-static: 0.8
75+
context-length: 9236
76+
max-total-tokens: 128000
77+
max-running-requests: 128
78+
cuda-graph-max-bs: 4
79+
chunked-prefill-size: 32768
80+
max-prefill-tokens: 32768
81+
scheduler-recv-interval: 1
82+
stream-interval: 30
83+
load-balance-method: round_robin
84+
page-size: 64
85+
watchdog-timeout: 1000000
86+
log-level: info
87+
decode:
88+
nodes: 1
89+
workers: 1
90+
env:
91+
NO_COLOR: '1'
92+
PYTHONUNBUFFERED: '1'
93+
HF_HUB_OFFLINE: '1'
94+
TRANSFORMERS_OFFLINE: '1'
95+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '3600'
96+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: '3600'
97+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '3600'
98+
NCCL_MNNVL_ENABLE: '1'
99+
NCCL_NVLS_ENABLE: '0'
100+
NCCL_CUMEM_ENABLE: '1'
101+
MC_FORCE_MNNVL: '1'
102+
MC_TE_METRIC: 'true'
103+
SGLANG_ENABLE_SPEC_V2: '1'
104+
SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache
105+
FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache
106+
SGLANG_ENABLE_JIT_DEEPGEMM: 'true'
107+
SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
108+
FLASHINFER_DISABLE_VERSION_CHECK: '1'
109+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
110+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
111+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
112+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
113+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
114+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
115+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
116+
SGLANG_HEALTH_CHECK_TIMEOUT: '3600'
117+
SGLANG_HEALTH_STARTING_OK: '1'
118+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0'
119+
args:
120+
served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
121+
model-path: /model
122+
trust-remote-code: true
123+
reasoning-parser: qwen3
124+
tool-call-parser: qwen3_coder
125+
quantization: modelopt_mixed
126+
fp4-gemm-backend: flashinfer_cutlass
127+
kv-cache-dtype: fp8_e4m3
128+
tensor-parallel-size: 4
129+
data-parallel-size: 1
130+
expert-parallel-size: 1
131+
mamba-scheduler-strategy: no_buffer
132+
mamba-ssm-dtype: bfloat16
133+
mamba-track-interval: 128
134+
attention-backend: trtllm_mha
135+
mm-attention-backend: triton_attn
136+
moe-runner-backend: flashinfer_trtllm
137+
linear-attn-decode-backend: flashinfer
138+
speculative-algorithm: NEXTN
139+
speculative-num-steps: 3
140+
speculative-eagle-topk: 1
141+
speculative-num-draft-tokens: 4
142+
disaggregation-mode: decode
143+
disaggregation-transfer-backend: mooncake
144+
disaggregation-bootstrap-port: 31000
145+
disable-radix-cache: true
146+
mem-fraction-static: 0.8
147+
context-length: 9236
148+
max-total-tokens: 1500000
149+
max-mamba-cache-size: 256
150+
max-running-requests: 128
151+
cuda-graph-max-bs: 256
152+
chunked-prefill-size: 32768
153+
max-prefill-tokens: 32768
154+
scheduler-recv-interval: 10
155+
stream-interval: 30
156+
page-size: 64
157+
watchdog-timeout: 1000000
158+
decode-log-interval: 1
159+
health_check:
160+
max_attempts: 240
161+
interval_seconds: 10
162+
benchmark:
163+
type: sa-bench
164+
isl: 8192
165+
osl: 1024
166+
concurrencies: '4'
167+
req_rate: inf
168+
random_range_ratio: 0.8
169+
num_prompts_mult: 10
170+
num_warmup_mult: 2
171+
use_chat_template: true

0 commit comments

Comments
 (0)