Skip to content

Commit 8711b47

Browse files
cquil11cquil11
committed
fix(amd): preserve DSV4 fixed-sequence semantics
Restore the proven STP and MTP search-space grouping, server flags, router policy, CUDA graph batches, and MI355X runtime environment while keeping the srt-slurm orchestration port.
1 parent 54ca2b6 commit 8711b47

6 files changed

Lines changed: 154 additions & 54 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,12 @@ frontend:
2929
type: sglang
3030
enable_multiple_frontends: false
3131
args:
32-
policy: consistent_hashing
32+
policy: random
33+
prefill-policy: random
34+
decode-policy: random
3335
dp-aware: true
34-
cache-threshold: 0.3
35-
balance-abs-threshold: 2
36-
balance-rel-threshold: 1.1
37-
disable-circuit-breaker: true
38-
health-failure-threshold: 100
39-
health-check-timeout-secs: 600
40-
health-check-interval-secs: 30
36+
cb-timeout-duration-secs: 15
37+
retry-max-retries: 3
4138

4239
backend:
4340
type: sglang
@@ -48,12 +45,16 @@ backend:
4845
PYTHONUNBUFFERED: "1"
4946
TORCH_NCCL_BLOCKING_WAIT: "1"
5047
NCCL_BLOCKING_WAIT: "1"
48+
NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
5149
HSA_NO_SCRATCH_RECLAIM: "0"
5250
GPU_MAX_HW_QUEUES: "5"
51+
AITER_LOG_LEVEL: "ERROR"
5352
SGLANG_AITER_MLA_PERSIST: "0"
5453
SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
5554
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
5655
SGLANG_MORI_DISPATCH_DTYPE: "auto"
56+
SGLANG_MORI_QP_PER_TRANSFER: "4"
57+
SGLANG_MORI_NUM_WORKERS: "4"
5758
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096"
5859
SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false"
5960
SGLANG_USE_AITER: "1"
@@ -72,19 +73,32 @@ backend:
7273
SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false"
7374
SGLANG_ROCM_USE_MULTI_STREAM: "false"
7475
AITER_BF16_FP8_MOE_BOUND: "0"
76+
SGLANG_DEFAULT_THINKING: "1"
77+
SGLANG_DSV4_REASONING_EFFORT: "high"
7578
SGLANG_EAGER_INPUT_NO_COPY: "true"
7679
SGLANG_SHARED_EXPERT_TP1: "1"
7780
SGLANG_DP_SHARED_EXPERT_LOCAL: "1"
7881
SGLANG_DP_USE_GATHERV: "1"
7982
SGLANG_DP_USE_REDUCE_SCATTER: "1"
8083
IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
8184
MORI_RDMA_TC: "104"
82-
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000"
85+
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000"
8386
MORI_IO_QP_MAX_SEND_WR: "32767"
87+
MORI_IO_QP_MAX_CQE: "32768"
88+
MORI_IO_QP_MAX_SGE: "2"
89+
MORI_IO_TC_DISABLE: "0"
90+
MORI_SHMEM_MODE: "ISOLATION"
91+
MORI_EP_LAUNCH_CONFIG_MODE: "AUTO"
92+
MORI_APP_LOG_LEVEL: "WARNING"
8493
SGLANG_MORI_COMBINE_DTYPE: "auto"
8594
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384"
86-
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800"
87-
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800"
95+
SGLANG_ENABLE_SPEC_V2: "1"
96+
SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0"
97+
SGLANG_LOG_MS: "true"
98+
SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32"
99+
SGLANG_HEALTH_CHECK_TIMEOUT: "600"
100+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
101+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
88102
# The native router may select a different DPA rank from the rank encoded
89103
# in bootstrap_room. Use SGLang's supported rank-query fallback so decode
90104
# retrieves the actual prefill rank instead of rejecting the KV transfer.
@@ -100,14 +114,14 @@ backend:
100114
ep-size: 8
101115
dp-size: 8
102116
enable-dp-attention: true
103-
enable-prefill-delayer: true
104117
ep-dispatch-algorithm: fake
105118
moe-a2a-backend: mori
106119
deepep-mode: normal
107120
moe-dense-tp-size: 1
108121
enable-dp-lm-head: true
109122
watchdog-timeout: 3600
110-
load-balance-method: auto
123+
decode-log-interval: 100
124+
load-balance-method: round_robin
111125
kv-cache-dtype: fp8_e4m3
112126
attention-backend: dsv4
113127
page-size: 256
@@ -141,7 +155,7 @@ backend:
141155
disable-radix-cache: false
142156
disable-cuda-graph: false
143157
prefill-round-robin-balance: true
144-
cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128]
158+
cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128]
145159

146160
sbatch_directives:
147161
cpus-per-task: "128"
@@ -161,7 +175,7 @@ benchmark:
161175
result_root="/results/${SLURM_JOB_ID}/fixed-seq"
162176
mkdir -p "${result_root}"
163177
trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
164-
for concurrency in 256 512; do
178+
for concurrency in ${CONC_LIST}; do
165179
python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
166180
--backend openai \
167181
--base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,12 @@ frontend:
2929
type: sglang
3030
enable_multiple_frontends: false
3131
args:
32-
policy: consistent_hashing
32+
policy: random
33+
prefill-policy: random
34+
decode-policy: random
3335
dp-aware: true
34-
cache-threshold: 0.3
35-
balance-abs-threshold: 2
36-
balance-rel-threshold: 1.1
37-
disable-circuit-breaker: true
38-
health-failure-threshold: 100
39-
health-check-timeout-secs: 600
40-
health-check-interval-secs: 30
36+
cb-timeout-duration-secs: 15
37+
retry-max-retries: 3
4138

4239
backend:
4340
type: sglang
@@ -48,12 +45,16 @@ backend:
4845
PYTHONUNBUFFERED: "1"
4946
TORCH_NCCL_BLOCKING_WAIT: "1"
5047
NCCL_BLOCKING_WAIT: "1"
48+
NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
5149
HSA_NO_SCRATCH_RECLAIM: "0"
5250
GPU_MAX_HW_QUEUES: "5"
51+
AITER_LOG_LEVEL: "ERROR"
5352
SGLANG_AITER_MLA_PERSIST: "0"
5453
SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
5554
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
5655
SGLANG_MORI_DISPATCH_DTYPE: "auto"
56+
SGLANG_MORI_QP_PER_TRANSFER: "4"
57+
SGLANG_MORI_NUM_WORKERS: "4"
5758
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096"
5859
SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false"
5960
SGLANG_USE_AITER: "1"
@@ -72,19 +73,32 @@ backend:
7273
SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false"
7374
SGLANG_ROCM_USE_MULTI_STREAM: "false"
7475
AITER_BF16_FP8_MOE_BOUND: "0"
76+
SGLANG_DEFAULT_THINKING: "1"
77+
SGLANG_DSV4_REASONING_EFFORT: "high"
7578
SGLANG_EAGER_INPUT_NO_COPY: "true"
7679
SGLANG_SHARED_EXPERT_TP1: "1"
7780
SGLANG_DP_SHARED_EXPERT_LOCAL: "1"
7881
SGLANG_DP_USE_GATHERV: "1"
7982
SGLANG_DP_USE_REDUCE_SCATTER: "1"
8083
IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
8184
MORI_RDMA_TC: "104"
82-
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000"
85+
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000"
8386
MORI_IO_QP_MAX_SEND_WR: "32767"
87+
MORI_IO_QP_MAX_CQE: "32768"
88+
MORI_IO_QP_MAX_SGE: "2"
89+
MORI_IO_TC_DISABLE: "0"
90+
MORI_SHMEM_MODE: "ISOLATION"
91+
MORI_EP_LAUNCH_CONFIG_MODE: "AUTO"
92+
MORI_APP_LOG_LEVEL: "WARNING"
8493
SGLANG_MORI_COMBINE_DTYPE: "auto"
8594
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384"
86-
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800"
87-
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800"
95+
SGLANG_ENABLE_SPEC_V2: "1"
96+
SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0"
97+
SGLANG_LOG_MS: "true"
98+
SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32"
99+
SGLANG_HEALTH_CHECK_TIMEOUT: "600"
100+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
101+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
88102
# The native router may select a different DPA rank from the rank encoded
89103
# in bootstrap_room. Use SGLang's supported rank-query fallback so decode
90104
# retrieves the actual prefill rank instead of rejecting the KV transfer.
@@ -100,14 +114,14 @@ backend:
100114
ep-size: 8
101115
dp-size: 8
102116
enable-dp-attention: true
103-
enable-prefill-delayer: true
104117
ep-dispatch-algorithm: fake
105118
moe-a2a-backend: mori
106119
deepep-mode: normal
107120
moe-dense-tp-size: 1
108121
enable-dp-lm-head: true
109122
watchdog-timeout: 3600
110-
load-balance-method: auto
123+
decode-log-interval: 100
124+
load-balance-method: round_robin
111125
kv-cache-dtype: fp8_e4m3
112126
attention-backend: dsv4
113127
page-size: 256
@@ -133,7 +147,7 @@ backend:
133147
disable-radix-cache: false
134148
disable-cuda-graph: false
135149
prefill-round-robin-balance: true
136-
cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128]
150+
cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128]
137151

138152
sbatch_directives:
139153
cpus-per-task: "128"
@@ -153,7 +167,7 @@ benchmark:
153167
result_root="/results/${SLURM_JOB_ID}/fixed-seq"
154168
mkdir -p "${result_root}"
155169
trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
156-
for concurrency in 256 512; do
170+
for concurrency in ${CONC_LIST}; do
157171
python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
158172
--backend openai \
159173
--base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-tp8-mtp.yaml‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,11 @@ frontend:
2929
type: sglang
3030
enable_multiple_frontends: false
3131
args:
32-
policy: round_robin
33-
prefill-policy: round_robin
34-
decode-policy: round_robin
32+
policy: random
33+
prefill-policy: random
34+
decode-policy: random
35+
cb-timeout-duration-secs: 15
36+
retry-max-retries: 3
3537

3638
backend:
3739
type: sglang
@@ -42,12 +44,16 @@ backend:
4244
PYTHONUNBUFFERED: "1"
4345
TORCH_NCCL_BLOCKING_WAIT: "1"
4446
NCCL_BLOCKING_WAIT: "1"
47+
NCCL_IB_HCA: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
4548
HSA_NO_SCRATCH_RECLAIM: "0"
4649
GPU_MAX_HW_QUEUES: "5"
50+
AITER_LOG_LEVEL: "ERROR"
4751
SGLANG_AITER_MLA_PERSIST: "0"
4852
SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
4953
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
5054
SGLANG_MORI_DISPATCH_DTYPE: "auto"
55+
SGLANG_MORI_QP_PER_TRANSFER: "4"
56+
SGLANG_MORI_NUM_WORKERS: "4"
5157
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096"
5258
SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false"
5359
SGLANG_USE_AITER: "1"
@@ -66,29 +72,43 @@ backend:
6672
SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false"
6773
SGLANG_ROCM_USE_MULTI_STREAM: "false"
6874
AITER_BF16_FP8_MOE_BOUND: "0"
75+
SGLANG_DEFAULT_THINKING: "1"
76+
SGLANG_DSV4_REASONING_EFFORT: "high"
6977
SGLANG_EAGER_INPUT_NO_COPY: "true"
7078
SGLANG_SHARED_EXPERT_TP1: "1"
7179
SGLANG_DP_SHARED_EXPERT_LOCAL: "1"
7280
SGLANG_DP_USE_GATHERV: "1"
7381
SGLANG_DP_USE_REDUCE_SCATTER: "1"
7482
IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
7583
MORI_RDMA_TC: "104"
76-
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000"
84+
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000"
7785
MORI_IO_QP_MAX_SEND_WR: "32767"
86+
MORI_IO_QP_MAX_CQE: "32768"
87+
MORI_IO_QP_MAX_SGE: "2"
88+
MORI_IO_TC_DISABLE: "0"
89+
MORI_SHMEM_MODE: "ISOLATION"
90+
MORI_EP_LAUNCH_CONFIG_MODE: "AUTO"
91+
MORI_APP_LOG_LEVEL: "WARNING"
7892
SGLANG_MORI_COMBINE_DTYPE: "auto"
7993
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384"
80-
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800"
81-
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800"
94+
SGLANG_ENABLE_SPEC_V2: "1"
95+
SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0"
96+
SGLANG_LOG_MS: "true"
97+
SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32"
98+
SGLANG_HEALTH_CHECK_TIMEOUT: "600"
99+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
100+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
82101
decode_environment:
83102
<<: *common_environment
84-
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024"
103+
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "128"
85104
sglang_config:
86105
prefill: &common_server
87106
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
88107
trust-remote-code: true
89108
tp-size: 8
90109
watchdog-timeout: 3600
91-
load-balance-method: auto
110+
decode-log-interval: 100
111+
load-balance-method: round_robin
92112
kv-cache-dtype: fp8_e4m3
93113
attention-backend: dsv4
94114
page-size: 256
@@ -122,7 +142,7 @@ backend:
122142
disable-radix-cache: false
123143
disable-cuda-graph: false
124144
prefill-round-robin-balance: true
125-
cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128]
145+
cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128]
126146

127147
sbatch_directives:
128148
cpus-per-task: "128"
@@ -142,7 +162,7 @@ benchmark:
142162
result_root="/results/${SLURM_JOB_ID}/fixed-seq"
143163
mkdir -p "${result_root}"
144164
trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
145-
for concurrency in 1 2 4 8 16 32 64 128; do
165+
for concurrency in ${CONC_LIST}; do
146166
num_prompts=$((concurrency * 10))
147167
if (( num_prompts < 16 )); then num_prompts=16; fi
148168
python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \

0 commit comments

Comments
 (0)