Skip to content

Commit ee8c921

Browse files
cquil11cquil11
committed
feat(amd): port DSV4 fixed-sequence to srt-slurm
1 parent 8581966 commit ee8c921

6 files changed

Lines changed: 832 additions & 0 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-mtp.yaml‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
name: "mi355x-dsv4-disagg-1p1d-dep8-mtp-fixed-seq"
2+
3+
model:
4+
path: "hf:deepseek-ai/DeepSeek-V4-Pro"
5+
container: "sglang-rocm-dsv4-mi355x"
6+
precision: "fp4"
7+
8+
identity:
9+
model: {repo: "deepseek-ai/DeepSeek-V4-Pro"}
10+
container: {image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"}
11+
frameworks:
12+
sglang: "0.5.15.post1.dev20260719+ga03ca46a28"
13+
sglang-router: "0.3.2"
14+
amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28"
15+
16+
slurm: {time_limit: "04:00:00"}
17+
18+
resources:
19+
gpu_type: "mi355x"
20+
gpus_per_node: 8
21+
prefill_nodes: 1
22+
decode_nodes: 1
23+
prefill_workers: 1
24+
decode_workers: 1
25+
gpus_per_prefill: 8
26+
gpus_per_decode: 8
27+
28+
frontend:
29+
type: sglang
30+
enable_multiple_frontends: false
31+
args:
32+
policy: consistent_hashing
33+
dp-aware: true
34+
cache-threshold: 0.3
35+
balance-abs-threshold: 2
36+
balance-rel-threshold: 1.1
37+
disable-circuit-breaker: true
38+
health-failure-threshold: 100
39+
health-check-timeout-secs: 600
40+
health-check-interval-secs: 30
41+
42+
backend:
43+
type: sglang
44+
prefill_environment: &common_environment
45+
HF_HOME: "/hf_hub_cache"
46+
HF_HUB_CACHE: "/hf_hub_cache/hub"
47+
HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub"
48+
PYTHONUNBUFFERED: "1"
49+
TORCH_NCCL_BLOCKING_WAIT: "1"
50+
NCCL_BLOCKING_WAIT: "1"
51+
HSA_NO_SCRATCH_RECLAIM: "0"
52+
GPU_MAX_HW_QUEUES: "5"
53+
SGLANG_AITER_MLA_PERSIST: "0"
54+
SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
55+
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
56+
SGLANG_MORI_DISPATCH_DTYPE: "auto"
57+
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096"
58+
SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false"
59+
SGLANG_USE_AITER: "1"
60+
SGLANG_USE_ROCM700A: "0"
61+
SGLANG_OPT_USE_FUSED_COMPRESS: "true"
62+
SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton"
63+
SGLANG_OPT_FP8_WO_A_GEMM: "false"
64+
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false"
65+
SGLANG_OPT_USE_TOPK_V2: "false"
66+
SGLANG_OPT_USE_AITER_INDEXER: "true"
67+
SGLANG_OPT_USE_TILELANG_INDEXER: "false"
68+
SGLANG_OPT_USE_TILELANG_MHC_PRE: "false"
69+
SGLANG_OPT_USE_TILELANG_MHC_POST: "false"
70+
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1"
71+
SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true"
72+
SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false"
73+
SGLANG_ROCM_USE_MULTI_STREAM: "false"
74+
AITER_BF16_FP8_MOE_BOUND: "0"
75+
SGLANG_EAGER_INPUT_NO_COPY: "true"
76+
SGLANG_SHARED_EXPERT_TP1: "1"
77+
SGLANG_DP_SHARED_EXPERT_LOCAL: "1"
78+
SGLANG_DP_USE_GATHERV: "1"
79+
SGLANG_DP_USE_REDUCE_SCATTER: "1"
80+
IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
81+
MORI_RDMA_TC: "104"
82+
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000"
83+
MORI_IO_QP_MAX_SEND_WR: "32767"
84+
SGLANG_MORI_COMBINE_DTYPE: "auto"
85+
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384"
86+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800"
87+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800"
88+
# The native router may select a different DPA rank from the rank encoded
89+
# in bootstrap_room. Use SGLang's supported rank-query fallback so decode
90+
# retrieves the actual prefill rank instead of rejecting the KV transfer.
91+
SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK: "1"
92+
decode_environment:
93+
<<: *common_environment
94+
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024"
95+
sglang_config:
96+
prefill: &common_server
97+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
98+
trust-remote-code: true
99+
tp-size: 8
100+
ep-size: 8
101+
dp-size: 8
102+
enable-dp-attention: true
103+
enable-prefill-delayer: true
104+
ep-dispatch-algorithm: fake
105+
moe-a2a-backend: mori
106+
deepep-mode: normal
107+
moe-dense-tp-size: 1
108+
enable-dp-lm-head: true
109+
watchdog-timeout: 3600
110+
load-balance-method: auto
111+
kv-cache-dtype: fp8_e4m3
112+
attention-backend: dsv4
113+
page-size: 256
114+
swa-full-tokens-ratio: 0.1
115+
disable-shared-experts-fusion: true
116+
tool-call-parser: deepseekv4
117+
reasoning-parser: deepseek-v4
118+
disaggregation-transfer-backend: mori
119+
speculative-algorithm: EAGLE
120+
speculative-eagle-topk: 1
121+
speculative-num-steps: 1
122+
speculative-num-draft-tokens: 2
123+
log-level: info
124+
log-level-http: error
125+
# The DSV4 chat encoder adds three framing tokens to the nominal 8192
126+
# prompt. Match the workflow's 9472-token admission budget so the full
127+
# 1024-token completion fits (8195 + 1024) instead of being rejected.
128+
context-length: 9472
129+
enable-metrics: true
130+
enable-cache-report: true
131+
mem-fraction-static: 0.80
132+
max-running-requests: 1024
133+
chunked-prefill-size: 131072
134+
max-total-tokens: 2097152
135+
disable-radix-cache: true
136+
disable-cuda-graph: true
137+
decode:
138+
<<: *common_server
139+
mem-fraction-static: 0.85
140+
max-total-tokens: null
141+
disable-radix-cache: false
142+
disable-cuda-graph: false
143+
prefill-round-robin-balance: true
144+
cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128]
145+
146+
sbatch_directives:
147+
cpus-per-task: "128"
148+
mem: "0"
149+
150+
srun_options:
151+
mem: "0"
152+
container-writable: ""
153+
container-remap-root: ""
154+
155+
health_check: {max_attempts: 720, interval_seconds: 5}
156+
157+
benchmark:
158+
type: custom
159+
command: |
160+
set -euo pipefail
161+
result_root="/results/${SLURM_JOB_ID}/fixed-seq"
162+
mkdir -p "${result_root}"
163+
trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
164+
for concurrency in 256 512; do
165+
python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
166+
--backend openai \
167+
--base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
168+
--endpoint /v1/completions \
169+
--model deepseek-ai/DeepSeek-V4-Pro \
170+
--tokenizer deepseek-ai/DeepSeek-V4-Pro \
171+
--trust-remote-code \
172+
--dataset-name random \
173+
--random-input-len 8192 \
174+
--random-output-len 1024 \
175+
--random-range-ratio 1.0 \
176+
--random-num-workers 1 \
177+
--use-chat-template \
178+
--dsv4 \
179+
--num-warmups "$((concurrency * 2))" \
180+
--num-prompts "$((concurrency * 10))" \
181+
--max-concurrency "${concurrency}" \
182+
--request-rate inf \
183+
--ignore-eos \
184+
--disable-tqdm \
185+
--save-result \
186+
--result-dir "${result_root}" \
187+
--result-filename "dsv4-fp4-disagg-mtp1-dep8-isl8192-osl1024-c${concurrency}.json"
188+
done
189+
env:
190+
HF_HOME: /hf_hub_cache
191+
HF_HUB_CACHE: /hf_hub_cache/hub
192+
HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/mi355x/fixed-seq/disagg-1p1d-dep8-stp.yaml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
name: "mi355x-dsv4-disagg-1p1d-dep8-stp-fixed-seq"
2+
3+
model:
4+
path: "hf:deepseek-ai/DeepSeek-V4-Pro"
5+
container: "sglang-rocm-dsv4-mi355x"
6+
precision: "fp4"
7+
8+
identity:
9+
model: {repo: "deepseek-ai/DeepSeek-V4-Pro"}
10+
container: {image: "lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719"}
11+
frameworks:
12+
sglang: "0.5.15.post1.dev20260719+ga03ca46a28"
13+
sglang-router: "0.3.2"
14+
amd-mori: "0.5.15.post1.dev20260719+ga03ca46a28"
15+
16+
slurm: {time_limit: "04:00:00"}
17+
18+
resources:
19+
gpu_type: "mi355x"
20+
gpus_per_node: 8
21+
prefill_nodes: 1
22+
decode_nodes: 1
23+
prefill_workers: 1
24+
decode_workers: 1
25+
gpus_per_prefill: 8
26+
gpus_per_decode: 8
27+
28+
frontend:
29+
type: sglang
30+
enable_multiple_frontends: false
31+
args:
32+
policy: consistent_hashing
33+
dp-aware: true
34+
cache-threshold: 0.3
35+
balance-abs-threshold: 2
36+
balance-rel-threshold: 1.1
37+
disable-circuit-breaker: true
38+
health-failure-threshold: 100
39+
health-check-timeout-secs: 600
40+
health-check-interval-secs: 30
41+
42+
backend:
43+
type: sglang
44+
prefill_environment: &common_environment
45+
HF_HOME: "/hf_hub_cache"
46+
HF_HUB_CACHE: "/hf_hub_cache/hub"
47+
HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub"
48+
PYTHONUNBUFFERED: "1"
49+
TORCH_NCCL_BLOCKING_WAIT: "1"
50+
NCCL_BLOCKING_WAIT: "1"
51+
HSA_NO_SCRATCH_RECLAIM: "0"
52+
GPU_MAX_HW_QUEUES: "5"
53+
SGLANG_AITER_MLA_PERSIST: "0"
54+
SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
55+
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
56+
SGLANG_MORI_DISPATCH_DTYPE: "auto"
57+
SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: "4096"
58+
SGLANG_OPT_DEEPGEMM_HC_PRENORM: "false"
59+
SGLANG_USE_AITER: "1"
60+
SGLANG_USE_ROCM700A: "0"
61+
SGLANG_OPT_USE_FUSED_COMPRESS: "true"
62+
SGLANG_HACK_FLASHMLA_BACKEND: "unified_kv_triton"
63+
SGLANG_OPT_FP8_WO_A_GEMM: "false"
64+
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "false"
65+
SGLANG_OPT_USE_TOPK_V2: "false"
66+
SGLANG_OPT_USE_AITER_INDEXER: "true"
67+
SGLANG_OPT_USE_TILELANG_INDEXER: "false"
68+
SGLANG_OPT_USE_TILELANG_MHC_PRE: "false"
69+
SGLANG_OPT_USE_TILELANG_MHC_POST: "false"
70+
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH: "1"
71+
SGLANG_OPT_USE_FUSED_COMPRESS_TRITON: "true"
72+
SGLANG_OPT_USE_MULTI_STREAM_OVERLAP: "false"
73+
SGLANG_ROCM_USE_MULTI_STREAM: "false"
74+
AITER_BF16_FP8_MOE_BOUND: "0"
75+
SGLANG_EAGER_INPUT_NO_COPY: "true"
76+
SGLANG_SHARED_EXPERT_TP1: "1"
77+
SGLANG_DP_SHARED_EXPERT_LOCAL: "1"
78+
SGLANG_DP_USE_GATHERV: "1"
79+
SGLANG_DP_USE_REDUCE_SCATTER: "1"
80+
IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7"
81+
MORI_RDMA_TC: "104"
82+
MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000"
83+
MORI_IO_QP_MAX_SEND_WR: "32767"
84+
SGLANG_MORI_COMBINE_DTYPE: "auto"
85+
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "16384"
86+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800"
87+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800"
88+
# The native router may select a different DPA rank from the rank encoded
89+
# in bootstrap_room. Use SGLang's supported rank-query fallback so decode
90+
# retrieves the actual prefill rank instead of rejecting the KV transfer.
91+
SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK: "1"
92+
decode_environment:
93+
<<: *common_environment
94+
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024"
95+
sglang_config:
96+
prefill: &common_server
97+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
98+
trust-remote-code: true
99+
tp-size: 8
100+
ep-size: 8
101+
dp-size: 8
102+
enable-dp-attention: true
103+
enable-prefill-delayer: true
104+
ep-dispatch-algorithm: fake
105+
moe-a2a-backend: mori
106+
deepep-mode: normal
107+
moe-dense-tp-size: 1
108+
enable-dp-lm-head: true
109+
watchdog-timeout: 3600
110+
load-balance-method: auto
111+
kv-cache-dtype: fp8_e4m3
112+
attention-backend: dsv4
113+
page-size: 256
114+
swa-full-tokens-ratio: 0.1
115+
disable-shared-experts-fusion: true
116+
tool-call-parser: deepseekv4
117+
reasoning-parser: deepseek-v4
118+
disaggregation-transfer-backend: mori
119+
log-level: info
120+
log-level-http: error
121+
context-length: 9217
122+
enable-metrics: true
123+
mem-fraction-static: 0.80
124+
max-running-requests: 1024
125+
chunked-prefill-size: 131072
126+
max-total-tokens: 2097152
127+
disable-radix-cache: true
128+
disable-cuda-graph: true
129+
decode:
130+
<<: *common_server
131+
mem-fraction-static: 0.85
132+
max-total-tokens: null
133+
disable-radix-cache: false
134+
disable-cuda-graph: false
135+
prefill-round-robin-balance: true
136+
cuda-graph-bs: [1, 2, 4, 8, 16, 32, 64, 128]
137+
138+
sbatch_directives:
139+
cpus-per-task: "128"
140+
mem: "0"
141+
142+
srun_options:
143+
mem: "0"
144+
container-writable: ""
145+
container-remap-root: ""
146+
147+
health_check: {max_attempts: 720, interval_seconds: 5}
148+
149+
benchmark:
150+
type: custom
151+
command: |
152+
set -euo pipefail
153+
result_root="/results/${SLURM_JOB_ID}/fixed-seq"
154+
mkdir -p "${result_root}"
155+
trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT
156+
for concurrency in 256 512; do
157+
python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
158+
--backend openai \
159+
--base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
160+
--endpoint /v1/completions \
161+
--model deepseek-ai/DeepSeek-V4-Pro \
162+
--tokenizer deepseek-ai/DeepSeek-V4-Pro \
163+
--trust-remote-code \
164+
--dataset-name random \
165+
--random-input-len 8192 \
166+
--random-output-len 1024 \
167+
--random-range-ratio 1.0 \
168+
--random-num-workers 1 \
169+
--num-warmups "$((concurrency * 2))" \
170+
--num-prompts "$((concurrency * 10))" \
171+
--max-concurrency "${concurrency}" \
172+
--request-rate inf \
173+
--ignore-eos \
174+
--disable-tqdm \
175+
--save-result \
176+
--result-dir "${result_root}" \
177+
--result-filename "dsv4-fp4-disagg-stp-dep8-isl8192-osl1024-c${concurrency}.json"
178+
done
179+
env:
180+
HF_HOME: /hf_hub_cache
181+
HF_HUB_CACHE: /hf_hub_cache/hub
182+
HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub

0 commit comments

Comments
 (0)