Skip to content

Commit 1767220

Browse files
adibarraadibarra
committed
chore(recipes): keep retired 1k1k recipes for reference
Restore the 35 multi-node recipes under the retired 1k1k directories. They are not referenced by any master config but are kept as a historical record.
1 parent 2e3884b commit 1767220

35 files changed

Lines changed: 3866 additions & 0 deletions

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_maxtpt[0]): 1p1d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.75).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (<model-prefix>/<engine>/<gpu>-<precision>/<workload>/<recipe>.yaml).
5+
6+
schema: 2
7+
name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d
8+
model:
9+
path: dsr1
10+
container: dynamo-sglang
11+
precision: fp4
12+
13+
dynamo:
14+
install: true
15+
16+
source:
17+
rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
18+
resources:
19+
gpu_type: b200
20+
gpus_per_node: 8
21+
engine: sglang
22+
roles:
23+
prefill:
24+
nodes: 1
25+
workers: 1
26+
gpus: 4
27+
env:
28+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
29+
PYTHONUNBUFFERED: '1'
30+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
31+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
32+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
33+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
34+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
35+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
36+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
37+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
38+
MC_FORCE_MNNVL: '1'
39+
NCCL_MNNVL_ENABLE: '1'
40+
NCCL_CUMEM_ENABLE: '1'
41+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
42+
DYN_REQUEST_PLANE: nats
43+
SGLANG_ENABLE_SPEC_V2: '1'
44+
args:
45+
served-model-name: deepseek-ai/DeepSeek-R1
46+
trust-remote-code: true
47+
quantization: modelopt_fp4
48+
disaggregation-mode: prefill
49+
disaggregation-transfer-backend: nixl
50+
mem-fraction-static: 0.85
51+
max-prefill-tokens: 32768
52+
chunked-prefill-size: 32768
53+
context-length: 2200
54+
max-running-requests: 1024
55+
disable-cuda-graph: true
56+
tensor-parallel-size: 4
57+
data-parallel-size: 4
58+
expert-parallel-size: 4
59+
enable-dp-attention: true
60+
enable-dp-lm-head: true
61+
attention-backend: trtllm_mla
62+
kv-cache-dtype: fp8_e4m3
63+
moe-runner-backend: flashinfer_trtllm
64+
moe-dense-tp-size: 1
65+
stream-interval: 30
66+
watchdog-timeout: 1000000
67+
enable-flashinfer-allreduce-fusion: true
68+
disable-radix-cache: true
69+
fp4-gemm-backend: flashinfer_trtllm
70+
decode:
71+
nodes: 1
72+
workers: 1
73+
env:
74+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
75+
PYTHONUNBUFFERED: '1'
76+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
77+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
78+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
79+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
80+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
81+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
82+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
83+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
84+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
85+
MC_FORCE_MNNVL: '1'
86+
NCCL_MNNVL_ENABLE: '1'
87+
NCCL_CUMEM_ENABLE: '1'
88+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
89+
DYN_REQUEST_PLANE: nats
90+
SGLANG_MOE_NVFP4_DISPATCH: '1'
91+
SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass
92+
SGLANG_ENABLE_SPEC_V2: '1'
93+
args:
94+
served-model-name: deepseek-ai/DeepSeek-R1
95+
trust-remote-code: true
96+
quantization: modelopt_fp4
97+
disaggregation-mode: decode
98+
disaggregation-transfer-backend: nixl
99+
mem-fraction-static: 0.75
100+
max-prefill-tokens: 32768
101+
chunked-prefill-size: 32768
102+
context-length: 2200
103+
max-running-requests: 1024
104+
cuda-graph-max-bs: 1024
105+
tensor-parallel-size: 8
106+
data-parallel-size: 8
107+
expert-parallel-size: 8
108+
attention-backend: trtllm_mla
109+
kv-cache-dtype: fp8_e4m3
110+
moe-runner-backend: flashinfer_trtllm
111+
stream-interval: 30
112+
watchdog-timeout: 1000000
113+
enable-flashinfer-allreduce-fusion: true
114+
disable-radix-cache: true
115+
fp4-gemm-backend: flashinfer_trtllm
116+
enable-dp-attention: true
117+
enable-dp-lm-head: true
118+
moe-dense-tp-size: 1
119+
speculative-algorithm: EAGLE
120+
speculative-num-steps: 2
121+
speculative-eagle-topk: 1
122+
speculative-num-draft-tokens: 3
123+
health_check:
124+
max_attempts: 360
125+
interval_seconds: 10
126+
benchmark:
127+
type: sa-bench
128+
isl: 1024
129+
osl: 1024
130+
req_rate: inf
131+
concurrencies: 512x1024
132+
use_chat_template: true

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_maxtpt[1]): 1p2d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.85).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (<model-prefix>/<engine>/<gpu>-<precision>/<workload>/<recipe>.yaml).
5+
6+
schema: 2
7+
name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d
8+
model:
9+
path: dsr1
10+
container: dynamo-sglang
11+
precision: fp4
12+
13+
dynamo:
14+
install: true
15+
16+
source:
17+
rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
18+
resources:
19+
gpu_type: b200
20+
gpus_per_node: 8
21+
engine: sglang
22+
roles:
23+
prefill:
24+
nodes: 1
25+
workers: 1
26+
gpus: 4
27+
env:
28+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
29+
PYTHONUNBUFFERED: '1'
30+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
31+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
32+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
33+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
34+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
35+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
36+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
37+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
38+
MC_FORCE_MNNVL: '1'
39+
NCCL_MNNVL_ENABLE: '1'
40+
NCCL_CUMEM_ENABLE: '1'
41+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
42+
DYN_REQUEST_PLANE: nats
43+
SGLANG_ENABLE_SPEC_V2: '1'
44+
args:
45+
served-model-name: deepseek-ai/DeepSeek-R1
46+
trust-remote-code: true
47+
quantization: modelopt_fp4
48+
disaggregation-mode: prefill
49+
disaggregation-transfer-backend: nixl
50+
mem-fraction-static: 0.85
51+
max-prefill-tokens: 32768
52+
chunked-prefill-size: 32768
53+
context-length: 2200
54+
max-running-requests: 512
55+
disable-cuda-graph: true
56+
tensor-parallel-size: 4
57+
data-parallel-size: 4
58+
expert-parallel-size: 4
59+
enable-dp-attention: true
60+
enable-dp-lm-head: true
61+
attention-backend: trtllm_mla
62+
kv-cache-dtype: fp8_e4m3
63+
moe-runner-backend: flashinfer_trtllm
64+
moe-dense-tp-size: 1
65+
stream-interval: 30
66+
watchdog-timeout: 1000000
67+
enable-flashinfer-allreduce-fusion: true
68+
disable-radix-cache: true
69+
fp4-gemm-backend: flashinfer_trtllm
70+
decode:
71+
nodes: 2
72+
workers: 2
73+
env:
74+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
75+
PYTHONUNBUFFERED: '1'
76+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
77+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
78+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
79+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
80+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
81+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
82+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
83+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
84+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
85+
MC_FORCE_MNNVL: '1'
86+
NCCL_MNNVL_ENABLE: '1'
87+
NCCL_CUMEM_ENABLE: '1'
88+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
89+
DYN_REQUEST_PLANE: nats
90+
SGLANG_MOE_NVFP4_DISPATCH: '1'
91+
SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass
92+
SGLANG_ENABLE_SPEC_V2: '1'
93+
args:
94+
served-model-name: deepseek-ai/DeepSeek-R1
95+
trust-remote-code: true
96+
quantization: modelopt_fp4
97+
disaggregation-mode: decode
98+
disaggregation-transfer-backend: nixl
99+
mem-fraction-static: 0.85
100+
max-prefill-tokens: 32768
101+
chunked-prefill-size: 32768
102+
context-length: 2200
103+
max-running-requests: 512
104+
cuda-graph-max-bs: 512
105+
tensor-parallel-size: 8
106+
data-parallel-size: 8
107+
expert-parallel-size: 8
108+
attention-backend: trtllm_mla
109+
kv-cache-dtype: fp8_e4m3
110+
moe-runner-backend: flashinfer_trtllm
111+
stream-interval: 30
112+
watchdog-timeout: 1000000
113+
enable-flashinfer-allreduce-fusion: true
114+
disable-radix-cache: true
115+
fp4-gemm-backend: flashinfer_trtllm
116+
enable-dp-attention: true
117+
enable-dp-lm-head: true
118+
moe-dense-tp-size: 1
119+
speculative-algorithm: EAGLE
120+
speculative-num-steps: 2
121+
speculative-eagle-topk: 1
122+
speculative-num-draft-tokens: 3
123+
health_check:
124+
max_attempts: 360
125+
interval_seconds: 10
126+
benchmark:
127+
type: sa-bench
128+
isl: 1024
129+
osl: 1024
130+
req_rate: inf
131+
concurrencies: '512'
132+
use_chat_template: true

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_lowlat[0]): 1p5d low-latency (dep4 prefill / tep8 decode, 5 decode nodes).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (<model-prefix>/<engine>/<gpu>-<precision>/<workload>/<recipe>.yaml).
5+
6+
schema: 2
7+
name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d
8+
model:
9+
path: dsr1
10+
container: dynamo-sglang
11+
precision: fp4
12+
13+
dynamo:
14+
install: true
15+
16+
source:
17+
rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
18+
resources:
19+
gpu_type: b200
20+
gpus_per_node: 8
21+
engine: sglang
22+
roles:
23+
prefill:
24+
nodes: 1
25+
workers: 1
26+
gpus: 4
27+
env:
28+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
29+
PYTHONUNBUFFERED: '1'
30+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
31+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
32+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
33+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
34+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
35+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
36+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
37+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
38+
MC_FORCE_MNNVL: '1'
39+
NCCL_MNNVL_ENABLE: '1'
40+
NCCL_CUMEM_ENABLE: '1'
41+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
42+
DYN_REQUEST_PLANE: nats
43+
SGLANG_ENABLE_SPEC_V2: '1'
44+
args:
45+
served-model-name: deepseek-ai/DeepSeek-R1
46+
trust-remote-code: true
47+
quantization: modelopt_fp4
48+
disaggregation-mode: prefill
49+
disaggregation-transfer-backend: nixl
50+
mem-fraction-static: 0.85
51+
max-prefill-tokens: 32768
52+
chunked-prefill-size: 32768
53+
context-length: 2200
54+
max-running-requests: 512
55+
disable-cuda-graph: true
56+
tensor-parallel-size: 4
57+
data-parallel-size: 4
58+
expert-parallel-size: 4
59+
enable-dp-attention: true
60+
enable-dp-lm-head: true
61+
attention-backend: trtllm_mla
62+
kv-cache-dtype: fp8_e4m3
63+
moe-runner-backend: flashinfer_trtllm
64+
moe-dense-tp-size: 1
65+
stream-interval: 30
66+
watchdog-timeout: 1000000
67+
enable-flashinfer-allreduce-fusion: true
68+
disable-radix-cache: true
69+
fp4-gemm-backend: flashinfer_trtllm
70+
decode:
71+
nodes: 5
72+
workers: 5
73+
env:
74+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
75+
PYTHONUNBUFFERED: '1'
76+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
77+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
78+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
79+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
80+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
81+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
82+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
83+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
84+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
85+
MC_FORCE_MNNVL: '1'
86+
NCCL_MNNVL_ENABLE: '1'
87+
NCCL_CUMEM_ENABLE: '1'
88+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
89+
DYN_REQUEST_PLANE: nats
90+
SGLANG_ENABLE_SPEC_V2: '1'
91+
args:
92+
served-model-name: deepseek-ai/DeepSeek-R1
93+
trust-remote-code: true
94+
quantization: modelopt_fp4
95+
disaggregation-mode: decode
96+
disaggregation-transfer-backend: nixl
97+
mem-fraction-static: 0.85
98+
max-prefill-tokens: 32768
99+
chunked-prefill-size: 32768
100+
context-length: 2200
101+
max-running-requests: 512
102+
cuda-graph-max-bs: 512
103+
tensor-parallel-size: 8
104+
data-parallel-size: 1
105+
expert-parallel-size: 8
106+
attention-backend: trtllm_mla
107+
kv-cache-dtype: fp8_e4m3
108+
moe-runner-backend: flashinfer_trtllm
109+
stream-interval: 30
110+
watchdog-timeout: 1000000
111+
enable-flashinfer-allreduce-fusion: true
112+
disable-radix-cache: true
113+
fp4-gemm-backend: flashinfer_trtllm
114+
speculative-algorithm: EAGLE
115+
speculative-num-steps: 2
116+
speculative-eagle-topk: 1
117+
speculative-num-draft-tokens: 3
118+
health_check:
119+
max_attempts: 360
120+
interval_seconds: 10
121+
benchmark:
122+
type: sa-bench
123+
isl: 1024
124+
osl: 1024
125+
req_rate: inf
126+
concurrencies: 16x512
127+
use_chat_template: true

0 commit comments

Comments
 (0)