Skip to content

Commit 6450c4d

Browse files
seungrokjseungrokj
authored
Merge branch 'main' into amd/agentx_dsv4_vllm_mtp_0728
2 parents 8ee9d1a + 1766c9d commit 6450c4d

12 files changed

Lines changed: 935 additions & 14 deletions

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: kimi-k2.6-vllm-disagg-b200-1p1d-dep4-dep8-c1024
2+
model:
3+
path: kimi-k2.6-nvfp4
4+
container: vllm/vllm-openai:v0.25.1
5+
precision: fp4
6+
dynamo:
7+
wheel: 1.3.0.dev20260721
8+
install: true
9+
resources:
10+
gpu_type: b200
11+
gpus_per_node: 8
12+
prefill_nodes: 1
13+
decode_nodes: 1
14+
prefill_workers: 1
15+
decode_workers: 1
16+
gpus_per_prefill: 4
17+
gpus_per_decode: 8
18+
infra:
19+
etcd_nats_dedicated_node: true
20+
frontend:
21+
type: dynamo
22+
enable_multiple_frontends: false
23+
backend:
24+
type: vllm
25+
connector: null
26+
dp_launch_mode: per_gpu
27+
prefill_environment:
28+
PYTHONUNBUFFERED: '1'
29+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
30+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
31+
VLLM_USE_NCCL_SYMM_MEM: '0'
32+
NCCL_CUMEM_ENABLE: '1'
33+
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900'
34+
NCCL_WATCHDOG_TIMEOUT: '1800'
35+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
36+
TORCH_CUDA_ARCH_LIST: '10.0'
37+
PYTHONNOUSERSITE: '1'
38+
UCX_MEMTYPE_CACHE: n
39+
UCX_MEMTYPE_REG_WHOLE: n
40+
UCX_CUDA_IPC_ENABLE_MNNVL: n
41+
UCX_MAX_RMA_RAILS: '1'
42+
UCX_MAX_RNDV_RAILS: '1'
43+
UCX_RNDV_SCHEME: put_zcopy
44+
decode_environment:
45+
PYTHONUNBUFFERED: '1'
46+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
47+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
48+
VLLM_USE_NCCL_SYMM_MEM: '0'
49+
NCCL_CUMEM_ENABLE: '1'
50+
NCCL_WATCHDOG_TIMEOUT: '1800'
51+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
52+
TORCH_CUDA_ARCH_LIST: '10.0'
53+
PYTHONNOUSERSITE: '1'
54+
UCX_MEMTYPE_CACHE: n
55+
UCX_MEMTYPE_REG_WHOLE: n
56+
UCX_CUDA_IPC_ENABLE_MNNVL: n
57+
UCX_MAX_RMA_RAILS: '1'
58+
UCX_MAX_RNDV_RAILS: '1'
59+
UCX_RNDV_SCHEME: put_zcopy
60+
vllm_config:
61+
prefill:
62+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}'
63+
served-model-name: nvidia/Kimi-K2.6-NVFP4
64+
kv-cache-dtype: fp8
65+
tensor-parallel-size: 1
66+
pipeline-parallel-size: 1
67+
data-parallel-size: 4
68+
data-parallel-rpc-port: 13346
69+
enable-expert-parallel: true
70+
max-model-len: 10240
71+
max-num-seqs: 2048
72+
enforce-eager: true
73+
compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
74+
max-num-batched-tokens: 8192
75+
safetensors-load-strategy: prefetch
76+
trust-remote-code: true
77+
no-enable-prefix-caching: true
78+
no-enable-flashinfer-autotune: true
79+
attention-backend: FLASHINFER_MLA
80+
block-size: 128
81+
attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}'
82+
gpu-memory-utilization: 0.94
83+
decode:
84+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
85+
served-model-name: nvidia/Kimi-K2.6-NVFP4
86+
kv-cache-dtype: fp8
87+
tensor-parallel-size: 1
88+
pipeline-parallel-size: 1
89+
data-parallel-size: 8
90+
data-parallel-rpc-port: 13345
91+
enable-expert-parallel: true
92+
max-model-len: 10240
93+
max-num-seqs: 2048
94+
max-num-batched-tokens: 8192
95+
safetensors-load-strategy: prefetch
96+
trust-remote-code: true
97+
no-enable-prefix-caching: true
98+
no-enable-flashinfer-autotune: true
99+
async-scheduling: true
100+
attention-backend: FLASHINFER_MLA
101+
block-size: 128
102+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
103+
gpu-memory-utilization: 0.9
104+
stream-interval: 50
105+
max-cudagraph-capture-size: 1024
106+
benchmark:
107+
type: sa-bench
108+
isl: 8192
109+
osl: 1024
110+
concurrencies: '1024'
111+
req_rate: inf
112+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048
2+
model:
3+
path: kimi-k2.6-nvfp4
4+
container: vllm/vllm-openai:v0.25.1
5+
precision: fp4
6+
dynamo:
7+
wheel: 1.3.0.dev20260721
8+
install: true
9+
resources:
10+
gpu_type: b200
11+
gpus_per_node: 8
12+
prefill_nodes: 1
13+
decode_nodes: 1
14+
prefill_workers: 1
15+
decode_workers: 1
16+
gpus_per_prefill: 8
17+
gpus_per_decode: 8
18+
infra:
19+
etcd_nats_dedicated_node: true
20+
frontend:
21+
type: dynamo
22+
enable_multiple_frontends: false
23+
backend:
24+
type: vllm
25+
connector: null
26+
dp_launch_mode: per_gpu
27+
prefill_environment:
28+
PYTHONUNBUFFERED: '1'
29+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
30+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
31+
VLLM_USE_NCCL_SYMM_MEM: '0'
32+
NCCL_CUMEM_ENABLE: '1'
33+
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900'
34+
NCCL_WATCHDOG_TIMEOUT: '1800'
35+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
36+
TORCH_CUDA_ARCH_LIST: '10.0'
37+
PYTHONNOUSERSITE: '1'
38+
UCX_MEMTYPE_CACHE: n
39+
UCX_MEMTYPE_REG_WHOLE: n
40+
UCX_CUDA_IPC_ENABLE_MNNVL: n
41+
UCX_MAX_RMA_RAILS: '1'
42+
UCX_MAX_RNDV_RAILS: '1'
43+
UCX_RNDV_SCHEME: put_zcopy
44+
decode_environment:
45+
PYTHONUNBUFFERED: '1'
46+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
47+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
48+
VLLM_USE_NCCL_SYMM_MEM: '0'
49+
NCCL_CUMEM_ENABLE: '1'
50+
NCCL_WATCHDOG_TIMEOUT: '1800'
51+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
52+
TORCH_CUDA_ARCH_LIST: '10.0'
53+
PYTHONNOUSERSITE: '1'
54+
UCX_MEMTYPE_CACHE: n
55+
UCX_MEMTYPE_REG_WHOLE: n
56+
UCX_CUDA_IPC_ENABLE_MNNVL: n
57+
UCX_MAX_RMA_RAILS: '1'
58+
UCX_MAX_RNDV_RAILS: '1'
59+
UCX_RNDV_SCHEME: put_zcopy
60+
vllm_config:
61+
prefill:
62+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}'
63+
served-model-name: nvidia/Kimi-K2.6-NVFP4
64+
kv-cache-dtype: fp8
65+
tensor-parallel-size: 1
66+
pipeline-parallel-size: 1
67+
data-parallel-size: 8
68+
data-parallel-rpc-port: 13346
69+
enable-expert-parallel: true
70+
max-model-len: 10240
71+
max-num-seqs: 2048
72+
enforce-eager: true
73+
compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
74+
max-num-batched-tokens: 16384
75+
safetensors-load-strategy: prefetch
76+
trust-remote-code: true
77+
no-enable-prefix-caching: true
78+
no-enable-flashinfer-autotune: true
79+
attention-backend: FLASHINFER_MLA
80+
block-size: 128
81+
attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}'
82+
gpu-memory-utilization: 0.9
83+
decode:
84+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
85+
served-model-name: nvidia/Kimi-K2.6-NVFP4
86+
kv-cache-dtype: fp8
87+
tensor-parallel-size: 1
88+
pipeline-parallel-size: 1
89+
data-parallel-size: 8
90+
data-parallel-rpc-port: 13345
91+
enable-expert-parallel: true
92+
max-model-len: 10240
93+
max-num-seqs: 2048
94+
max-num-batched-tokens: 8192
95+
safetensors-load-strategy: prefetch
96+
trust-remote-code: true
97+
no-enable-prefix-caching: true
98+
no-enable-flashinfer-autotune: true
99+
async-scheduling: true
100+
attention-backend: FLASHINFER_MLA
101+
block-size: 128
102+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
103+
gpu-memory-utilization: 0.9
104+
stream-interval: 50
105+
max-cudagraph-capture-size: 1024
106+
benchmark:
107+
type: sa-bench
108+
isl: 8192
109+
osl: 1024
110+
concurrencies: '2048'
111+
req_rate: inf
112+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1
2+
model:
3+
path: kimi-k2.6-nvfp4
4+
container: vllm/vllm-openai:v0.25.1
5+
precision: fp4
6+
dynamo:
7+
wheel: 1.3.0.dev20260721
8+
install: true
9+
resources:
10+
gpu_type: b200
11+
gpus_per_node: 8
12+
prefill_nodes: 1
13+
decode_nodes: 1
14+
prefill_workers: 1
15+
decode_workers: 1
16+
gpus_per_prefill: 8
17+
gpus_per_decode: 8
18+
infra:
19+
etcd_nats_dedicated_node: true
20+
frontend:
21+
type: dynamo
22+
enable_multiple_frontends: false
23+
backend:
24+
type: vllm
25+
connector: null
26+
dp_launch_mode: per_gpu
27+
prefill_environment:
28+
PYTHONUNBUFFERED: '1'
29+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
30+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
31+
VLLM_USE_NCCL_SYMM_MEM: '0'
32+
NCCL_CUMEM_ENABLE: '1'
33+
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900'
34+
NCCL_WATCHDOG_TIMEOUT: '1800'
35+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
36+
TORCH_CUDA_ARCH_LIST: '10.0'
37+
PYTHONNOUSERSITE: '1'
38+
UCX_MEMTYPE_CACHE: n
39+
UCX_MEMTYPE_REG_WHOLE: n
40+
UCX_CUDA_IPC_ENABLE_MNNVL: n
41+
UCX_MAX_RMA_RAILS: '1'
42+
UCX_MAX_RNDV_RAILS: '1'
43+
UCX_RNDV_SCHEME: put_zcopy
44+
decode_environment:
45+
PYTHONUNBUFFERED: '1'
46+
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0'
47+
VLLM_USE_FLASHINFER_MOE_FP4: '1'
48+
VLLM_USE_NCCL_SYMM_MEM: '0'
49+
NCCL_CUMEM_ENABLE: '0'
50+
NCCL_WATCHDOG_TIMEOUT: '1800'
51+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800'
52+
TORCH_CUDA_ARCH_LIST: '10.0'
53+
PYTHONNOUSERSITE: '1'
54+
UCX_MEMTYPE_CACHE: n
55+
UCX_MEMTYPE_REG_WHOLE: n
56+
UCX_CUDA_IPC_ENABLE_MNNVL: n
57+
UCX_MAX_RMA_RAILS: '1'
58+
UCX_MAX_RNDV_RAILS: '1'
59+
UCX_RNDV_SCHEME: put_zcopy
60+
vllm_config:
61+
prefill:
62+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}'
63+
served-model-name: nvidia/Kimi-K2.6-NVFP4
64+
kv-cache-dtype: fp8
65+
tensor-parallel-size: 1
66+
pipeline-parallel-size: 1
67+
data-parallel-size: 8
68+
data-parallel-rpc-port: 13346
69+
enable-expert-parallel: true
70+
max-model-len: 10240
71+
max-num-seqs: 2048
72+
enforce-eager: true
73+
compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
74+
max-num-batched-tokens: 16384
75+
safetensors-load-strategy: prefetch
76+
trust-remote-code: true
77+
no-enable-prefix-caching: true
78+
no-enable-flashinfer-autotune: true
79+
attention-backend: FLASHINFER_MLA
80+
block-size: 128
81+
attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}'
82+
gpu-memory-utilization: 0.9
83+
decode:
84+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
85+
served-model-name: nvidia/Kimi-K2.6-NVFP4
86+
kv-cache-dtype: fp8
87+
tensor-parallel-size: 8
88+
pipeline-parallel-size: 1
89+
disable-custom-all-reduce: true
90+
max-model-len: 9216
91+
max-num-seqs: 1024
92+
max-num-batched-tokens: 8192
93+
safetensors-load-strategy: prefetch
94+
trust-remote-code: true
95+
no-enable-prefix-caching: true
96+
no-enable-flashinfer-autotune: true
97+
async-scheduling: true
98+
attention-backend: FLASHINFER_MLA
99+
block-size: 128
100+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}'
101+
gpu-memory-utilization: 0.9
102+
stream-interval: 50
103+
max-cudagraph-capture-size: 256
104+
benchmark:
105+
type: sa-bench
106+
isl: 8192
107+
osl: 1024
108+
concurrencies: '1'
109+
req_rate: inf
110+
use_chat_template: true

0 commit comments

Comments
 (0)