Skip to content

Commit df96f2a

Browse files
seungrokjseungrokj
authored
Merge branch 'main' into amd/kimik3-mi355x-vllm-1012
2 parents 4823086 + 9fa7658 commit df96f2a

17 files changed

Lines changed: 354 additions & 27 deletions

File tree

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,8 @@ env:
166166
PREFILL_PP_SIZE: ${{ fromJSON(inputs.config).prefill.pp }}
167167
PREFILL_DCP_SIZE: ${{ fromJSON(inputs.config).prefill.dcp-size }}
168168
PREFILL_PCP_SIZE: ${{ fromJSON(inputs.config).prefill.pcp-size }}
169+
PREFILL_DWDP_SIZE: ${{ fromJSON(inputs.config).prefill.dwdp-size || 1 }}
170+
DWDP_ARTIFACT_SUFFIX: ${{ fromJSON(inputs.config).prefill.dwdp-size > 1 && format('-dwdp{0}', fromJSON(inputs.config).prefill.dwdp-size) || '' }}
169171
PREFILL_EP: ${{ fromJSON(inputs.config).prefill.ep }}
170172
PREFILL_DP_ATTN: ${{ fromJSON(inputs.config).prefill.dp-attn }}
171173

@@ -296,7 +298,7 @@ jobs:
296298
DECODE_ADDITIONAL_SETTINGS: ${{ toJSON(fromJSON(inputs.config).decode.additional-settings) }}
297299
RUNNER_NAME: ${{ runner.name }}
298300
RUNNER_TYPE: ${{ inputs.runner }}
299-
RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_prefill-tp${{ env.PREFILL_TP }}-pp${{ env.PREFILL_PP_SIZE }}-dcp${{ env.PREFILL_DCP_SIZE }}-pcp${{ env.PREFILL_PCP_SIZE }}-ep${{ env.PREFILL_EP }}-dp${{ env.PREFILL_DP_ATTN }}-nw${{ env.PREFILL_NUM_WORKERS }}_decode-tp${{ env.DECODE_TP }}-pp${{ env.DECODE_PP_SIZE }}-dcp${{ env.DECODE_DCP_SIZE }}-pcp${{ env.DECODE_PCP_SIZE }}-ep${{ env.DECODE_EP }}-dp${{ env.DECODE_DP_ATTN }}-nw${{ env.DECODE_NUM_WORKERS }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }}
301+
RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_prefill-tp${{ env.PREFILL_TP }}-pp${{ env.PREFILL_PP_SIZE }}-dcp${{ env.PREFILL_DCP_SIZE }}-pcp${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}-ep${{ env.PREFILL_EP }}-dp${{ env.PREFILL_DP_ATTN }}-nw${{ env.PREFILL_NUM_WORKERS }}_decode-tp${{ env.DECODE_TP }}-pp${{ env.DECODE_PP_SIZE }}-dcp${{ env.DECODE_DCP_SIZE }}-pcp${{ env.DECODE_PCP_SIZE }}-ep${{ env.DECODE_EP }}-dp${{ env.DECODE_DP_ATTN }}-nw${{ env.DECODE_NUM_WORKERS }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }}
300302
run: |
301303
if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
302304
export GITHUB_WORKSPACE="$PWD"
@@ -533,7 +535,7 @@ jobs:
533535
continue-on-error: true
534536
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
535537
with:
536-
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
538+
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
537539
path: |
538540
${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
539541
${{ env.INFERENCEX_E2E_ROOT }}/results*.json
@@ -558,7 +560,7 @@ jobs:
558560
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
559561
with:
560562
overwrite: true
561-
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
563+
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
562564
path: |
563565
${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
564566
${{ env.INFERENCEX_E2E_ROOT }}/results*.json
@@ -581,7 +583,7 @@ jobs:
581583
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
582584
with:
583585
overwrite: true
584-
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
586+
name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
585587
path: |
586588
${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
587589
${{ env.INFERENCEX_E2E_ROOT }}/results*.json

‎inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml‎

Lines changed: 108 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ base:
55
name: dsv41flash-fp4-mi355x-sglang-agentic
66
model:
77
path: hf:deepseek-ai/DeepSeek-V4.1-Flash
8-
container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261006
8+
container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012
99
precision: fp4
1010
resources:
1111
gpu_type: mi355x
@@ -53,7 +53,8 @@ base:
5353
SGLANG_USE_AITER: '1'
5454
SGLANG_USE_ROCM700A: '0'
5555
TRITON_HIP_USE_ASYNC_COPY: '0'
56-
ROCM_QUICK_REDUCE_QUANTIZATION: NONE
56+
ROCM_QUICK_REDUCE_QUANTIZATION: INT8
57+
SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL: 'true'
5758
AITER_BF16_FP8_MOE_BOUND: '0'
5859
TORCH_BLAS_PREFER_HIPBLASLT: '1'
5960
HSA_NO_SCRATCH_RECLAIM: '0'
@@ -69,7 +70,9 @@ base:
6970
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
7071
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
7172

72-
# One variant per point; the saturation point gets a longer warmup drain.
73+
# One variant per point; the saturation point gets a longer warmup drain. A longer
74+
# prefill-decode interval trades TTFT for P90 ITL: 192 with lpm up to c8, 96 at c16
75+
# and 32 at c32. amd-master.yaml runs c32 and c64 on the DP2 variants only.
7376
override_tp2_c1:
7477
roles:
7578
agg:
@@ -78,7 +81,8 @@ override_tp2_c1:
7881
tensor-parallel-size: 2
7982
mem-fraction-static: 0.70
8083
chunked-prefill-size: 16384
81-
prefill-decode-interval: 16
84+
prefill-decode-interval: 192
85+
schedule-policy: lpm
8286
swa-prefix-tails: 128
8387
max-running-requests: 2
8488
benchmark:
@@ -93,7 +97,8 @@ override_tp2_c2:
9397
tensor-parallel-size: 2
9498
mem-fraction-static: 0.70
9599
chunked-prefill-size: 16384
96-
prefill-decode-interval: 16
100+
prefill-decode-interval: 192
101+
schedule-policy: lpm
97102
swa-prefix-tails: 128
98103
max-running-requests: 4
99104
benchmark:
@@ -108,7 +113,8 @@ override_tp2_c4:
108113
tensor-parallel-size: 2
109114
mem-fraction-static: 0.70
110115
chunked-prefill-size: 16384
111-
prefill-decode-interval: 16
116+
prefill-decode-interval: 192
117+
schedule-policy: lpm
112118
swa-prefix-tails: 256
113119
max-running-requests: 8
114120
benchmark:
@@ -123,7 +129,8 @@ override_tp2_c8:
123129
tensor-parallel-size: 2
124130
mem-fraction-static: 0.70
125131
chunked-prefill-size: 16384
126-
prefill-decode-interval: 16
132+
prefill-decode-interval: 192
133+
schedule-policy: lpm
127134
swa-prefix-tails: 512
128135
max-running-requests: 16
129136
benchmark:
@@ -138,7 +145,7 @@ override_tp2_c16:
138145
tensor-parallel-size: 2
139146
mem-fraction-static: 0.80
140147
chunked-prefill-size: 16384
141-
prefill-decode-interval: 16
148+
prefill-decode-interval: 96
142149
swa-prefix-tails: 1024
143150
max-running-requests: 32
144151
benchmark:
@@ -153,7 +160,7 @@ override_tp2_c32:
153160
tensor-parallel-size: 2
154161
mem-fraction-static: 0.80
155162
chunked-prefill-size: 16384
156-
prefill-decode-interval: 4
163+
prefill-decode-interval: 32
157164
swa-prefix-tails: 2048
158165
max-running-requests: 64
159166
benchmark:
@@ -178,6 +185,89 @@ override_tp2_c64:
178185
CONC: '64'
179186
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
180187

188+
# DP2 attention with TP MoE.
189+
override_tp2_c32_dp:
190+
frontend:
191+
type: sglang-router
192+
args:
193+
policy: cache_aware
194+
request-id-headers: x-correlation-id
195+
dp-aware: true
196+
connect-timeout-secs: 900
197+
request-timeout-secs: 14400
198+
disable-health-check: true
199+
disable-retries: true
200+
roles:
201+
agg:
202+
gpus: 2
203+
args:
204+
tensor-parallel-size: 2
205+
data-parallel-size: 2
206+
enable-dp-attention: true
207+
enable-dp-lm-head: true
208+
enable-prefill-delayer: true
209+
enable-dp-attention-local-control-broadcast: true
210+
tokenizer-worker-num: 2
211+
prefill-delayer-token-usage-low-watermark: 0.7
212+
load-balance-method: total_requests
213+
mem-fraction-static: 0.80
214+
chunked-prefill-size: 32768
215+
prefill-decode-interval: 32
216+
swa-prefix-tails: 2048
217+
max-running-requests: 64
218+
env:
219+
SGLANG_SHARED_EXPERT_TP1: '1'
220+
SGLANG_DP_USE_GATHERV: '1'
221+
SGLANG_DP_USE_REDUCE_SCATTER: '1'
222+
SGLANG_AITER_CAPTURE_COPY_IN: '1'
223+
GPU_MAX_HW_QUEUES: '5'
224+
benchmark:
225+
env:
226+
CONC: '32'
227+
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
228+
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
229+
230+
override_tp2_c64_dp:
231+
frontend:
232+
type: sglang-router
233+
args:
234+
policy: cache_aware
235+
request-id-headers: x-correlation-id
236+
dp-aware: true
237+
connect-timeout-secs: 900
238+
request-timeout-secs: 14400
239+
disable-health-check: true
240+
disable-retries: true
241+
roles:
242+
agg:
243+
gpus: 2
244+
args:
245+
tensor-parallel-size: 2
246+
data-parallel-size: 2
247+
enable-dp-attention: true
248+
enable-dp-lm-head: true
249+
enable-prefill-delayer: true
250+
enable-dp-attention-local-control-broadcast: true
251+
tokenizer-worker-num: 2
252+
prefill-delayer-token-usage-low-watermark: 0.7
253+
load-balance-method: total_requests
254+
mem-fraction-static: 0.85
255+
chunked-prefill-size: 8192
256+
prefill-decode-interval: 16
257+
swa-prefix-tails: 4096
258+
max-running-requests: 64
259+
env:
260+
SGLANG_SHARED_EXPERT_TP1: '1'
261+
SGLANG_DP_USE_GATHERV: '1'
262+
SGLANG_DP_USE_REDUCE_SCATTER: '1'
263+
SGLANG_AITER_CAPTURE_COPY_IN: '1'
264+
GPU_MAX_HW_QUEUES: '5'
265+
benchmark:
266+
env:
267+
CONC: '64'
268+
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
269+
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
270+
181271
override_tp4_c1:
182272
roles:
183273
agg:
@@ -186,7 +276,8 @@ override_tp4_c1:
186276
tensor-parallel-size: 4
187277
mem-fraction-static: 0.70
188278
chunked-prefill-size: 16384
189-
prefill-decode-interval: 16
279+
prefill-decode-interval: 192
280+
schedule-policy: lpm
190281
swa-prefix-tails: 128
191282
max-running-requests: 2
192283
benchmark:
@@ -201,7 +292,8 @@ override_tp4_c2:
201292
tensor-parallel-size: 4
202293
mem-fraction-static: 0.70
203294
chunked-prefill-size: 16384
204-
prefill-decode-interval: 16
295+
prefill-decode-interval: 192
296+
schedule-policy: lpm
205297
swa-prefix-tails: 128
206298
max-running-requests: 4
207299
benchmark:
@@ -216,7 +308,8 @@ override_tp4_c4:
216308
tensor-parallel-size: 4
217309
mem-fraction-static: 0.70
218310
chunked-prefill-size: 16384
219-
prefill-decode-interval: 16
311+
prefill-decode-interval: 192
312+
schedule-policy: lpm
220313
swa-prefix-tails: 256
221314
max-running-requests: 8
222315
benchmark:
@@ -231,7 +324,8 @@ override_tp4_c8:
231324
tensor-parallel-size: 4
232325
mem-fraction-static: 0.70
233326
chunked-prefill-size: 16384
234-
prefill-decode-interval: 16
327+
prefill-decode-interval: 192
328+
schedule-policy: lpm
235329
swa-prefix-tails: 512
236330
max-running-requests: 16
237331
benchmark:
@@ -246,7 +340,7 @@ override_tp4_c16:
246340
tensor-parallel-size: 4
247341
mem-fraction-static: 0.70
248342
chunked-prefill-size: 16384
249-
prefill-decode-interval: 16
343+
prefill-decode-interval: 96
250344
swa-prefix-tails: 1024
251345
max-running-requests: 32
252346
benchmark:

‎inferencex-e2e/configs/CONFIGS.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,22 @@ Notes:
145145
- The fields above are the common ones, not the full schema. The Pydantic models in [`infx/matrix/validation.py`](../infx/matrix/validation.py) are the authoritative contract; they also accept fields such as `spec-decoding`, `srt-recipe`, `num-nodes`, and `require-power`, and they reject any field they do not define, which fails matrix generation.
146146
- Setting the fields above only guarantees that their values are passed as environment variables to benchmark scripts. Single-node jobs receive `PP_SIZE`, `DCP_SIZE`, and `PCP_SIZE`. Multinode jobs receive `PREFILL_PP_SIZE`, `PREFILL_DCP_SIZE`, `PREFILL_PCP_SIZE`, `DECODE_PP_SIZE`, `DECODE_DCP_SIZE`, and `DECODE_PCP_SIZE`. Actually using those variables is an implementation detail of the benchmark Bash script.
147147

148+
### DWDP prefill groups
149+
150+
Multi-node prefill workers may set `dwdp-size` to the number of ranks in a
151+
distributed weight-data parallel group. It defaults to 1 (disabled). For DWDP,
152+
`tp`, `ep`, `pp`, `dcp-size`, and `pcp-size` must all be 1 and `dp-attn` must be
153+
false. `num-worker` counts groups, not ranks: two workers with `dwdp-size: 8`
154+
allocate 16 prefill GPUs. Decode workers cannot enable DWDP.
155+
156+
The native TensorRT-LLM recipe must enable the corresponding DWDP mode and match
157+
the declared group layout. This field records topology and sizes allocations;
158+
it does not inject backend launch flags or enable a benchmark recipe. Results
159+
carry `prefill_dwdp_size` when greater than 1; TP remains 1. Existing result
160+
identities are unchanged when DWDP is disabled.
161+
162+
See NVIDIA's [DWDP implementation notes](https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/docs/source/blogs/tech_blog/blog19_DWDP_Distributed_Weight_Data_Parallelism_for_High_Performance_LLM_Inference_on_NVL72.md).
163+
148164
## Runners
149165

150166
`runners.yaml` holds the schedulable runner labels and the static facts of each

‎inferencex-e2e/configs/amd-master.yaml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1567,10 +1567,10 @@ glm5.3-fp8-mi355x-tilert-agentic:
15671567
ep: 1
15681568
dp-attn: false
15691569

1570-
# SGLang ROCm 7.2 MI35X release image. TP2/EP1 across the curve, plus TP4/EP1
1571-
# on the interactive band. Host Engram tables on both.
1570+
# SGLang ROCm 7.2 MI35X release image. TP2/EP1 and TP4/EP1 up to c16, DP2
1571+
# attention with TP MoE at c32 and c64. Host Engram tables on all of them.
15721572
dsv41flash-fp4-mi355x-sglang-agentic-dspark:
1573-
image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261006
1573+
image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012
15741574
model: deepseek-ai/DeepSeek-V4.1-Flash
15751575
model-prefix: dsv41flash
15761576
runner: cluster:mi355x-amds
@@ -1581,5 +1581,6 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark:
15811581
agentic-coding:
15821582
- dram-utilization: 0.60
15831583
search-space:
1584-
- { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
1584+
- { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
15851585
- { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }
1586+
- { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml }

‎inferencex-e2e/infx/bench/eval/meta.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ def dp(*names: str) -> bool:
9898
"prefill_pp": size("prefill_pp", "PREFILL_PP_SIZE", "PP_SIZE"),
9999
"prefill_dcp_size": size("prefill_dcp_size", "PREFILL_DCP_SIZE", "DCP_SIZE"),
100100
"prefill_pcp_size": size("prefill_pcp_size", "PREFILL_PCP_SIZE", "PCP_SIZE"),
101+
**(
102+
{"prefill_dwdp_size": size("prefill_dwdp_size", "PREFILL_DWDP_SIZE")}
103+
if size("prefill_dwdp_size", "PREFILL_DWDP_SIZE") > 1
104+
else {}
105+
),
101106
"prefill_ep": size("prefill_ep", "PREFILL_EP", "EP_SIZE"),
102107
"prefill_dp_attention": dp("PREFILL_DP_ATTENTION", "DP_ATTENTION"),
103108
"prefill_num_workers": size("prefill_num_workers", "PREFILL_NUM_WORKERS"),

‎inferencex-e2e/infx/launch/drivers/srt/submit.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
WORKLOAD_ENV = (
3939
"EVAL_*", "AIPERF_*", "AGENTIC_*",
4040
"TP", "EP_SIZE", "DP_ATTENTION", "PP_SIZE", "DCP_SIZE", "PCP_SIZE", "CONC",
41+
"PREFILL_DWDP_SIZE",
4142
"IS_AGENTIC", "SCENARIO_TYPE",
4243
"OPENAI_API_KEY", "REQUIRE_POWER", "ENABLE_AGENTX_POWER", "VLLM_ENGINE_READY_TIMEOUT_S",
4344
"SGLANG_TORCH_PROFILER_DIR", "VLLM_TORCH_PROFILER_DIR",

0 commit comments

Comments
 (0)