Skip to content

Commit 16b7719

Browse files
cquil11cquil11
committed
Extend MI325X AgentX frontier coverage and multi-prefill comparison
1 parent 6506a3f commit 16b7719

4 files changed

Lines changed: 106 additions & 53 deletions

File tree

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml renamed to inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -70,23 +70,18 @@ base:
7070
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
7171
RESULT_DIR: /logs/agentic
7272
IS_MULTINODE: 'true'
73-
CONC: '2'
73+
CONC: '1'
7474
KV_OFFLOADING: none
7575
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
7676
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
7777
AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true'
7878
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
7979
HF_HUB_CACHE: /hf_hub_cache
8080

81-
override_tp8_c2:
81+
override_tp8_c1:
8282
benchmark:
8383
env:
84-
CONC: '2'
85-
86-
override_tp8_c3:
87-
benchmark:
88-
env:
89-
CONC: '3'
84+
CONC: '1'
9085

9186
override_tp8_c4:
9287
benchmark:
@@ -98,11 +93,6 @@ override_tp8_c8:
9893
env:
9994
CONC: '8'
10095

101-
override_tp8_c16:
102-
benchmark:
103-
env:
104-
CONC: '16'
105-
10696
override_tp8_c20:
10797
benchmark:
10898
env:
@@ -114,16 +104,16 @@ override_tp8_c32:
114104
CONC: '32'
115105

116106
override_2tp4_c8:
117-
frontend:
107+
frontend: &smg_frontend
118108
type: smg
119109
container_image: lightseekorg/smg:1.11.0@sha256:bf6bf2671df71606b03da98679e65497dc62329a83685f6a4c1ff8dd345fb077
120-
args:
110+
args: &smg_policy
121111
policy: cache_aware
122112
cache-index: tree
123113
balance-abs-threshold: 1
124114
balance-rel-threshold: 1.5
125115
roles:
126-
agg:
116+
agg: &two_tp4
127117
workers: 2
128118
gpus: 4
129119
args:
@@ -132,3 +122,33 @@ override_2tp4_c8:
132122
benchmark:
133123
env:
134124
CONC: '8'
125+
126+
override_2tp4_c2:
127+
frontend:
128+
<<: *smg_frontend
129+
args:
130+
<<: *smg_policy
131+
balance-abs-threshold: 0
132+
roles:
133+
agg: *two_tp4
134+
benchmark:
135+
env:
136+
CONC: '2'
137+
138+
override_4tp2_c64:
139+
slurm:
140+
time_limit: '04:00:00'
141+
frontend: *smg_frontend
142+
roles:
143+
agg:
144+
workers: 4
145+
gpus: 2
146+
args:
147+
tensor-parallel-size: 2
148+
engram-config: '{"cpu_offload":true}'
149+
max-num-seqs: 32
150+
max-cudagraph-capture-size: 512
151+
max-num-batched-tokens: 8192
152+
benchmark:
153+
env:
154+
CONC: '64'

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-colocated.yaml renamed to inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-variants.yaml

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
base:
22
schema: 2
3-
name: dsv41flash-fp4-mi325x-vllm-nixl-p4-2d2-agentic
3+
name: dsv41flash-fp4-mi325x-vllm-nixl-agentic
44
model:
55
path: hf:deepseek-ai/DeepSeek-V4.1-Flash
66
container: vllm/vllm-openai-rocm:nightly-81198e97ba7eee2a22540caaa756b7fdddcb4d93@sha256:a401e4f46872dcde77e07ae3d7fa2d3a71b7f2773714316116532e2b39deb179
@@ -86,19 +86,14 @@ base:
8686
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
8787
RESULT_DIR: /logs/agentic
8888
IS_MULTINODE: 'true'
89-
CONC: '8'
89+
CONC: '16'
9090
KV_OFFLOADING: none
9191
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
9292
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
9393
AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true'
9494
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
9595
HF_HUB_CACHE: /hf_hub_cache
9696

97-
override_c8:
98-
benchmark:
99-
env:
100-
CONC: '8'
101-
10297
override_c16:
10398
benchmark:
10499
env:
@@ -108,3 +103,28 @@ override_c32:
108103
benchmark:
109104
env:
110105
CONC: '32'
106+
107+
override_c64:
108+
slurm:
109+
time_limit: '04:00:00'
110+
benchmark:
111+
env:
112+
CONC: '64'
113+
114+
override_3p4d4_c64:
115+
slurm:
116+
time_limit: '04:00:00'
117+
roles:
118+
prefill:
119+
nodes: 2
120+
workers: 3
121+
decode:
122+
workers: 1
123+
gpus: 4
124+
args:
125+
tensor-parallel-size: 4
126+
max-num-seqs: 128
127+
max-cudagraph-capture-size: 1024
128+
benchmark:
129+
env:
130+
CONC: '64'

‎inferencex-e2e/configs/amd-master.yaml‎

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1120,7 +1120,7 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11201120
scenarios:
11211121
agentic-coding:
11221122
- search-space:
1123-
- conc-list: [2]
1123+
- conc-list: [1]
11241124
spec-decoding: mtp
11251125
kv-offloading: none
11261126
num-nodes: 1
@@ -1130,8 +1130,8 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11301130
ep: 1
11311131
dp-attn: false
11321132
additional-settings:
1133-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c2
1134-
- conc-list: [3]
1133+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_tp8_c1
1134+
- conc-list: [4]
11351135
spec-decoding: mtp
11361136
kv-offloading: none
11371137
num-nodes: 1
@@ -1141,8 +1141,8 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11411141
ep: 1
11421142
dp-attn: false
11431143
additional-settings:
1144-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c3
1145-
- conc-list: [4]
1144+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_tp8_c4
1145+
- conc-list: [8]
11461146
spec-decoding: mtp
11471147
kv-offloading: none
11481148
num-nodes: 1
@@ -1152,8 +1152,8 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11521152
ep: 1
11531153
dp-attn: false
11541154
additional-settings:
1155-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c4
1156-
- conc-list: [8]
1155+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_tp8_c8
1156+
- conc-list: [20]
11571157
spec-decoding: mtp
11581158
kv-offloading: none
11591159
num-nodes: 1
@@ -1163,8 +1163,8 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11631163
ep: 1
11641164
dp-attn: false
11651165
additional-settings:
1166-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c8
1167-
- conc-list: [16]
1166+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_tp8_c20
1167+
- conc-list: [32]
11681168
spec-decoding: mtp
11691169
kv-offloading: none
11701170
num-nodes: 1
@@ -1174,41 +1174,43 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark:
11741174
ep: 1
11751175
dp-attn: false
11761176
additional-settings:
1177-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c16
1178-
- conc-list: [20]
1177+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_tp8_c32
1178+
- conc-list: [2]
1179+
router: {name: smg, version: '1.11.0'}
11791180
spec-decoding: mtp
11801181
kv-offloading: none
11811182
num-nodes: 1
11821183
worker:
1183-
num-worker: 1
1184-
tp: 8
1184+
num-worker: 2
1185+
tp: 4
11851186
ep: 1
11861187
dp-attn: false
11871188
additional-settings:
1188-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c20
1189-
- conc-list: [32]
1189+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_2tp4_c2
1190+
- conc-list: [8]
1191+
router: {name: smg, version: '1.11.0'}
11901192
spec-decoding: mtp
11911193
kv-offloading: none
11921194
num-nodes: 1
11931195
worker:
1194-
num-worker: 1
1195-
tp: 8
1196+
num-worker: 2
1197+
tp: 4
11961198
ep: 1
11971199
dp-attn: false
11981200
additional-settings:
1199-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_tp8_c32
1200-
- conc-list: [8]
1201+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_2tp4_c8
1202+
- conc-list: [64]
12011203
router: {name: smg, version: '1.11.0'}
12021204
spec-decoding: mtp
12031205
kv-offloading: none
12041206
num-nodes: 1
12051207
worker:
1206-
num-worker: 2
1207-
tp: 4
1208+
num-worker: 4
1209+
tp: 2
12081210
ep: 1
12091211
dp-attn: false
12101212
additional-settings:
1211-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg.yaml:override_2tp4_c8
1213+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/agg-variants.yaml:override_4tp2_c64
12121214

12131215
dsv41flash-fp4-mi325x-vllm-agentic-nixl:
12141216
image: vllm/vllm-openai-rocm:nightly-81198e97ba7eee2a22540caaa756b7fdddcb4d93@sha256:a401e4f46872dcde77e07ae3d7fa2d3a71b7f2773714316116532e2b39deb179
@@ -1224,7 +1226,7 @@ dsv41flash-fp4-mi325x-vllm-agentic-nixl:
12241226
scenarios:
12251227
agentic-coding:
12261228
- search-space:
1227-
- conc-list: [8]
1229+
- conc-list: [16]
12281230
spec-decoding: mtp
12291231
kv-offloading: none
12301232
prefill:
@@ -1233,9 +1235,9 @@ dsv41flash-fp4-mi325x-vllm-agentic-nixl:
12331235
ep: 1
12341236
dp-attn: false
12351237
additional-settings:
1236-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-colocated.yaml:override_c8
1238+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-variants.yaml:override_c16
12371239
decode: {num-worker: 2, tp: 2, ep: 1, dp-attn: false}
1238-
- conc-list: [16]
1240+
- conc-list: [32]
12391241
spec-decoding: mtp
12401242
kv-offloading: none
12411243
prefill:
@@ -1244,9 +1246,9 @@ dsv41flash-fp4-mi325x-vllm-agentic-nixl:
12441246
ep: 1
12451247
dp-attn: false
12461248
additional-settings:
1247-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-colocated.yaml:override_c16
1249+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-variants.yaml:override_c32
12481250
decode: {num-worker: 2, tp: 2, ep: 1, dp-attn: false}
1249-
- conc-list: [32]
1251+
- conc-list: [64]
12501252
spec-decoding: mtp
12511253
kv-offloading: none
12521254
prefill:
@@ -1255,8 +1257,19 @@ dsv41flash-fp4-mi325x-vllm-agentic-nixl:
12551257
ep: 1
12561258
dp-attn: false
12571259
additional-settings:
1258-
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-colocated.yaml:override_c32
1260+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-variants.yaml:override_c64
12591261
decode: {num-worker: 2, tp: 2, ep: 1, dp-attn: false}
1262+
- conc-list: [64]
1263+
spec-decoding: mtp
1264+
kv-offloading: none
1265+
prefill:
1266+
num-worker: 3
1267+
tp: 4
1268+
ep: 1
1269+
dp-attn: false
1270+
additional-settings:
1271+
- CONFIG_FILE=recipes/dsv41flash/vllm/mi325x-fp4/agentx/disagg-variants.yaml:override_3p4d4_c64
1272+
decode: {num-worker: 1, tp: 4, ep: 1, dp-attn: false}
12601273

12611274
dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark:
12621275
image: lmsysorg/sglang-rocm:v0.5.21-rocm724-mi35x-20261001

‎inferencex-e2e/perf-changelog.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9496,7 +9496,7 @@
94969496
- agentic-coding
94979497
description:
94989498
- "Update DeepSeek-V4.1-Flash FP4 AgentX on MI325X to the pinned October 8 vLLM ROCm nightly with native five-token DSpark, full context, and automatically selected golden acceptance."
9499-
- "Compare TP8 and packed TP4 aggregation with colocated NIXL prefill/decode pools. Every deployment uses and accounts for all eight GPUs on one node."
9500-
- "Use stock SMG cache-aware routing for aggregate replicas and session-affinity routing for NIXL pools. Engram tables are GPU-resident at TP8 and CPU-resident in the packed deployments."
9499+
- "Compare TP8 and packed TP4/TP2 aggregation with colocated and multi-prefill NIXL pools across low-latency and high-throughput settings. Deployments use and account for all eight GPUs on one node or all sixteen GPUs across two nodes."
9500+
- "Use stock SMG cache-aware routing for aggregate replicas, including stricter low-concurrency load balancing, and session-affinity routing for NIXL pools. Engram tables are GPU-resident at TP8 and CPU-resident in the packed deployments."
95019501
no-evals: true
95029502
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3843

0 commit comments

Comments
 (0)