Skip to content

Commit 0d7ca6f

Browse files
cquil11cquil11
committed
Merge latest main into the shared AMD integration
2 parents 3684ab6 + d13df37 commit 0d7ca6f

20 files changed

Lines changed: 272 additions & 28 deletions

‎benchmarks/multi_node/agentic_srt.sh‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ fi
2929

3030
check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION
3131

32+
if [[ -z "${AIPERF_SERVER_URL:-}" ]]; then
33+
if [[ -n "${SRT_FRONTEND_HOST:-}" ]]; then
34+
export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT:-$PORT}"
35+
else
36+
export AIPERF_SERVER_URL="http://localhost:${PORT}"
37+
fi
38+
fi
39+
echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL"
40+
3241
BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}"
3342
BASE_RESULT_FILENAME="$RESULT_FILENAME"
3443
read -r -a CONCURRENCIES <<< "${CONC_LIST:-$CONC}"
@@ -108,7 +117,7 @@ clear_agentic_worker_caches() {
108117
wait_for_agentic_servers_idle() {
109118
local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}"
110119
local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}"
111-
local frontend_metrics_url="http://localhost:${PORT}/metrics"
120+
local frontend_metrics_url="${AIPERF_SERVER_URL%/}/metrics"
112121

113122
"$AIPERF_PYTHON" - \
114123
"$timeout_seconds" \

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ resources:
2929
backend:
3030
type: trtllm
3131
prefill_environment:
32+
OMP_NUM_THREADS: '1'
3233
TLLM_LOG_LEVEL: INFO
3334
TRTLLM_SERVER_DISABLE_GC: '1'
3435
TRTLLM_WORKER_DISABLE_GC: '1'
@@ -42,6 +43,7 @@ backend:
4243
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
4344
CUDA_SCALE_LAUNCH_QUEUES: 4x
4445
decode_environment:
46+
OMP_NUM_THREADS: '1'
4547
TLLM_LOG_LEVEL: INFO
4648
TRTLLM_SERVER_DISABLE_GC: '1'
4749
TRTLLM_WORKER_DISABLE_GC: '1'

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ frontend:
160160

161161
benchmark:
162162
type: custom
163-
client_placement: last_decode
163+
client_placement: head
164164
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
165165
env:
166166
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5
2+
3+
model:
4+
path: nvidia/GLM-5.2-NVFP4
5+
container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1
6+
precision: fp4
7+
8+
dynamo:
9+
install: true
10+
wheel: 1.4.0.dev20260807
11+
request_plane: tcp
12+
13+
health_check:
14+
max_attempts: 270
15+
interval_seconds: 10
16+
17+
resources:
18+
gpu_type: gb300
19+
gpus_per_node: 4
20+
prefill_nodes: 1
21+
prefill_workers: 1
22+
gpus_per_prefill: 4
23+
decode_nodes: 2
24+
decode_workers: 1
25+
gpus_per_decode: 8
26+
27+
backend:
28+
type: trtllm
29+
prefill_environment: &server_environment
30+
HF_HUB_OFFLINE: "1"
31+
TRANSFORMERS_OFFLINE: "1"
32+
TQDM_DISABLE: "1"
33+
HF_HUB_DISABLE_PROGRESS_BARS: "1"
34+
TLLM_LOG_LEVEL: INFO
35+
TRTLLM_SERVER_DISABLE_GC: "1"
36+
TRTLLM_WORKER_DISABLE_GC: "1"
37+
TRTLLM_ENABLE_PDL: "1"
38+
NCCL_GRAPH_MIXING_SUPPORT: "0"
39+
MIMALLOC_PURGE_DELAY: "0"
40+
PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
41+
TRTLLM_SERVE_ENABLE_MSGSPEC: "1"
42+
TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10"
43+
TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600"
44+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
45+
UCX_RNDV_SCHEME: put_zcopy
46+
TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1"
47+
TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1"
48+
TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61"
49+
DYN_TRTLLM_ENABLE_ATTENTION_DP: "1"
50+
DYN_ENGINE_CONV_AFFINITY: "1"
51+
DYN_TOKENIZER: fastokens
52+
DYN_PUBLISH_KV_EVENTS: "0"
53+
decode_environment: *server_environment
54+
trtllm_config:
55+
prefill:
56+
attention_dp_config:
57+
enable_kv_cache_aware_routing: false
58+
kv_cache_routing_conversation_affinity: true
59+
kv_cache_routing_max_sessions: 65536
60+
cache_transceiver_config:
61+
backend: NIXL
62+
transceiver_runtime: PYTHON
63+
kv_cache_bounce_size_mb: 5120
64+
max_tokens_in_buffer: 1048576
65+
kv_transfer_timeout_ms: 600000
66+
cuda_graph_config: null
67+
disable_overlap_scheduler: false
68+
enable_attention_dp: true
69+
enable_chunked_prefill: true
70+
trust_remote_code: true
71+
kv_cache_config:
72+
dtype: fp8
73+
enable_block_reuse: true
74+
event_buffer_max_size: 0
75+
free_gpu_memory_fraction: 0.75
76+
host_cache_size: 137438953472
77+
tokens_per_block: 64
78+
max_batch_size: 256
79+
max_num_tokens: 8192
80+
max_seq_len: 1048576
81+
moe_config:
82+
backend: CUTEDSL
83+
moe_expert_parallel_size: 4
84+
num_postprocess_workers: 8
85+
pipeline_parallel_size: 1
86+
print_iter_log: true
87+
enable_iter_perf_stats: true
88+
return_perf_metrics: false
89+
scheduler_config:
90+
capacity_scheduler_policy: MAX_UTILIZATION
91+
sparse_attention_config:
92+
algorithm: dsa
93+
enable_heuristic_topk: true
94+
speculative_config:
95+
decoding_type: MTP
96+
max_draft_len: 5
97+
tensor_parallel_size: 4
98+
decode:
99+
cache_transceiver_config:
100+
backend: NIXL
101+
transceiver_runtime: PYTHON
102+
kv_cache_bounce_size_mb: 5120
103+
max_tokens_in_buffer: 1048576
104+
kv_transfer_timeout_ms: 600000
105+
cuda_graph_config:
106+
batch_sizes: [1]
107+
enable_padding: true
108+
enable_attention_dp: false
109+
trust_remote_code: true
110+
kv_cache_config:
111+
dtype: fp8
112+
enable_block_reuse: false
113+
event_buffer_max_size: 0
114+
free_gpu_memory_fraction: 0.8
115+
tokens_per_block: 64
116+
max_batch_size: 1
117+
max_num_tokens: 128
118+
max_seq_len: 1048576
119+
moe_config:
120+
backend: TRTLLM
121+
use_low_precision_moe_combine: true
122+
moe_expert_parallel_size: 1
123+
num_postprocess_workers: 4
124+
pipeline_parallel_size: 1
125+
print_iter_log: true
126+
enable_iter_perf_stats: true
127+
return_perf_metrics: false
128+
sparse_attention_config:
129+
algorithm: dsa
130+
enable_heuristic_topk: true
131+
use_cute_dsl_paged_mqa_logits: true
132+
use_cute_dsl_topk: true
133+
speculative_config:
134+
decoding_type: MTP
135+
max_draft_len: 5
136+
stream_interval: 20
137+
tensor_parallel_size: 8
138+
publish_events_and_metrics: false
139+
140+
frontend:
141+
type: dynamo
142+
enable_multiple_frontends: false
143+
env:
144+
ETCD_LEASE_TTL: '120'
145+
DYN_ROUTER_QUEUE_THRESHOLD: None
146+
DYN_ROUTER_TEMPERATURE: '0'
147+
DYN_TOKENIZER_CACHE: '1'
148+
DYN_TOKENIZER_CACHE_BYTES: '8000000000'
149+
DYN_TOKENIZER: fastokens
150+
DYN_TCP_REQUEST_TIMEOUT: '30'
151+
DYN_LOG: warn
152+
DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400'
153+
args:
154+
router-mode: kv
155+
no-kv-events: true
156+
active-decode-blocks-threshold: None
157+
active-prefill-tokens-threshold: None
158+
active-prefill-tokens-threshold-frac: None
159+
orchestrator_placement: head
160+
161+
benchmark:
162+
type: custom
163+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
164+
env:
165+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
166+
RESULT_DIR: /logs/agentic
167+
PORT: '8000'
168+
IS_MULTINODE: 'true'
169+
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400'
170+
AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
171+
HF_HUB_CACHE: /hf_hub_cache
172+
WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
173+
AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0"
174+
SERVED_MODEL_NAME: GLM-5.2-NVFP4
175+
MAX_MODEL_LEN: '1048576'
176+
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
177+
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
178+
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization
179+
OPENAI_API_KEY: EMPTY
180+
KV_OFFLOADING: dram

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ frontend:
160160

161161
benchmark:
162162
type: custom
163-
client_placement: last_decode
163+
client_placement: head
164164
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
165165
env:
166166
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace

0 commit comments

Comments
 (0)