Repository navigation
Expand file tree
/
Copy pathindex.html
More file actions
664 lines (619 loc) · 34.7 KB
/
Copy pathindex.html
File metadata and controls
664 lines (619 loc) · 34.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<title>FlashRT</title>
<link rel="icon" type="image/svg+xml" href="favicon.svg" />
<link rel="apple-touch-icon" href="favicon.svg" />
<meta name="theme-color" content="#ffffff" />
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@400;500;600;700;800;900&display=swap" rel="stylesheet">
<link rel="stylesheet" href="styles.css" />
</head>
<body>
<div class="wrap">
<!-- ===================== HERO ===================== -->
<header class="hero" id="top">
<h1>FlashRT: Agent Harness for Guiding Agents to Deploy Real-Time Multimodal Applications</h1>
<div class="authors">
<a href="https://krishagarwal.github.io/" target="_blank" rel="noopener">Krish Agarwal</a><sup>1</sup>,
<a href="https://dreaming-panda.github.io/" target="_blank" rel="noopener">Zhuoming Chen</a><sup>1</sup>,
<a href="mailto:yanyuan.qin@amd.com">Yanyuan Qin</a><sup>2</sup>,
<a href="mailto:zhenyu.gu@amd.com">Zhenyu Gu</a><sup>2</sup>,
<a href="https://cse.buffalo.edu/faculty/atri/" target="_blank" rel="noopener">Atri Rudra</a><sup>3</sup>,
<a href="https://www.andrew.cmu.edu/user/beidic/" target="_blank" rel="noopener">Beidi Chen</a><sup>1</sup>
</div>
<div class="affil">
<sup>1</sup>Carnegie Mellon University ·
<sup>2</sup>AMD ·
<sup>3</sup>University at Buffalo
</div>
<div class="hero-btns">
<a class="btn" href="https://arxiv.org/abs/2607.18171" target="_blank" rel="noopener">
<svg viewBox="0 0 384 512" aria-hidden="true"><path d="M320 464H64V48h160v112h96v304zM256 0H64C28.7 0 0 28.7 0 64v384c0 35.3 28.7 64 64 64h256c35.3 0 64-28.7 64-64V160L256 0z"/></svg>
arXiv
</a>
<a class="btn" href="https://github.com/Infini-AI-Lab/FlashRT/" target="_blank" rel="noopener">
<svg viewBox="0 0 16 16" aria-hidden="true"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>
Code
</a>
</div>
</header>
<!-- ===================== TL;DR ===================== -->
<div class="tldr">
<p>
<span class="lbl">Real-time multimodal applications each deserve their own serving systems.</span>
Multimodal applications are often quite complicated. Each sits at its own set of tradeoffs on key metrics like latency and throughput, and balancing these metrics can require different hardware budgets for each application. Each application also consists models compute that differently: autoregressive LLMs, diffusion transformers, etc. Ultimately, no shared serving stack can fit them all. Instead, we introduce <strong>FlashRT</strong>, an agent harness that guides a coding agent to customize a system for each application. From a plain user-provided reference, it directs the agent to <strong>develop a customized system to serve it</strong>. The specialization pays off: Across five real-time applications, on both NVIDIA B200 and AMD MI355X GPUs, FlashRT reaches up to
<span class="hi">70×</span> lower latency and <span class="hi">3.6×</span> higher throughput, all with no hand-written serving code.
</p>
</div>
<!-- ===================== RESULTS ===================== -->
<section id="results" data-plat="b200">
<h2 class="sec"><span class="ic">📊</span>One agent, Many Real-time Applications</h2>
<p class="sec-sub">
For each application the agent is given two things: the reference implementation and a GPU
budget. No hints, no templates. It works out streaming, disaggregation, sequence parallelism,
and pipeline parallelism on its own, and checks every variant against the reference before
keeping it. We run the agent separately on a node of 8 NVIDIA B200 GPUs and a node of 8 AMD MI355X
GPUs; use the toggle to switch between them. (Agent: Claude Code with Opus 4.8.)
</p>
<div class="plat-toggle" role="tablist" aria-label="Hardware">
<span class="plat-slider" aria-hidden="true"></span>
<button class="plat-btn" data-plat="b200" type="button">NVIDIA B200</button>
<button class="plat-btn" data-plat="mi355x" type="button">AMD MI355X</button>
</div>
<div class="apps">
<div class="apps-nav" role="tablist" aria-label="Applications">
<button class="app-tab active" data-i="0">Face-to-Face Avatar</button>
<button class="app-tab" data-i="1">Qwen3-Omni</button>
<button class="app-tab" data-i="2">Video Background Editor</button>
<button class="app-tab" data-i="3">Video World Model</button>
<button class="app-tab" data-i="4">Video Narrator</button>
</div>
<div class="apps-viewport">
<button class="app-arrow" data-dir="-1" type="button" aria-label="Previous application"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 18 9 12 15 6"/></svg></button>
<div class="apps-scroll" id="appsScroll">
<div class="apps-strip">
<!-- LiveAvatar -->
<article class="app-card">
<h3>Face-to-Face Conversational Agent <span class="model">Live-Avatar-14B (S2V)</span></h3>
<p>
The pipeline runs ASR → LLM → streaming TTS → an audio-driven video avatar (Live-Avatar).
The agent overlaps the stages with chunk-level streaming and pipelines the four denoising
steps inside the video model across separate GPUs. A minutes-long offline baseline becomes an
interactive stream that responds in under two seconds and runs well above real-time frame
rates.
</p>
<div class="plat-panel" data-plat="b200">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential, no streaming)</td><td class="num">117.10 s</td><td class="num">—</td></tr>
<tr><td>2</td><td>FlashRT — streaming</td><td class="num"><strong>1.69 s</strong></td><td class="num">30.92 FPS</td></tr>
<tr class="best"><td>6</td><td>FlashRT — streaming + S2V pipeline parallelism</td><td class="num">1.72 s</td><td class="num"><strong>133.48 FPS</strong> <span class="fn">(theoretical)</span></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
About 70× lower latency than the single-GPU reference (117 s → 1.7 s). The 6-GPU frame rate
is the pipeline's theoretical sustainable rate (the deployment produces frames at higher FPS than
the videos the S2V model is trained on) so latency is the binding constraint here.
</p>
</div>
<div class="plat-panel" data-plat="mi355x">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential, no streaming)</td><td class="num">101.17 s</td><td class="num">—</td></tr>
<tr><td>3</td><td>FlashRT — streaming</td><td class="num">1.59 s</td><td class="num">37.76 FPS</td></tr>
<tr class="best"><td>8</td><td>FlashRT — streaming + S2V PP + SP</td><td class="num"><strong>1.47 s</strong></td><td class="num"><strong>78.61 FPS</strong> <span class="fn">(theoretical)</span></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
About 69× lower latency than the sequential reference (101 s → 1.5 s). Here the agent
splits the denoising steps into two pipeline stages and shards each across three GPUs, a
slightly different decomposition than it picks on B200.
</p>
</div>
</article>
<!-- Qwen3-Omni -->
<article class="app-card">
<h3>Qwen3-Omni <span class="model">Multimodal LLM</span></h3>
<p>
Qwen3-Omni is a natively multimodal model with three stages to support text input → audio output:
a Thinker LLM, a Talker LLM, and a vocoder. We compare with vLLM-Omni, which disaggregates and streams
these stages with hand-written code. Given only a synchronous baseline, the agent recovers the same
structure on its own, then edges past it with lighter-weight data transfer between components than
vLLM-Omni's more general implementation.
</p>
<div class="plat-panel" data-plat="b200">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>Deployment</th><th>GPUs</th><th class="num">Latency ↓</th><th class="num">RTF < 1</th></tr>
</thead>
<tbody>
<tr><td>Sequential (no streaming)</td><td>1</td><td class="num">42.713 s</td><td class="num">✓</td></tr>
<tr><td>vLLM-Omni (hand-engineered)</td><td>3</td><td class="num">0.433 s</td><td class="num">✓</td></tr>
<tr class="best"><td>FlashRT</td><td>3</td><td class="num"><strong>0.323 s</strong></td><td class="num">✓</td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
25% lower latency than vLLM-Omni's hand-engineered deployment, while keeping the real-time
factor below 1.
</p>
</div>
<div class="plat-panel" data-plat="mi355x">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>Deployment</th><th>GPUs</th><th class="num">Latency ↓</th><th class="num">RTF < 1</th></tr>
</thead>
<tbody>
<tr><td>Sequential (no streaming)</td><td>1</td><td class="num">29.66 s</td><td class="num">✓</td></tr>
<tr><td>vLLM-Omni (hand-engineered)</td><td>3</td><td class="num">0.779 s</td><td class="num">✓</td></tr>
<tr class="best"><td>FlashRT</td><td>3</td><td class="num"><strong>0.276 s</strong></td><td class="num">✓</td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
65% lower latency than vLLM-Omni's hand-engineered deployment — a wider margin than on
B200 — while keeping the real-time factor below 1.
</p>
</div>
</article>
<!-- Krea + SAM 3 -->
<article class="app-card">
<h3>Video Background Editor <span class="model">Krea-Realtime-14B + SAM 3</span></h3>
<p>
A live webcam stream runs down two paths at once. Krea-Realtime restyles each frame while
SAM 3 segments the person, and the two results are composited so only the background changes.
The agent puts SAM 3 on its own GPU, off the restyling path, and parallelizes the DiT and VAE
that do the restyling.
</p>
<div class="plat-panel" data-plat="b200">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>Deployment</th><th>GPUs</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>Baseline (sequential)</td><td>1</td><td class="num">1715 ms</td><td class="num">6.82 FPS</td></tr>
<tr><td>FlashRT — parallel</td><td>2</td><td class="num">1014 ms</td><td class="num">11.54 FPS</td></tr>
<tr><td>FlashRT — latency-optimized</td><td>4</td><td class="num"><strong>491 ms</strong></td><td class="num">17.18 FPS</td></tr>
<tr class="best"><td>FlashRT — frame-rate-optimized</td><td>5</td><td class="num">517 ms</td><td class="num"><strong>19.41 FPS</strong></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
About 3.5× lower latency (1.7 s → 0.5 s) and 2.8× the baseline frame rate (6.8 → 19.4 FPS),
with SAM 3 kept on its own GPU throughout.
</p>
</div>
<div class="plat-panel" data-plat="mi355x">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>Deployment</th><th>GPUs</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>Baseline (sequential)</td><td>1</td><td class="num">2393 ms</td><td class="num">5.40 FPS</td></tr>
<tr class="best"><td>FlashRT — latency-optimized</td><td>4</td><td class="num"><strong>612 ms</strong></td><td class="num">7.93 FPS</td></tr>
<tr><td>FlashRT — frame-rate-optimized</td><td>5</td><td class="num">646 ms</td><td class="num"><strong>8.90 FPS</strong></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
About 3.9× lower latency. Here the pipeline is bound by the SAM 3 segmentation path,
so disaggregating the VAE adds little frame rate.
</p>
</div>
</article>
<!-- WorldPlay -->
<article class="app-card">
<h3>Video World Model <span class="model">WorldPlay-5B</span></h3>
<p>
A video world model turns keyboard actions (WASD, arrow keys) into a video stream you can
move through in real time. We use WorldPlay, which splits into an autoregressive DiT and a
streaming VAE that share no state, so the agent can disaggregate them onto separate GPUs and
pipeline across frames for a higher frame rate, or co-locate them under sequence parallelism
for lower latency. It scales whichever choice it makes to the GPU budget.
</p>
<div class="plat-panel" data-plat="b200">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential)</td><td class="num">869 ms</td><td class="num">20.4 FPS</td></tr>
<tr><td>2</td><td>FlashRT — frame-rate-optimized</td><td class="num">625 ms</td><td class="num">31.0 FPS</td></tr>
<tr><td>2</td><td>FlashRT — latency-optimized</td><td class="num">493 ms</td><td class="num">25.5 FPS</td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
1.5× the baseline frame rate or 43% lower latency, depending on which target the agent
optimizes. See paper for extended results with scaling to more GPUs.
</p>
</div>
<div class="plat-panel" data-plat="mi355x">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential)</td><td class="num">1102 ms</td><td class="num">15.6 FPS</td></tr>
<tr><td>2</td><td>FlashRT — latency-optimized</td><td class="num">576 ms</td><td class="num">19.8 FPS</td></tr>
<tr><td>2</td><td>FlashRT — frame-rate-optimized</td><td class="num">723 ms</td><td class="num">29.5 FPS</td></tr>
<tr><td>4</td><td>FlashRT — latency-optimized</td><td class="num"><strong>320 ms</strong></td><td class="num">31.8 FPS</td></tr>
<tr class="best"><td>8</td><td>FlashRT — frame-rate-optimized</td><td class="num">326 ms</td><td class="num"><strong>56.8 FPS</strong></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
At 2 GPUs, the same co-location (latency) vs. disaggregation (frame rate) split as on B200; the
frame rate scales to 3.6× the baseline (56.8 FPS) at 8 GPUs.
</p>
</div>
</article>
<!-- LongLive -->
<article class="app-card">
<h3>Video Narrator <span class="model">LongLive-2.0-5B</span></h3>
<p>
The user speaks, an ASR model transcribes each prompt, and an autoregressive video model
(LongLive-2.0-5B) keeps rendering a stream that updates to match. The agent runs the ASR on
its own GPU so transcription never stalls generation, then trades latency against frame rate:
co-locate the DiT and VAE at a higher sequence-parallel degree for lower latency, or split them
apart and tile the VAE for a higher frame rate.
</p>
<div class="plat-panel" data-plat="b200">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential)</td><td class="num">674 ms</td><td class="num">25.8 FPS</td></tr>
<tr><td>2</td><td>FlashRT — latency-optimized</td><td class="num">512 ms</td><td class="num">36.5 FPS</td></tr>
<tr><td>2</td><td>FlashRT — frame-rate-optimized</td><td class="num">630 ms</td><td class="num">47.8 FPS</td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
1.9× the baseline frame rate, or 1.3× lower latency when the agent optimizes for latency
instead. See paper for extended results with scaling to more GPUs.
</p>
</div>
<div class="plat-panel" data-plat="mi355x">
<div class="tbl-wrap">
<table>
<thead>
<tr><th>GPUs</th><th>Deployment</th><th class="num">Latency ↓</th><th class="num">Frame rate ↑</th></tr>
</thead>
<tbody>
<tr><td>1</td><td>Baseline (sequential)</td><td class="num">784 ms</td><td class="num">18.9 FPS</td></tr>
<tr><td>2</td><td>FlashRT — latency-optimized</td><td class="num">660 ms</td><td class="num">20.4 FPS</td></tr>
<tr><td>2</td><td>FlashRT — frame-rate-optimized</td><td class="num">907 ms</td><td class="num">30.5 FPS</td></tr>
<tr><td>4</td><td>FlashRT — latency-optimized</td><td class="num"><strong>343 ms</strong></td><td class="num">38.8 FPS</td></tr>
<tr class="best"><td>8</td><td>FlashRT — frame-rate-optimized</td><td class="num">442 ms</td><td class="num"><strong>56.5 FPS</strong></td></tr>
</tbody>
</table>
</div>
<p class="cap-note">
At 2 GPUs, the same co-location (latency) vs. disaggregation (frame rate) split as on B200; the
frame rate scales to 3.0× the baseline (56.5 FPS) at 8 GPUs.
</p>
</div>
</article>
</div>
</div>
<button class="app-arrow" data-dir="1" type="button" aria-label="Next application"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 18 15 12 9 6"/></svg></button>
</div>
</div>
</section>
<!-- ===================== VISION ===================== -->
<section id="vision">
<h2 class="sec"><span class="ic">🧭</span>The Vision</h2>
<h3>Development efficiency and intelligence change how ML infrastructure is designed.</h3>
<p>
For decades we have built ML software as a layered stack: hand-written kernels beneath
frameworks, frameworks beneath compilers and serving systems, each layer written once and
shared by every application above it. Sharing was never the goal — it was the workaround
for two constraints of human-written software. AI agents remove both.
</p>
<div class="point">
<h4>01 · Every application gets its own system.</h4>
<div class="tn">
<div class="col then">
<div class="lbl">Then</div>
<p>
Code was expensive to write. Every line an engineer produced had to be amortized across
many applications, so everyone settled for the average case of a shared stack.
</p>
</div>
<div class="col now">
<div class="lbl">Now</div>
<p>
For an agent, lines of code are cheap. It writes, rewrites, and discards code at
negligible cost — so a system specialized to your <strong>application</strong>, your
<strong>hardware</strong>, and your <strong>efficiency</strong> targets stops being a
luxury.
</p>
</div>
</div>
</div>
<div class="point">
<h4>02 · Reuse ideas, not frozen code.</h4>
<div class="tn">
<div class="col then">
<div class="lbl">Then</div>
<p>
Traditional stacks compose only through exact, strictly compatible interfaces. Code that
is even slightly incompatible cannot be reused at all — so reuse meant frozen,
portable artifacts.
</p>
</div>
<div class="col now">
<div class="lbl">Now</div>
<p>
An agent reads code the way an engineer does. It takes the <strong>ideas</strong> from a
system that almost fits — kernels, schedules, parallelism strategies — and
re-implements them for the case at hand.
</p>
</div>
</div>
</div>
<h4>One reference, many systems, written online</h4>
<p>
This is not at all hypothetical — every deployment in the results above is, in fact, such a system. From
one reference implementation, the agent can write a latency-oriented system when you are chasing
responsiveness, a throughput-oriented one when you are chasing raw throughput, or a balanced
one in between, then re-specializes it for however many GPUs you actually have.
</p>
<p>
And there is a cost we rarely question. A shared stack must serve everyone, so it piles up
— compatibility paths, configuration surfaces, layers upon layers — until
maintaining and extending it becomes a project of its own, and understanding and debugging it
becomes a tremendous challenge for humans and agents alike. In the era of agents, it is worth
asking whether that structure is still a <strong>helper or a blocker</strong> —
especially for ML infrastructure, whose entire job is to connect models, systems, and hardware.
</p>
<p>
The foundation of ML software shifts <strong>from shared infrastructure to a general
agent</strong> — and FlashRT is our first concrete step.
</p>
</section>
<section id="build">
<h2 class="sec"><span class="ic">🎯</span>Why We Build This</h2>
<p class="sec-sub">
FlashRT sits between two problems we care about: making generative models fast enough to use
interactively, and handing the repetitive systems work behind them to an agent instead of an
engineer.
</p>
<div class="duo">
<div class="card">
<h4>Interactive generative models</h4>
<p>
Most generative models are still used in batches: send a prompt, wait, get a file back.
We are after the interactive case, where audio and video models run fast enough to hold a
conversation or react to a live webcam.
</p>
<ul>
<li>Streaming generation at sub-second latency</li>
<li>Audio and video models that respond in real time</li>
<li>Heterogeneous pipelines, composed from off-the-shelf models</li>
</ul>
</div>
<div class="card">
<h4>Agents that do the systems work</h4>
<p>
Making these pipelines run efficiently is slow, specialized work, and it has to be redone
for every new application. FlashRT is our first step toward letting an agent do it: write
the deployment, parallelize the models, and check the result against a reference.
</p>
<ul>
<li>Agent-written placement, streaming, and parallelism</li>
<li>Every variant checked against a ground-truth reference</li>
<li>Decisions gated on measurements, not guesses</li>
</ul>
</div>
</div>
</section>
<!-- ===================== SYSTEM ===================== -->
<section id="system">
<h2 class="sec"><span class="ic">⚙️</span>The System</h2>
<p class="sec-sub">
A developer writes a simple single-GPU reference. FlashRT guides a coding agent to lift it into
an optimized multi-GPU deployment, choosing placement, streaming, and parallelism, through a
<em>chain-of-program</em> workflow: lower the reference one pass at a time, and check the work
at every step.
</p>
<figure class="fig-float">
<img src="images/overview.png" alt="Before vs FlashRT: where teams once hand-engineered each pipeline, a single agent now produces all of them from references." />
<figcaption>
<strong>Before vs. FlashRT.</strong> Where each new pipeline once demanded its own systems
team, a single agent now produces all of them from references.
</figcaption>
</figure>
<p>
Until now, every new real-time multimodal application (a world model, a live avatar, a speech
agent) came with its own round of systems engineering: deciding what to disaggregate, what to
stream, how to shard, and where to put each model.
</p>
<p>
<strong>FlashRT hands that work to one agent.</strong> A developer writes a plain single-GPU reference
for the application, and the agent makes and verifies every placement, streaming, and
parallelism decision inside a measurement-gated loop.
</p>
<p>
What comes out is a single serving system that targets very different applications, with no
per-application code on the systems side.
</p>
<figure class="fig-clear">
<img src="images/flashrt-agent-loop.png" alt="FlashRT workflow: from human-written baseline code, the agent constructs and analyzes a hierarchical graph IR, then runs a self-driven validation loop that implements, verifies and benchmarks, and re-hypothesizes each variant before composing the best strategies." />
<figcaption>
<strong>The chain-of-program workflow.</strong> From a human-written baseline, the agent
builds and analyzes a hierarchical graph IR, then runs a self-driven loop that implements,
verifies, and benchmarks each variant. Strategies that hold up (here, streaming and a
disaggregated DiT + VAE pipeline) get composed; ones that don't (co-located sequence
parallelism, for this throughput target) are dropped.
</figcaption>
</figure>
<div class="steps">
<div class="step">
<div class="n">1</div>
<h4>Lift</h4>
<p>Turn the reference into a hierarchical graph IR that makes data dependencies, persistent state, and streaming edges explicit.</p>
</div>
<div class="step">
<div class="n">2</div>
<h4>Validate</h4>
<p>Run the IR through a sequential interpreter and diff it against the reference, so every later step stands on verified ground.</p>
</div>
<div class="step">
<div class="n">3</div>
<h4>Analyze</h4>
<p>Run static analyses over the IR to surface a finite list of legal moves: streaming, disaggregation, and intra-model parallelism.</p>
</div>
<div class="step">
<div class="n">4</div>
<h4>Iterate</h4>
<p>Work the candidates in a measurement-gated loop: implement, verify against the reference, benchmark, then re-plan from the numbers.</p>
</div>
</div>
<div class="box">
<p><span class="tag">Insight 1 — Plan in passes, not one jump.</span> An agent can't turn a
reference into an efficient deployment in a single jump. Make it build an IR, analyze that, and
only then lower it into a deployment, and the quality climbs. We call this
<em>chain-of-program</em>, after the way compilers lower code in passes.</p>
</div>
<div class="box">
<p><span class="tag">Insight 2 — Ground the loop in real measurements.</span> Prompted naively,
the agent writes broken code and stops exploring early. Instead it writes a test harness that
drives simulated input through the deployment's buffers, the way a real frontend would, then
<em>proposes</em> a change, <em>verifies</em> it against the reference, and <em>re-plans</em>
from the measured latency and frame rate.</p>
</div>
</section>
<!-- ===================== CITATION ===================== -->
<section id="citation">
<h2 class="sec"><span class="ic">📄</span>Citation</h2>
<pre class="cite">@misc{agarwal2026flashrtagentharnessguiding,
title = {FlashRT: Agent Harness for Guiding Agents to Deploy Real-Time Multimodal Applications},
author = {Krish Agarwal and Zhuoming Chen and Yanyuan Qin and Zhenyu Gu and Atri Rudra and Beidi Chen},
year = {2026},
eprint = {2607.18171},
archivePrefix = {arXiv},
primaryClass = {cs.LG},
url = {https://arxiv.org/abs/2607.18171}
}</pre>
</section>
<!-- ===================== FOOTER ===================== -->
<footer>
<div class="flinks">
<a href="https://arxiv.org/abs/2607.18171" target="_blank" rel="noopener">arXiv</a>
<a href="https://github.com/Infini-AI-Lab/FlashRT/" target="_blank" rel="noopener">GitHub</a>
<a href="mailto:krisha2@andrew.cmu.edu">Contact</a>
<a href="#top">Back to top ↑</a>
</div>
<div class="meta">© 2026 Infini-AI Lab · Carnegie Mellon University</div>
</footer>
</div>
<script>
(function(){
var scroll = document.getElementById('appsScroll');
if(!scroll) return;
var strip = scroll.querySelector('.apps-strip');
var tabs = Array.prototype.slice.call(document.querySelectorAll('.app-tab'));
var real = Array.prototype.slice.call(strip.querySelectorAll('.app-card'));
var n = real.length;
if(n < 2) return;
// Clone the first and last cards so the ends wrap seamlessly:
// [clone(last), card0, card1, ..., card(n-1), clone(first)]
var firstClone = real[0].cloneNode(true);
var lastClone = real[n - 1].cloneNode(true);
[firstClone, lastClone].forEach(function(c){
c.classList.add('clone'); c.setAttribute('aria-hidden', 'true');
});
strip.insertBefore(lastClone, real[0]);
strip.appendChild(firstClone);
var count = n + 2; // total slides including clones
var realIndex = 0;
function stride(){ return strip.children[1].offsetLeft - strip.children[0].offsetLeft; }
function posOf(){ var s = stride(); return s ? Math.round(scroll.scrollLeft / s) : 1; }
function setActive(i){ tabs.forEach(function(t, j){ t.classList.toggle('active', j === i); }); }
function jump(left){ // instant, seamless reposition
var prev = scroll.style.scrollBehavior;
scroll.style.scrollBehavior = 'auto';
scroll.scrollLeft = left;
requestAnimationFrame(function(){ scroll.style.scrollBehavior = prev; });
}
function onScroll(){
var p = Math.max(0, Math.min(count - 1, posOf()));
realIndex = (p === 0) ? n - 1 : (p === count - 1) ? 0 : p - 1;
setActive(realIndex);
}
function onSettle(){ // after a scroll/flick lands on a clone, wrap
var p = posOf();
if(p === 0 || p === count - 1) jump((realIndex + 1) * stride());
}
var hasScrollend = ('onscrollend' in window), timer;
scroll.addEventListener('scroll', function(){
onScroll();
if(!hasScrollend){ clearTimeout(timer); timer = setTimeout(onSettle, 150); }
}, { passive: true });
if(hasScrollend) scroll.addEventListener('scrollend', onSettle);
tabs.forEach(function(t, i){
t.addEventListener('click', function(){
setActive(i);
scroll.scrollTo({ left: (i + 1) * stride(), behavior: 'smooth' });
});
});
// Prev/next arrows — circular via the clone slides.
document.querySelectorAll('.app-arrow').forEach(function(btn){
btn.addEventListener('click', function(){
var dir = parseInt(btn.getAttribute('data-dir'), 10);
var target = (dir > 0)
? (realIndex === n - 1 ? count - 1 : realIndex + 2) // last -> clone-first, else next real
: (realIndex === 0 ? 0 : realIndex); // first -> clone-last, else prev real
scroll.scrollTo({ left: target * stride(), behavior: 'smooth' });
});
});
var rtimer;
window.addEventListener('resize', function(){
clearTimeout(rtimer);
rtimer = setTimeout(function(){ jump((realIndex + 1) * stride()); }, 120);
});
// Start on the first real card (offset past the leading clone).
jump(stride());
setActive(0);
})();
</script>
<script>
(function(){
var results = document.getElementById('results');
if(!results) return;
var btns = Array.prototype.slice.call(results.querySelectorAll('.plat-btn'));
btns.forEach(function(b){
b.addEventListener('click', function(){
results.setAttribute('data-plat', b.getAttribute('data-plat'));
btns.forEach(function(x){ x.classList.toggle('active', x === b); });
window.dispatchEvent(new Event('resize')); // let the carousel re-measure after height change
});
});
})();
</script>
</body>
</html>