Skip to content

Commit 091d807

Browse files
committed
[executorch][muse] Accept images in CUDA DFlash batching
Pull Request resolved: #24066 The CUDA DFlash executor accepts prepared image inputs on admission; image steps run the target from embeddings, then hidden taps and drafting proceed as for tokens. ghstack-source-id: 465398729 @exported-using-ghexport Differential Revision: [D125987295](https://our.internmc.facebook.com/intern/diff/D125987295/)
1 parent fdcc076 commit 091d807

4 files changed

Lines changed: 47 additions & 33 deletions

File tree

‎examples/models/muse-glimmer/BATCHING_SERVING.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ The batching runner and multiplexed worker support MLX and CUDA autoregressive
44
generation with text only or text plus one image across the submitted
55
conversation. They are opt-in entry points; the
66
legacy runners and export defaults remain unchanged. DFlash batching runs on MLX
7-
and CUDA (text only).
7+
(text only) and CUDA, which also serves images when exported with `--mmproj`.
88
CUDA specifics are in [CUDA](#cuda) below.
99

1010
## Export

‎examples/models/muse-glimmer/export/export_dflash.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -261,13 +261,6 @@ def export_dflash(
261261
activation_dtype=activation_dtype,
262262
)
263263

264-
if (
265-
draft_model is not None
266-
and use_offgraph_kv_cache
267-
and backend == "cuda"
268-
and vision_model is not None
269-
):
270-
raise ValueError("the CUDA batching DFlash export does not support vision yet")
271264
if use_offgraph_kv_cache and backend == "cuda":
272265
from executorch.examples.models.muse_glimmer.export.export_solo import (
273266
DEFAULT_FORWARD_SIZES,

‎examples/models/muse-glimmer/runtime/cuda_dflash_executor.cpp‎

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -561,10 +561,9 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
561561
ET_CHECK_OR_RETURN_ERROR(
562562
names.count(df::kForward) && names.count(df::kVerify) &&
563563
names.count(df::kVerifyArgmax) &&
564-
!names.count("get_mutable_buffer_metadata") &&
565-
!names.count("vision_encoder"),
564+
!names.count("get_mutable_buffer_metadata"),
566565
InvalidProgram,
567-
"CUDA DFlash requires a tapped text-only off-graph artifact with its verifiers");
566+
"CUDA DFlash requires a tapped off-graph artifact with its verifiers");
568567
ET_ASSIGN_OR_RETURN(block, positive_int(module, df::kBlock));
569568
ET_ASSIGN_OR_RETURN(
570569
declared_block, positive_int(module, "get_draft_dflash_block_size"));
@@ -690,19 +689,11 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
690689
config.bos_id < static_cast<std::uint64_t>(target->vocab_size()),
691690
InvalidProgram,
692691
"Muse Glimmer DFlash vocabulary cannot hold its special tokens");
693-
auto spec = std::make_shared<const MuseGlimmerPreparationSpec>(
694-
MuseGlimmerPreparationSpec{
695-
aten::ScalarType::BFloat16,
696-
0,
697-
max_context,
698-
static_cast<std::int32_t>(target->preferred_batch_tokens()),
699-
target->vocab_size(),
700-
false,
701-
0,
702-
config.image_limits,
703-
config.bos_id});
704-
auto executor =
705-
std::unique_ptr<batching::Executor>(new MuseGlimmerCudaDFlashExecutor(
692+
std::shared_ptr<MuseGlimmerModelEmbeddings> embeddings;
693+
ET_ASSIGN_OR_RETURN(
694+
spec, make_cuda_preparation(*target, config, max_context, &embeddings));
695+
auto executor = std::unique_ptr<MuseGlimmerCudaDFlashExecutor>(
696+
new MuseGlimmerCudaDFlashExecutor(
706697
std::move(target),
707698
std::move(draft_cache),
708699
block,
@@ -713,6 +704,7 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
713704
config.max_sessions,
714705
max_context,
715706
hidden_width));
707+
executor->embeddings_ = std::move(embeddings);
716708
return MuseGlimmerBackend{std::move(executor), std::move(spec)};
717709
}
718710

@@ -1390,6 +1382,11 @@ bool MuseGlimmerCudaDFlashExecutor::fail_batch(
13901382
return false;
13911383
}
13921384

1385+
bool MuseGlimmerCudaDFlashExecutor::accepts(
1386+
const batching::PreparedInput& input) const {
1387+
return embeddings_ && embeddings_->accepts(input);
1388+
}
1389+
13931390
bool MuseGlimmerCudaDFlashExecutor::execute(
13941391
const batching::BatchInput& batch,
13951392
batching::BatchOutput& out) {
@@ -1409,14 +1406,21 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14091406
std::unordered_map<SessionId, int> cursor;
14101407
std::vector<std::pair<SessionId, int>> rewinds;
14111408
bool pending_decode = !batch.inputs.empty();
1409+
bool prepared_step = false;
14121410
for (const auto& input : batch.inputs) {
14131411
const auto it = sessions_.find(input.sid);
14141412
const auto* payload =
14151413
std::get_if<batching::TokenInputPtr>(&input.payload);
1416-
if (it == sessions_.end() || it->second.poisoned || !payload ||
1417-
!*payload || input.size == 0 || input.offset > (*payload)->size() ||
1418-
input.size > (*payload)->size() - input.offset ||
1419-
input.position < 0 ||
1414+
const auto* prepared =
1415+
std::get_if<batching::PreparedInputPtr>(&input.payload);
1416+
const bool tokens = payload && *payload;
1417+
const bool image = prepared && *prepared && accepts(**prepared);
1418+
const std::size_t available = tokens ? (*payload)->size()
1419+
: image ? (*prepared)->size()
1420+
: 0;
1421+
if (it == sessions_.end() || it->second.poisoned || (!tokens && !image) ||
1422+
input.size == 0 || input.offset > available ||
1423+
input.size > available - input.offset || input.position < 0 ||
14201424
input.offset > static_cast<std::size_t>(max_context_) ||
14211425
input.size > static_cast<std::size_t>(max_context_) ||
14221426
(input.produce_output && !it->second.sampling) ||
@@ -1447,13 +1451,16 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14471451
// A one-token prompt/replay is not necessarily emitted decode feedback.
14481452
// Stay on the legacy path unless every owner has an exact pending seed,
14491453
// one input, and room for the complete trained draft block.
1450-
pending_decode = pending_decode && first && input.produce_output &&
1451-
input.size == 1 && start == it->second.frontier &&
1452-
start <= max_context_ - block_ && it->second.pending &&
1454+
pending_decode = pending_decode && tokens && first &&
1455+
input.produce_output && input.size == 1 &&
1456+
start == it->second.frontier && start <= max_context_ - block_ &&
1457+
it->second.pending &&
14531458
*it->second.pending == (**payload)[input.offset];
1459+
prepared_step = prepared_step || image;
14541460
at->second = static_cast<int>(end);
14551461
for (std::size_t j = 0; j < input.size; ++j) {
1456-
const auto token = (**payload)[input.offset + j];
1462+
// Prepared rows reach the target as embeddings, not tokens.
1463+
const Token token = tokens ? (**payload)[input.offset + j] : 0;
14571464
if (token >= static_cast<std::uint64_t>(target_->vocab_size())) {
14581465
return reject_batch();
14591466
}
@@ -1467,6 +1474,15 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14671474
? static_cast<std::int64_t>(step.tokens.size()) - 1
14681475
: -1);
14691476
}
1477+
if (prepared_step) {
1478+
auto embeddings =
1479+
embeddings_->materialize(target_->module(), batch.inputs);
1480+
if (!embeddings.ok()) {
1481+
ET_LOG(Error, "CUDA DFlash: could not embed the prepared inputs");
1482+
return reject_batch();
1483+
}
1484+
step.embeddings = std::move(*embeddings);
1485+
}
14701486
// All inputs are validated before either cache is truncated.
14711487
for (const auto& item : rewinds) {
14721488
if (!rewind(sessions_.at(item.first), item.second)) {

‎examples/models/muse-glimmer/runtime/cuda_dflash_executor.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,9 @@ struct CudaDFlashStats {
172172
std::uint64_t failed_batches = 0;
173173
};
174174

175-
// Text-only model adapter over the existing CUDA target executor. Decode drafts
175+
// Model adapter over the existing CUDA target executor. Image prompts prefill
176+
// through the target's embeddings forward; their rows' taps then feed the
177+
// draft like any other context. Decode drafts
176178
// run the static draft_forward_{N} in CUDA graphs, as the target's static
177179
// methods do; longer contexts drain through the dynamic draft_forward. Drafts
178180
// are greedy; dflash_verify accepts or resamples them under each session's
@@ -199,6 +201,7 @@ class MuseGlimmerCudaDFlashExecutor final : public batching::Executor {
199201
batching::SessionId session,
200202
const batching::SamplingParams& params,
201203
std::optional<std::uint64_t> seed) override;
204+
bool accepts(const batching::PreparedInput& input) const override;
202205
bool execute(const batching::BatchInput& batch, batching::BatchOutput& out)
203206
override;
204207
// Engine-thread-only, between executes. Counters measure actual adapter work.
@@ -276,6 +279,8 @@ class MuseGlimmerCudaDFlashExecutor final : public batching::Executor {
276279
std::size_t row_bytes_;
277280
batching::SessionId next_session_ = 1;
278281
std::unordered_map<batching::SessionId, State> sessions_;
282+
// Null for a text-only program.
283+
std::shared_ptr<MuseGlimmerModelEmbeddings> embeddings_;
279284
CudaDFlashStats stats_;
280285
cuda_dflash_detail::HiddenSnapshot verification_hidden_;
281286
cuda_dflash_detail::DeviceStaging verify_inputs_;

0 commit comments

Comments
 (0)