@@ -561,10 +561,9 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
561561 ET_CHECK_OR_RETURN_ERROR (
562562 names.count (df::kForward ) && names.count (df::kVerify ) &&
563563 names.count (df::kVerifyArgmax ) &&
564- !names.count (" get_mutable_buffer_metadata" ) &&
565- !names.count (" vision_encoder" ),
564+ !names.count (" get_mutable_buffer_metadata" ),
566565 InvalidProgram,
567- " CUDA DFlash requires a tapped text-only off-graph artifact with its verifiers" );
566+ " CUDA DFlash requires a tapped off-graph artifact with its verifiers" );
568567 ET_ASSIGN_OR_RETURN (block, positive_int (module , df::kBlock ));
569568 ET_ASSIGN_OR_RETURN (
570569 declared_block, positive_int (module , " get_draft_dflash_block_size" ));
@@ -690,19 +689,11 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
690689 config.bos_id < static_cast <std::uint64_t >(target->vocab_size ()),
691690 InvalidProgram,
692691 " Muse Glimmer DFlash vocabulary cannot hold its special tokens" );
693- auto spec = std::make_shared<const MuseGlimmerPreparationSpec>(
694- MuseGlimmerPreparationSpec{
695- aten::ScalarType::BFloat16,
696- 0 ,
697- max_context,
698- static_cast <std::int32_t >(target->preferred_batch_tokens ()),
699- target->vocab_size (),
700- false ,
701- 0 ,
702- config.image_limits ,
703- config.bos_id });
704- auto executor =
705- std::unique_ptr<batching::Executor>(new MuseGlimmerCudaDFlashExecutor (
692+ std::shared_ptr<MuseGlimmerModelEmbeddings> embeddings;
693+ ET_ASSIGN_OR_RETURN (
694+ spec, make_cuda_preparation (*target, config, max_context, &embeddings));
695+ auto executor = std::unique_ptr<MuseGlimmerCudaDFlashExecutor>(
696+ new MuseGlimmerCudaDFlashExecutor (
706697 std::move (target),
707698 std::move (draft_cache),
708699 block,
@@ -713,6 +704,7 @@ Result<MuseGlimmerBackend> MuseGlimmerCudaDFlashExecutor::create(
713704 config.max_sessions ,
714705 max_context,
715706 hidden_width));
707+ executor->embeddings_ = std::move (embeddings);
716708 return MuseGlimmerBackend{std::move (executor), std::move (spec)};
717709}
718710
@@ -1390,6 +1382,11 @@ bool MuseGlimmerCudaDFlashExecutor::fail_batch(
13901382 return false ;
13911383}
13921384
1385+ bool MuseGlimmerCudaDFlashExecutor::accepts (
1386+ const batching::PreparedInput& input) const {
1387+ return embeddings_ && embeddings_->accepts (input);
1388+ }
1389+
13931390bool MuseGlimmerCudaDFlashExecutor::execute (
13941391 const batching::BatchInput& batch,
13951392 batching::BatchOutput& out) {
@@ -1409,14 +1406,21 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14091406 std::unordered_map<SessionId, int > cursor;
14101407 std::vector<std::pair<SessionId, int >> rewinds;
14111408 bool pending_decode = !batch.inputs .empty ();
1409+ bool prepared_step = false ;
14121410 for (const auto & input : batch.inputs ) {
14131411 const auto it = sessions_.find (input.sid );
14141412 const auto * payload =
14151413 std::get_if<batching::TokenInputPtr>(&input.payload );
1416- if (it == sessions_.end () || it->second .poisoned || !payload ||
1417- !*payload || input.size == 0 || input.offset > (*payload)->size () ||
1418- input.size > (*payload)->size () - input.offset ||
1419- input.position < 0 ||
1414+ const auto * prepared =
1415+ std::get_if<batching::PreparedInputPtr>(&input.payload );
1416+ const bool tokens = payload && *payload;
1417+ const bool image = prepared && *prepared && accepts (**prepared);
1418+ const std::size_t available = tokens ? (*payload)->size ()
1419+ : image ? (*prepared)->size ()
1420+ : 0 ;
1421+ if (it == sessions_.end () || it->second .poisoned || (!tokens && !image) ||
1422+ input.size == 0 || input.offset > available ||
1423+ input.size > available - input.offset || input.position < 0 ||
14201424 input.offset > static_cast <std::size_t >(max_context_) ||
14211425 input.size > static_cast <std::size_t >(max_context_) ||
14221426 (input.produce_output && !it->second .sampling ) ||
@@ -1447,13 +1451,16 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14471451 // A one-token prompt/replay is not necessarily emitted decode feedback.
14481452 // Stay on the legacy path unless every owner has an exact pending seed,
14491453 // one input, and room for the complete trained draft block.
1450- pending_decode = pending_decode && first && input.produce_output &&
1451- input.size == 1 && start == it->second .frontier &&
1452- start <= max_context_ - block_ && it->second .pending &&
1454+ pending_decode = pending_decode && tokens && first &&
1455+ input.produce_output && input.size == 1 &&
1456+ start == it->second .frontier && start <= max_context_ - block_ &&
1457+ it->second .pending &&
14531458 *it->second .pending == (**payload)[input.offset ];
1459+ prepared_step = prepared_step || image;
14541460 at->second = static_cast <int >(end);
14551461 for (std::size_t j = 0 ; j < input.size ; ++j) {
1456- const auto token = (**payload)[input.offset + j];
1462+ // Prepared rows reach the target as embeddings, not tokens.
1463+ const Token token = tokens ? (**payload)[input.offset + j] : 0 ;
14571464 if (token >= static_cast <std::uint64_t >(target_->vocab_size ())) {
14581465 return reject_batch ();
14591466 }
@@ -1467,6 +1474,15 @@ bool MuseGlimmerCudaDFlashExecutor::execute(
14671474 ? static_cast <std::int64_t >(step.tokens .size ()) - 1
14681475 : -1 );
14691476 }
1477+ if (prepared_step) {
1478+ auto embeddings =
1479+ embeddings_->materialize (target_->module (), batch.inputs );
1480+ if (!embeddings.ok ()) {
1481+ ET_LOG (Error, " CUDA DFlash: could not embed the prepared inputs" );
1482+ return reject_batch ();
1483+ }
1484+ step.embeddings = std::move (*embeddings);
1485+ }
14701486 // All inputs are validated before either cache is truncated.
14711487 for (const auto & item : rewinds) {
14721488 if (!rewind (sessions_.at (item.first ), item.second )) {
0 commit comments