Skip to content

Commit 76b8dcc

Browse files
committed
Arm backend: Fuse command stream in VGF.
This PR introduces fused Vulkan command-stream submission for the ExecuTorch VGF backend. The implementation allows application-provided preprocessing and postprocessing command buffers to be submitted together with the VGF inference command buffer using a single vkQueueSubmit() and fence wait, reducing CPU submission and synchronisation overhead. The existing VGF execution path remains unchanged when fusion is not enabled. Fusion was faster in 8 out of 10 repetitions, with a median paired improvement of 1.144 ms. Signed-off-by: Elena Zhelezina <elena.zhelezina@arm.com> Change-Id: I7d918442b578e30383f0c8dc4f8cfad9e9eecda3
1 parent 4411b8d commit 76b8dcc

3 files changed

Lines changed: 219 additions & 5 deletions

File tree

‎backends/arm/runtime/VGFBackend.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,6 +1422,11 @@ VkResult vkml_allocate_basics(
14221422
return result;
14231423
}
14241424

1425+
// VGF_FUSED_GRAPHICS_QUEUE_BEGIN: optional strict queue selection.
1426+
const char* graphics_env =
1427+
std::getenv("EXECUTORCH_VGF_FUSED_REQUIRE_GRAPHICS_QUEUE");
1428+
const bool require_graphics_queue = graphics_env != nullptr &&
1429+
graphics_env[0] == '1' && graphics_env[1] == '\0';
14251430
// Find suitable queue family
14261431
uint32_t qf_count;
14271432
vkGetPhysicalDeviceQueueFamilyProperties(
@@ -1431,14 +1436,24 @@ VkResult vkml_allocate_basics(
14311436
*physical_device, &qf_count, qps.data());
14321437
uint32_t qf = UINT32_MAX;
14331438
for (uint32_t i = 0; i < qf_count; ++i) {
1434-
if (qps[i].queueFlags &
1435-
(VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM)) {
1439+
const VkQueueFlags flags = qps[i].queueFlags;
1440+
const VkQueueFlags fused_required = VK_QUEUE_GRAPHICS_BIT |
1441+
VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM;
1442+
if (require_graphics_queue
1443+
? (flags & fused_required) == fused_required
1444+
: (flags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM)) !=
1445+
0) {
14361446
qf = i;
14371447
break;
14381448
}
14391449
}
14401450
if (qf == UINT32_MAX) {
1441-
ET_LOG(Error, "Failed to find suitable queue");
1451+
ET_LOG(
1452+
Error,
1453+
"%s",
1454+
require_graphics_queue
1455+
? "No queue family supports graphics + compute + data graph"
1456+
: "Failed to find suitable queue");
14421457
return VK_ERROR_UNKNOWN;
14431458
}
14441459
if (queue_family_index != nullptr) {
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright 2026 Arm Limited and/or its affiliates.
3+
*
4+
* This source code is licensed under the BSD-style license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*/
7+
#pragma once
8+
9+
#include <cstdint>
10+
11+
#include <executorch/backends/vulkan/runtime/vk_api/vk_api.h>
12+
13+
#if defined(__GNUC__) || defined(__clang__)
14+
#define VGF_FUSED_STREAM_API __attribute__((visibility("default")))
15+
#else
16+
#define VGF_FUSED_STREAM_API
17+
#endif
18+
19+
namespace executorch::backends::vgf {
20+
21+
// Callback runs synchronously inside VGFBackend::execute(), after input
22+
// preparation and before queue submission, while the backend's queue/pool
23+
// mutex is held. Never re-enter ExecuTorch/VGF from this callback.
24+
//
25+
// Record *executable primary* producer and consumer command buffers with
26+
// `device`, for `queue_family_index`. Use a separate application-owned command
27+
// pool for this device/family, rather than mutating VGF's command pool.
28+
// Both commands must remain valid through the return of Method::execute().
29+
// The caller is responsible for producer->VGF and VGF->consumer barriers.
30+
struct VgfFusedDeviceContext {
31+
VkPhysicalDevice physical_device = VK_NULL_HANDLE;
32+
VkDevice device = VK_NULL_HANDLE;
33+
uint32_t queue_family_index = UINT32_MAX;
34+
};
35+
36+
using VgfPrepareFrameCommands = bool (*)(
37+
void* user_data,
38+
const VgfFusedDeviceContext& context,
39+
VkCommandBuffer* producer,
40+
VkCommandBuffer* consumer) noexcept;
41+
42+
// Exactly one VGF delegate execution is allowed per request. This is a
43+
// synchronous, non-owning API. Register it on the same thread that calls
44+
// Method::execute(), and retain its resources until execute returns.
45+
//
46+
// Without prepare(), pre-recorded commands require both expected_device and
47+
// expected_queue_family_index to avoid submitting on the wrong device.
48+
// With prepare(), the callback receives VGF's actual device and family.
49+
struct VgfFusedCommandStreamRequest {
50+
VgfPrepareFrameCommands prepare = nullptr;
51+
void* user_data = nullptr;
52+
VkCommandBuffer producer = VK_NULL_HANDLE;
53+
VkCommandBuffer consumer = VK_NULL_HANDLE;
54+
VkDevice expected_device = VK_NULL_HANDLE;
55+
uint32_t expected_queue_family_index = UINT32_MAX;
56+
bool attempted = false;
57+
bool submitted = false;
58+
bool completed = false;
59+
VkResult submit_result = VK_NOT_READY;
60+
VkResult wait_result = VK_NOT_READY;
61+
};
62+
63+
// Fails closed for overlapping/nested registrations on the same thread.
64+
// The request is not owned by the runtime.
65+
VGF_FUSED_STREAM_API bool vgf_try_register_fused_command_stream(
66+
VgfFusedCommandStreamRequest* request) noexcept;
67+
VGF_FUSED_STREAM_API void vgf_unregister_fused_command_stream(
68+
const VgfFusedCommandStreamRequest* request) noexcept;
69+
70+
class VgfFusedCommandStreamScope final {
71+
public:
72+
explicit VgfFusedCommandStreamScope(
73+
VgfFusedCommandStreamRequest& request) noexcept
74+
: request_(&request),
75+
active_(vgf_try_register_fused_command_stream(request_)) {}
76+
77+
~VgfFusedCommandStreamScope() {
78+
if (active_) {
79+
vgf_unregister_fused_command_stream(request_);
80+
}
81+
}
82+
83+
bool active() const noexcept {
84+
return active_;
85+
}
86+
87+
VgfFusedCommandStreamScope(const VgfFusedCommandStreamScope&) = delete;
88+
VgfFusedCommandStreamScope& operator=(const VgfFusedCommandStreamScope&) =
89+
delete;
90+
91+
private:
92+
VgfFusedCommandStreamRequest* request_ = nullptr;
93+
bool active_ = false;
94+
};
95+
96+
} // namespace executorch::backends::vgf

‎backends/arm/runtime/VGFSetup.cpp‎

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
* appropriate vulkan structures.
1111
*/
1212

13+
#include <executorch/backends/arm/runtime/VGFCommandStream.h>
1314
#include <executorch/backends/arm/runtime/VGFExecutionStats.h>
1415
#include <executorch/backends/arm/runtime/VGFMutableAlias.h>
1516
#include <executorch/backends/arm/runtime/VGFSetup.h>
@@ -5241,6 +5242,30 @@ bool VgfRepr::record_execute_command_buffer(VkCommandBuffer command_buffer) {
52415242
return true;
52425243
}
52435244

5245+
// VGF_FUSED_MAIN_BEGIN (opt-in real producer/consumer commands)
5246+
namespace {
5247+
thread_local VgfFusedCommandStreamRequest* active_vgf_fused_stream = nullptr;
5248+
} // namespace
5249+
5250+
// cppcheck-suppress unusedFunction
5251+
bool vgf_try_register_fused_command_stream(
5252+
VgfFusedCommandStreamRequest* request) noexcept {
5253+
if (request == nullptr || active_vgf_fused_stream != nullptr ||
5254+
request->attempted || request->submitted || request->completed) {
5255+
return false;
5256+
}
5257+
active_vgf_fused_stream = request;
5258+
return true;
5259+
}
5260+
5261+
// cppcheck-suppress unusedFunction
5262+
void vgf_unregister_fused_command_stream(
5263+
const VgfFusedCommandStreamRequest* request) noexcept {
5264+
if (active_vgf_fused_stream == request) {
5265+
active_vgf_fused_stream = nullptr;
5266+
}
5267+
}
5268+
52445269
bool VgfRepr::execute_vgf(executorch::runtime::EventTracer* event_tracer) {
52455270
if (!execution_valid_) {
52465271
ET_LOG(
@@ -5256,14 +5281,84 @@ bool VgfRepr::execute_vgf(executorch::runtime::EventTracer* event_tracer) {
52565281
return false;
52575282
}
52585283

5284+
// Optional application command stream. The callback sees this VGF
5285+
// representation's actual VkDevice and family so it can record its own
5286+
// commands even when the generic shared-Vulkan-context bridge is absent.
5287+
// It is called under VGFBackend's mutex, and MUST NOT re-enter ExecuTorch.
5288+
VgfFusedCommandStreamRequest* fused = active_vgf_fused_stream;
5289+
std::array<VkCommandBuffer, 3> fused_commands{};
5290+
if (fused != nullptr) {
5291+
if (fused->attempted || fused->submitted || fused->completed ||
5292+
(fused->expected_device != VK_NULL_HANDLE &&
5293+
fused->expected_device != vk_device) ||
5294+
(fused->expected_queue_family_index != UINT32_MAX &&
5295+
fused->expected_queue_family_index != vk_queue_family_index) ||
5296+
(fused->prepare == nullptr &&
5297+
(fused->expected_device == VK_NULL_HANDLE ||
5298+
fused->expected_queue_family_index == UINT32_MAX))) {
5299+
ET_LOG(Error, "Invalid or reused VGF fused command-stream request");
5300+
return false;
5301+
}
5302+
// Fail closed even if the main-branch queue picker has moved into another
5303+
// translation unit and could not be changed by the installer.
5304+
// This check is active only when the strict graphics queue mode is set.
5305+
const char* graphics_required =
5306+
std::getenv("EXECUTORCH_VGF_FUSED_REQUIRE_GRAPHICS_QUEUE");
5307+
if (graphics_required != nullptr && graphics_required[0] == '1' &&
5308+
graphics_required[1] == '\0') {
5309+
uint32_t family_count = 0;
5310+
vkGetPhysicalDeviceQueueFamilyProperties(
5311+
vk_physical, &family_count, nullptr);
5312+
if (vk_queue_family_index >= family_count) {
5313+
ET_LOG(Error, "Invalid VGF fused graphics queue family index");
5314+
return false;
5315+
}
5316+
std::vector<VkQueueFamilyProperties> families(family_count);
5317+
vkGetPhysicalDeviceQueueFamilyProperties(
5318+
vk_physical, &family_count, families.data());
5319+
constexpr VkQueueFlags required = VK_QUEUE_GRAPHICS_BIT |
5320+
VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM;
5321+
if ((families[vk_queue_family_index].queueFlags & required) != required) {
5322+
ET_LOG(
5323+
Error,
5324+
"VGF fused stream needs graphics+compute+data graph queue; "
5325+
"selected family %u cannot run the requested commands",
5326+
vk_queue_family_index);
5327+
return false;
5328+
}
5329+
}
5330+
fused->attempted = true;
5331+
if (fused->prepare != nullptr) {
5332+
const VgfFusedDeviceContext context{
5333+
vk_physical, vk_device, vk_queue_family_index};
5334+
if (!fused->prepare(
5335+
fused->user_data, context, &fused->producer, &fused->consumer)) {
5336+
ET_LOG(Error, "VGF fused command recording callback failed");
5337+
return false;
5338+
}
5339+
}
5340+
if (fused->producer == VK_NULL_HANDLE ||
5341+
fused->consumer == VK_NULL_HANDLE ||
5342+
fused->producer == fused->consumer ||
5343+
fused->producer == vk_execute_cmd ||
5344+
fused->consumer == vk_execute_cmd) {
5345+
ET_LOG(Error, "VGF fused stream requires two distinct primary commands");
5346+
return false;
5347+
}
5348+
// The caller guarantees both commands were recorded on vk_device for
5349+
// vk_queue_family_index, and supplies Vulkan memory/layout barriers.
5350+
fused_commands = {fused->producer, vk_execute_cmd, fused->consumer};
5351+
}
5352+
52595353
VkSubmitInfo submit{
52605354
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
52615355
.pNext = nullptr,
52625356
.waitSemaphoreCount = 0,
52635357
.pWaitSemaphores = nullptr,
52645358
.pWaitDstStageMask = nullptr,
5265-
.commandBufferCount = 1,
5266-
.pCommandBuffers = &vk_execute_cmd,
5359+
.commandBufferCount = fused != nullptr ? 3u : 1u,
5360+
.pCommandBuffers =
5361+
fused != nullptr ? fused_commands.data() : &vk_execute_cmd,
52675362
.signalSemaphoreCount = 0,
52685363
.pSignalSemaphores = nullptr,
52695364
};
@@ -5287,6 +5382,10 @@ bool VgfRepr::execute_vgf(executorch::runtime::EventTracer* event_tracer) {
52875382
// Submit + fence wait only; reset is above.
52885383
VGF_STATS_TIME(submit_wait_ns);
52895384
result = vkQueueSubmit(vk_queue, 1, &submit, vk_execute_fence);
5385+
if (fused != nullptr) {
5386+
fused->submit_result = result;
5387+
fused->submitted = result == VK_SUCCESS;
5388+
}
52905389
if (result != VK_SUCCESS) {
52915390
ET_LOG(Error, "VGF/VkFence wait failed, error %d", result);
52925391
return false;
@@ -5299,6 +5398,10 @@ bool VgfRepr::execute_vgf(executorch::runtime::EventTracer* event_tracer) {
52995398
&vk_execute_fence,
53005399
VK_TRUE,
53015400
std::numeric_limits<uint64_t>::max());
5401+
if (fused != nullptr) {
5402+
fused->wait_result = result;
5403+
fused->completed = result == VK_SUCCESS;
5404+
}
53025405
if (result == VK_SUCCESS) {
53035406
execute_submission_pending_ = false;
53045407
}

0 commit comments

Comments
 (0)