From 6269417c1a529f43a6e8f670caefb29302231c4a Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Sat, 7 Nov 2026 15:10:49 +0000 Subject: [PATCH 1/7] Arm backend: Add shared device to VGF backend. Signed-off-by: Elena Zhelezina Change-Id: Idef95270b41ab60464d1027530d3a8ed760962a5 --- CMakeLists.txt | 11 + backends/arm/CMakeLists.txt | 53 ++- backends/arm/runtime/VGFBackend.cpp | 288 ++++++++++--- backends/arm/runtime/VGFSetup.h | 69 ++- backends/arm/runtime/VGFSharedDevice.cpp | 329 ++++++++++++++ backends/arm/runtime/VGFSharedDevice.h | 80 ++++ backends/arm/runtime/targets.bzl | 14 +- backends/arm/test/vgf_shared_device_test.cpp | 123 ++++++ backends/vulkan/CMakeLists.txt | 5 + backends/vulkan/runtime/vk_api/Runtime.cpp | 6 +- .../runtime/SharedVulkanDevice.h | 14 + examples/arm/vgf_vulkan_shared/CMakeLists.txt | 32 ++ examples/arm/vgf_vulkan_shared/README.md | 408 ++++++++++++++++++ examples/arm/vgf_vulkan_shared/export.py | 133 ++++++ examples/arm/vgf_vulkan_shared/runner.cpp | 180 ++++++++ setup.py | 9 +- 16 files changed, 1671 insertions(+), 83 deletions(-) create mode 100644 backends/arm/runtime/VGFSharedDevice.cpp create mode 100644 backends/arm/runtime/VGFSharedDevice.h create mode 100644 backends/arm/test/vgf_shared_device_test.cpp create mode 100644 examples/arm/vgf_vulkan_shared/CMakeLists.txt create mode 100644 examples/arm/vgf_vulkan_shared/README.md create mode 100644 examples/arm/vgf_vulkan_shared/export.py create mode 100644 examples/arm/vgf_vulkan_shared/runner.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 43cfe244b87..0dfe85959d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -819,6 +819,17 @@ configure_file( install(FILES ${CMAKE_CURRENT_BINARY_DIR}/executorch-backend-dependencies.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ExecuTorch ) +set(EXECUTORCH_VULKAN_SHARED_RUNTIME_REQUIRED + OFF + CACHE INTERNAL "Vulkan or VGF requires the shared registry DSO" FORCE +) +if(EXECUTORCH_BUILD_VULKAN OR EXECUTORCH_BUILD_VGF) + set(EXECUTORCH_VULKAN_SHARED_RUNTIME_REQUIRED + ON + CACHE INTERNAL "Vulkan or VGF requires the shared registry DSO" FORCE + ) +endif() + if(EXECUTORCH_BUILD_VULKAN OR EXECUTORCH_BUILD_VGF OR EXECUTORCH_BUILD_NATIVE_VULKAN diff --git a/backends/arm/CMakeLists.txt b/backends/arm/CMakeLists.txt index f6d2dfdbb0b..290bf101ca8 100644 --- a/backends/arm/CMakeLists.txt +++ b/backends/arm/CMakeLists.txt @@ -278,7 +278,9 @@ if(EXECUTORCH_BUILD_VGF) # Add backend delegate for VGF set(_vgf_backend_sources - backends/arm/runtime/VGFBackend.cpp backends/arm/runtime/VGFSetup.cpp + backends/arm/runtime/VGFBackend.cpp + backends/arm/runtime/VGFSetup.cpp + backends/arm/runtime/VGFSharedDevice.cpp backends/arm/runtime/VGFZeroCopy.cpp backends/arm/runtime/VGFNeuralStatistics.cpp ) @@ -349,6 +351,23 @@ if(EXECUTORCH_BUILD_VGF) target_link_libraries(vgf_backend PRIVATE executorch_core) target_link_libraries(vgf_backend PRIVATE vgf) + target_compile_features(vgf_backend PUBLIC cxx_std_17) + target_link_libraries(vgf_backend PUBLIC executorch_vulkan_shared_runtime) + target_compile_options( + vgf_backend PRIVATE "$<$>:-fexceptions>" + "$<$:/EHsc>" + ) + if(EXECUTORCH_BUILD_VULKAN) + target_compile_definitions( + vgf_backend PUBLIC EXECUTORCH_VGF_SHARED_DEVICE=1 + ) + target_link_libraries(vgf_backend PRIVATE vulkan_backend) + else() + target_compile_definitions( + vgf_backend PUBLIC EXECUTORCH_VGF_SHARED_DEVICE=0 + ) + endif() + # When VGF compiles volk.c itself, link libdl for dlopen/dlsym if(NOT EXECUTORCH_BUILD_VULKAN) target_link_libraries(vgf_backend PRIVATE ${CMAKE_DL_LIBS}) @@ -436,5 +455,37 @@ if(EXECUTORCH_BUILD_VGF) add_test(NAME vgf_zero_copy_test COMMAND vgf_zero_copy_test) endif() + if(EXECUTORCH_BUILD_TESTS) + add_executable( + vgf_shared_device_test + ${EXECUTORCH_ROOT}/backends/arm/test/vgf_shared_device_test.cpp + ) + target_include_directories( + vgf_shared_device_test + PRIVATE ${_common_include_directories} ${VULKAN_HEADERS_PATH} + ${VOLK_HEADERS_PATH} + ) + target_compile_options( + vgf_shared_device_test PRIVATE -DUSE_VULKAN_WRAPPER -DUSE_VULKAN_VOLK + ) + target_link_libraries(vgf_shared_device_test PRIVATE vgf_backend) + if(TARGET GTest::gtest_main) + target_link_libraries(vgf_shared_device_test PRIVATE GTest::gtest_main) + else() + target_link_libraries(vgf_shared_device_test PRIVATE gtest gtest_main) + endif() + add_test(NAME vgf_shared_device_test COMMAND vgf_shared_device_test) + endif() + + option(EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE + "Build the VGF/Vulkan Stage 4 example" OFF + ) + if(EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE) + add_subdirectory( + ${EXECUTORCH_ROOT}/examples/arm/vgf_vulkan_shared + ${CMAKE_CURRENT_BINARY_DIR}/vgf_vulkan_shared + ) + endif() + # end config for VGF builds endif() diff --git a/backends/arm/runtime/VGFBackend.cpp b/backends/arm/runtime/VGFBackend.cpp index f6535fc0c70..60a34148c51 100644 --- a/backends/arm/runtime/VGFBackend.cpp +++ b/backends/arm/runtime/VGFBackend.cpp @@ -69,6 +69,11 @@ using executorch::runtime::EventTracerEntry; #include #include #include +#include +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE +#include +#include +#endif namespace executorch { namespace backends { @@ -94,22 +99,6 @@ VkResult vkml_load_extensions(VkDevice const* device) { return VK_ERROR_UNKNOWN; } -/* - * Fetch vulkan basic objects - intended to be replaced with a shared - * device setup with the Vulkan backend. - */ -VkResult vkml_allocate_basics( - VkInstance* instance, - VkPhysicalDevice* physical_device, - VkDevice* device, - VkQueue* queue, - VkCommandPool* command_pool, - uint32_t* queue_family_index, - bool request_neural_statistics, - bool* neural_statistics_device_enabled, - bool request_host_memory_import, - VgfHostMemoryImportCapabilities* host_memory_import_capabilities); - // Runtime configuration and helper functions for VGF execution. constexpr const char* kVgfZeroCopyIoEnv = "ET_VGF_ZERO_COPY"; constexpr const char* kVgfZeroCopyBuildOption = "EXECUTORCH_VGF_ZERO_COPY_IO"; @@ -529,7 +518,9 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { neural_statistics_config_.requested, &neural_statistics_device_enabled_, request_host_memory_import, - &host_memory_import_capabilities_); + &host_memory_import_capabilities_, + &instance_lifetime_, + nullptr); if (result != VK_SUCCESS) { ET_LOG( Error, "Failed to initialize the Vulkan device error 0x%08X", result); @@ -585,6 +576,13 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { ~VGFBackend() override = default; bool is_available() const override { +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + // No RuntimeSpec is available during a backend availability probe. In a + // combined build, defer device capability checks to init(), which resolves + // the actual key/device. Probing an unrelated default GPU here can reject + // a valid externally registered device or create a redundant VkDevice. + return true; +#else auto* self = const_cast(this); const std::lock_guard lock(mutex_); ET_LOG(Debug, "Checking VGFBackend is available"); @@ -595,6 +593,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { // Existing delegates, if any, keep their shared device alive. self->release_basics_if_unused(); return available; +#endif } Result init( @@ -605,6 +604,12 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { const std::lock_guard lock(mutex_); ET_LOG(Debug, "Entered VGF init"); + auto resolved = resolve_vgf_shared_device_state(context); + if (!resolved.ok()) { + return resolved.error(); + } + const auto shared_state = resolved.get(); + #ifdef ET_EVENT_TRACER_ENABLED EventTracer* event_tracer = context.event_tracer(); @@ -620,13 +625,15 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { /*delegate_debug_id=*/-1); #endif - self->ensure_initialized(); + if (!shared_state) { + self->ensure_initialized(); + } #ifdef ET_EVENT_TRACER_ENABLED event_tracer_end_profiling_delegate(event_tracer, ensure_initialized_event); #endif - if (!is_initialized_) { + if (!shared_state && !is_initialized_) { #ifdef ET_EVENT_TRACER_ENABLED event_tracer_end_profiling_delegate(event_tracer, init_total_event); #endif @@ -656,17 +663,25 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { self->release_basics_if_unused(); return Error::MemoryAllocationFailed; } + const auto statistics_config = shared_state + ? get_vgf_neural_statistics_runtime_config() + : neural_statistics_config_; new (repr) VgfRepr( - vk_instance, - vk_physical_device, - vk_device, - vk_queue, - vk_command_pool, - vk_queue_family_index, - neural_statistics_config_.requested, - neural_statistics_device_enabled_, - neural_statistics_config_.mode_index, - host_memory_import_capabilities_); + shared_state ? shared_state->device->instance() : vk_instance, + shared_state ? shared_state->device->physical_device() + : vk_physical_device, + shared_state ? shared_state->device->device() : vk_device, + shared_state ? VK_NULL_HANDLE : vk_queue, + shared_state ? shared_state->command_pool : vk_command_pool, + shared_state ? shared_state->device->queue_family_index() + : vk_queue_family_index, + statistics_config.requested, + shared_state ? shared_state->neural_statistics_device_enabled + : neural_statistics_device_enabled_, + statistics_config.mode_index, + shared_state ? shared_state->host_memory_import_capabilities + : host_memory_import_capabilities_, + shared_state); #ifdef ET_EVENT_TRACER_ENABLED event_tracer_end_profiling_delegate(event_tracer, allocate_repr_event); @@ -677,13 +692,14 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { /*delegate_debug_id=*/-1); #endif + const bool valid_vgf = repr->with_shared_queue([&]() { #ifdef ET_EVENT_TRACER_ENABLED - auto valid_vgf = repr->process_vgf( - vgf_data, processed->size(), compile_specs, event_tracer); + return repr->process_vgf( + vgf_data, processed->size(), compile_specs, event_tracer); #else - auto valid_vgf = - repr->process_vgf(vgf_data, processed->size(), compile_specs); + return repr->process_vgf(vgf_data, processed->size(), compile_specs); #endif + }); #ifdef ET_EVENT_TRACER_ENABLED event_tracer_end_profiling_delegate(event_tracer, process_vgf_event); @@ -695,7 +711,6 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { #endif ET_LOG(Error, "Failed to process VGF blob."); // The runtime never receives this handle, so it cannot destroy it. - self->wait_for_device_idle(); repr->~VgfRepr(); self->release_basics_if_unused(); return Error::Internal; @@ -712,6 +727,9 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { repr, self->next_mapped_boundary_generation_++}); ++self->live_delegates_; + if (!shared_state) { + ++self->live_legacy_delegates_; + } return repr; } @@ -719,9 +737,21 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { BackendExecutionContext& context, DelegateHandle* handle, Span args) const override { + const std::lock_guard lock(mutex_); + if (handle == nullptr) { + return Error::InvalidArgument; + } + auto* repr = static_cast(handle); + return repr->with_shared_queue( + [&]() { return execute_locked(context.event_tracer(), handle, args); }); + } + + Error execute_locked( + executorch::runtime::EventTracer* event_tracer, + DelegateHandle* handle, + Span args) const { // The queue and command pool are shared by all VGF delegates. // Serialize their host access with initialization and teardown. - const std::lock_guard lock(mutex_); VgfRepr* repr = static_cast(handle); if (!repr->is_execution_valid()) { ET_LOG( @@ -765,7 +795,6 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { VGF_STATS_EXECUTION(handle); #ifdef ET_EVENT_TRACER_ENABLED - EventTracer* event_tracer = context.event_tracer(); EventTracerEntry vgf_execute_event = event_tracer_start_profiling_delegate( event_tracer, @@ -777,7 +806,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { "VGF_COPY_INPUTS", /*delegate_debug_id=*/-1); #else - (void)context; + (void)event_tracer; #endif const bool zero_copy_requested = vgf_zero_copy_io_build_enabled() && @@ -1080,7 +1109,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { // while the caller's EValue storage is still alive. On device loss, // synchronously destroy this representation's Vulkan children, // including imported VkDeviceMemory, before returning. - const VkResult idle_result = wait_for_device_idle(); + const VkResult idle_result = repr->wait_for_device_idle_locked(); if (idle_result == VK_ERROR_DEVICE_LOST) { ET_LOG( Error, @@ -1188,6 +1217,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { const std::lock_guard lock(mutex_); ET_CHECK_MSG(live_delegates_ > 0, "Unbalanced VGF delegate destruction"); VgfRepr* repr = static_cast(handle); + const bool legacy_device = !repr->uses_shared_device(); // destroy() already holds mutex_; unregister before freeing the IO. self->mapped_boundary_reprs_.erase( std::remove_if( @@ -1199,35 +1229,25 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { self->mapped_boundary_reprs_.end()); // A failed execution may have submitted work without completing its // fence wait. Drain it before freeing any per-delegate resources. - self->wait_for_device_idle(); repr->~VgfRepr(); --self->live_delegates_; + if (legacy_device) { + ET_CHECK_MSG( + live_legacy_delegates_ > 0, "Unbalanced legacy VGF destruction"); + --self->live_legacy_delegates_; + } self->release_basics_if_unused(); } private: // Call only with mutex_ held. Return the idle result so execute() can // synchronously tear down its VgfRepr on terminal device loss. - VkResult wait_for_device_idle() const { - if (vk_device == VK_NULL_HANDLE) { - return VK_SUCCESS; - } - - const VkResult result = vkDeviceWaitIdle(vk_device); - if (result != VK_SUCCESS) { - ET_LOG(Error, "VGF teardown: vkDeviceWaitIdle failed: %d", result); - } - ET_CHECK_MSG( - result == VK_SUCCESS || result == VK_ERROR_DEVICE_LOST, - "Cannot free VGF resources while device completion is unknown"); - return result; - } // Call only with mutex_ held, after destroying the last VgfRepr (or // when a probe/initialization failed without publishing a handle). // Any submitted work must have been drained before freeing the repr. void release_basics_if_unused() { - if (live_delegates_ != 0) { + if (live_legacy_delegates_ != 0) { return; } if (vk_command_pool != VK_NULL_HANDLE) { @@ -1239,9 +1259,12 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { vk_device = VK_NULL_HANDLE; } if (vk_instance != VK_NULL_HANDLE) { - vkDestroyInstance(vk_instance, nullptr); + if (!instance_lifetime_) { + vkDestroyInstance(vk_instance, nullptr); + } vk_instance = VK_NULL_HANDLE; } + instance_lifetime_.reset(); vk_physical_device = VK_NULL_HANDLE; vk_queue = VK_NULL_HANDLE; vk_queue_family_index = UINT32_MAX; @@ -1263,6 +1286,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { std::vector mapped_boundary_reprs_; uint64_t next_mapped_boundary_generation_ = 1; size_t live_delegates_ = 0; + size_t live_legacy_delegates_ = 0; VkInstance vk_instance = VK_NULL_HANDLE; VkPhysicalDevice vk_physical_device = VK_NULL_HANDLE; VkDevice vk_device = VK_NULL_HANDLE; @@ -1272,6 +1296,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { VgfNeuralStatisticsRuntimeConfig neural_statistics_config_{}; bool neural_statistics_device_enabled_ = false; VgfHostMemoryImportCapabilities host_memory_import_capabilities_{}; + std::shared_ptr instance_lifetime_; bool is_initialized_ = false; }; @@ -1310,7 +1335,9 @@ VkResult vkml_allocate_basics( bool request_neural_statistics, bool* neural_statistics_device_enabled, bool request_host_memory_import, - VgfHostMemoryImportCapabilities* host_memory_import_capabilities) { + VgfHostMemoryImportCapabilities* host_memory_import_capabilities, + std::shared_ptr* instance_lifetime, + vulkan_shared::SharedVulkanDeviceCreateInfo* shared_info) { VkResult result; if (neural_statistics_device_enabled != nullptr) { @@ -1320,6 +1347,25 @@ VkResult vkml_allocate_basics( *host_memory_import_capabilities = {}; } +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + // One instance-loaded Volk dispatch domain, including availability probes and + // non-shared VGF execution in a combined build. Never switch global dispatch + // to a private instance or a single logical device. + if (instance_lifetime == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + try { + auto owner = vkcompute::vkapi::runtime_owner(); + *instance = owner->instance(); + *instance_lifetime = std::move(owner); + } catch (const std::exception& error) { + ET_LOG(Error, "VGF canonical instance creation failed: %s", error.what()); + return VK_ERROR_INITIALIZATION_FAILED; + } + if (!vkCreateTensorARM) { + return VK_ERROR_FEATURE_NOT_PRESENT; + } +#else if (VK_SUCCESS != volkInitialize()) { ET_LOG(Error, "Volk failed to initialize"); return VK_ERROR_INITIALIZATION_FAILED; @@ -1407,6 +1453,8 @@ VkResult vkml_allocate_basics( return VK_ERROR_FEATURE_NOT_PRESENT; } +#endif + // Pick first GPU uint32_t gpu_count = 0; vkEnumeratePhysicalDevices(*instance, &gpu_count, nullptr); @@ -1422,6 +1470,13 @@ VkResult vkml_allocate_basics( return result; } + VkPhysicalDeviceProperties device_properties{}; + vkGetPhysicalDeviceProperties(*physical_device, &device_properties); + if (device_properties.apiVersion < VK_API_VERSION_1_3) { + ET_LOG(Error, "VGF requires Vulkan 1.3"); + return VK_ERROR_INCOMPATIBLE_DRIVER; + } + // Find suitable queue family uint32_t qf_count; vkGetPhysicalDeviceQueueFamilyProperties( @@ -1431,8 +1486,9 @@ VkResult vkml_allocate_basics( *physical_device, &qf_count, qps.data()); uint32_t qf = UINT32_MAX; for (uint32_t i = 0; i < qf_count; ++i) { - if (qps[i].queueFlags & - (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM)) { + constexpr VkQueueFlags required = + VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM; + if (qps[i].queueCount > 0 && (qps[i].queueFlags & required) == required) { qf = i; break; } @@ -1457,9 +1513,19 @@ VkResult vkml_allocate_basics( }; // Query features + VkPhysicalDeviceTensorFeaturesARM available_tensor{}; + available_tensor.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TENSOR_FEATURES_ARM; + VkPhysicalDeviceShaderReplicatedCompositesFeaturesEXT available_replicated{}; + available_replicated.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_REPLICATED_COMPOSITES_FEATURES_EXT; + available_replicated.pNext = &available_tensor; + VkPhysicalDeviceVulkan13Features available_13{}; + available_13.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; + available_13.pNext = &available_replicated; VkPhysicalDeviceShaderBfloat16FeaturesKHR available_bfloat16{ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR, - .pNext = nullptr, + .pNext = &available_13, }; VkPhysicalDeviceVulkan12Features available_12 = { .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, @@ -1500,6 +1566,16 @@ VkResult vkml_allocate_basics( return VK_ERROR_FEATURE_NOT_PRESENT; } + if (!available_tensor.tensors || !available_tensor.shaderTensorAccess || + !available_13.synchronization2 || !available_13.maintenance4 || + !available_13.pipelineCreationCacheControl || + !available_12.hostQueryReset || !available_12.storageBuffer8BitAccess || + !available_12.shaderInt8 || !available_12.vulkanMemoryModel || + !available_2.features.shaderInt16 || !available_2.features.shaderInt64) { + ET_LOG(Error, "Required VGF device features are unavailable"); + return VK_ERROR_FEATURE_NOT_PRESENT; + } + // Select features VkPhysicalDeviceShaderBfloat16FeaturesKHR features_bfloat16{ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR, @@ -1509,13 +1585,13 @@ VkResult vkml_allocate_basics( VkPhysicalDeviceShaderReplicatedCompositesFeaturesEXT features_c{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_REPLICATED_COMPOSITES_FEATURES_EXT, nullptr}; - features_c.shaderReplicatedComposites = true; + features_c.shaderReplicatedComposites = VK_FALSE; VkPhysicalDeviceVulkan13Features features_13{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, nullptr}; features_13.synchronization2 = true; features_13.maintenance4 = true; features_13.pipelineCreationCacheControl = true; - features_13.pNext = &features_c; + features_13.pNext = nullptr; VkPhysicalDeviceVulkan12Features features_12{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, nullptr}; features_12.hostQueryReset = true; @@ -1638,7 +1714,7 @@ VkResult vkml_allocate_basics( if (bfloat16_extension_available && bfloat16_feature_available) { requested_exts.push_back(VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME); features_bfloat16.shaderBFloat16Type = VK_TRUE; - features_c.pNext = &features_bfloat16; + features_13.pNext = &features_bfloat16; ET_LOG( Info, "Enabled %s with shaderBFloat16Type", @@ -1701,6 +1777,28 @@ VkResult vkml_allocate_basics( } #endif + // This extension enables the optional SPIR-V ReplicatedCompositesEXT + // capability. Models that actually use it still need device support. + const bool replicated_composites_extension_available = std::any_of( + available.begin(), available.end(), [](const auto& extension) { + return std::strcmp( + extension.extensionName, + VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME) == 0; + }); + if (replicated_composites_extension_available && + available_replicated.shaderReplicatedComposites == VK_TRUE) { + features_c.shaderReplicatedComposites = VK_TRUE; + // Chain on top of BF16, if BF16 was independently selected above. + features_c.pNext = features_13.pNext; + features_13.pNext = &features_c; + } else { + ET_LOG( + Info, + "Optional VK_EXT_shader_replicated_composites is unsupported; " + "VGF shaders using ReplicatedCompositesEXT will require a " + "different driver or shader recompilation"); + } + for (auto& ext : dev_exts) { bool found = false; for (auto const& ext_avail : available) { @@ -1710,7 +1808,14 @@ VkResult vkml_allocate_basics( } } if (found == false) { - ET_LOG(Info, "Failed to find extension %s", ext); + // maintenance4 is enabled through core Vulkan 1.3 above. + if (std::strcmp(ext, "VK_KHR_maintenance4") == 0 || + std::strcmp( + ext, VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME) == 0) { + continue; + } + ET_LOG(Error, "Required VGF extension is missing: %s", ext); + return VK_ERROR_EXTENSION_NOT_PRESENT; } } @@ -1730,8 +1835,7 @@ VkResult vkml_allocate_basics( *device = VK_NULL_HANDLE; return result; } - // Load the device with volk and populate function pointers - volkLoadDevice(*device); + // Keep instance-loaded loader dispatch; multiple VkDevices coexist. if (host_memory_import_capabilities != nullptr) { // Only a successful vkCreateDevice makes extension enablement @@ -1756,6 +1860,60 @@ VkResult vkml_allocate_basics( vkGetDeviceQueue(*device, qf, 0, queue); + if (shared_info != nullptr) { + shared_info->instance = *instance; + shared_info->physical_device = *physical_device; + shared_info->device = *device; + shared_info->queue = *queue; + shared_info->queue_family_index = qf; + shared_info->enabled_device_extensions.clear(); + for (const char* extension : requested_exts) { + shared_info->enabled_device_extensions.emplace_back(extension); + } + auto& f = shared_info->enabled_device_features; + f = {}; + f.shader_int16 = device_features.shaderInt16 == VK_TRUE; + f.shader_int64 = device_features.shaderInt64 == VK_TRUE; + f.shader_float64 = device_features.shaderFloat64 == VK_TRUE; + f.storage_buffer_16bit_access = + features_11.storageBuffer16BitAccess == VK_TRUE; + f.uniform_and_storage_buffer_16bit_access = + features_11.uniformAndStorageBuffer16BitAccess == VK_TRUE; + f.storage_push_constant_16 = features_11.storagePushConstant16 == VK_TRUE; + f.storage_input_output_16 = features_11.storageInputOutput16 == VK_TRUE; + f.storage_buffer_8bit_access = + features_12.storageBuffer8BitAccess == VK_TRUE; + f.uniform_and_storage_buffer_8bit_access = + features_12.uniformAndStorageBuffer8BitAccess == VK_TRUE; + f.storage_push_constant_8 = features_12.storagePushConstant8 == VK_TRUE; + f.shader_float16 = features_12.shaderFloat16 == VK_TRUE; + f.shader_int8 = features_12.shaderInt8 == VK_TRUE; + f.arm_tensors = features_tensor.tensors == VK_TRUE; + f.arm_shader_tensor_access = features_tensor.shaderTensorAccess == VK_TRUE; + f.arm_data_graph = features_graph.dataGraph == VK_TRUE; + f.arm_data_graph_shader_module = + features_graph.dataGraphShaderModule == VK_TRUE; + f.synchronization2 = features_13.synchronization2 == VK_TRUE; + f.maintenance4 = features_13.maintenance4 == VK_TRUE; + f.pipeline_creation_cache_control = + features_13.pipelineCreationCacheControl == VK_TRUE; + f.host_query_reset = features_12.hostQueryReset == VK_TRUE; + f.vulkan_memory_model = features_12.vulkanMemoryModel == VK_TRUE; + f.vulkan_memory_model_device_scope = + features_12.vulkanMemoryModelDeviceScope == VK_TRUE; + f.shader_replicated_composites = + features_c.shaderReplicatedComposites == VK_TRUE; + f.shader_bfloat16 = features_bfloat16.shaderBFloat16Type == VK_TRUE; +#if defined(VK_ARM_data_graph_neural_accelerator_statistics) + f.arm_data_graph_neural_statistics = + features_neural_statistics.dataGraphNeuralAcceleratorStatistics == + VK_TRUE; +#endif + } + if (command_pool == nullptr) { + return VK_SUCCESS; + } + VkCommandPoolCreateInfo poolInfo{ .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .pNext = nullptr, diff --git a/backends/arm/runtime/VGFSetup.h b/backends/arm/runtime/VGFSetup.h index 31124d8604b..3fee532679a 100644 --- a/backends/arm/runtime/VGFSetup.h +++ b/backends/arm/runtime/VGFSetup.h @@ -23,8 +23,10 @@ using executorch::runtime::CompileSpec; #include #include +#include #include #include +#include namespace executorch { namespace backends { @@ -122,8 +124,10 @@ class VgfRepr { bool neural_statistics_requested = false, bool neural_statistics_device_enabled = false, int neural_statistics_mode_index = 1, - VgfHostMemoryImportCapabilities host_memory_import_capabilities = {}) - : vk_instance(inst), + VgfHostMemoryImportCapabilities host_memory_import_capabilities = {}, + std::shared_ptr shared_device_state = {}) + : shared_device_state_(std::move(shared_device_state)), + vk_instance(inst), vk_physical(phys), vk_device(dev), vk_queue(queue), @@ -269,19 +273,66 @@ class VgfRepr { // tensor view; imported Vulkan objects are destroyed only after no queued // work can still use them. The next execute_vgf() re-records the command // buffer through the existing dirty-binding path. - ~VgfRepr() { - free_vgf(); - if (vk_pipeline_cache != VK_NULL_HANDLE) { - // The cache is private to this VgfRepr, so no other delegate instance can - // be accessing it while this object is being destroyed. - vkDestroyPipelineCache(vk_device, vk_pipeline_cache, nullptr); - vk_pipeline_cache = VK_NULL_HANDLE; + + bool uses_shared_device() const { + return static_cast(shared_device_state_); + } + + // The callback covers all queue use, including constant uploads and cleanup. + // VkQueue is installed only while the registry's queue mutex is held. + template + decltype(auto) with_shared_queue(Fn&& fn) { + if (!shared_device_state_) { + return std::forward(fn)(); + } + return shared_device_state_->device->with_locked_queue( + [&](VkQueue queue) -> decltype(auto) { + ET_CHECK_MSG( + vk_queue == VK_NULL_HANDLE, "Nested VGF shared queue lease"); + struct ResetQueue final { + VkQueue& slot; + ~ResetQueue() { + slot = VK_NULL_HANDLE; + } + }; + vk_queue = queue; + ResetQueue reset{vk_queue}; + return std::forward(fn)(); + }); + } + + // Called only inside with_shared_queue(), or under the legacy backend mutex. + VkResult wait_for_device_idle_locked() const { + if (vk_device == VK_NULL_HANDLE) { + return VK_SUCCESS; } + // Do NOT wait an entire externally owned device: unrelated application + // queues are outside the shared queue's host-synchronization domain. + const VkResult result = shared_device_state_ ? vkQueueWaitIdle(vk_queue) + : vkDeviceWaitIdle(vk_device); + ET_CHECK_MSG( + result == VK_SUCCESS || result == VK_ERROR_DEVICE_LOST, + "Cannot free VGF resources while completion is unknown"); + return result; + } + + ~VgfRepr() { + with_shared_queue([&]() { + wait_for_device_idle_locked(); + free_vgf(); + if (vk_pipeline_cache != VK_NULL_HANDLE) { + // The cache is private to this VgfRepr, so no other delegate instance + // can be accessing it while this object is being destroyed. + vkDestroyPipelineCache(vk_device, vk_pipeline_cache, nullptr); + vk_pipeline_cache = VK_NULL_HANDLE; + } + }); } private: // false after terminal cleanup; the DelegateHandle remains valid only // for destroy(). + std::shared_ptr shared_device_state_; bool execution_valid_ = true; // Basic Vulkan objects passed to us and re-used VkInstance vk_instance; diff --git a/backends/arm/runtime/VGFSharedDevice.cpp b/backends/arm/runtime/VGFSharedDevice.cpp new file mode 100644 index 00000000000..b8c3cca9257 --- /dev/null +++ b/backends/arm/runtime/VGFSharedDevice.cpp @@ -0,0 +1,329 @@ +/* + * Copyright 2026 Arm Limited and/or its affiliates. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include +#include +#include +#include + +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE +#include +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace backends { +namespace vgf { +using runtime::Error; +using runtime::Result; +using namespace vulkan_shared; + +VgfSharedDeviceState::~VgfSharedDeviceState() { + // VgfRepr drains its queue and frees its children before releasing this + // state. + if (command_pool != VK_NULL_HANDLE && device) { + vkDestroyCommandPool(device->device(), command_pool, nullptr); + } +} + +Result parse_vgf_shared_device_config( + const runtime::BackendInitContext& context) { + auto parsed = parse_shared_vulkan_runtime_config(context); + if (!parsed.ok()) { + return parsed.error(); + } + const auto name = + context.get_runtime_spec(kSharedDeviceNameOption); + const auto legacy_name = + context.get_runtime_spec(kLegacySharedContextNameOption); + const auto mode = + context.get_runtime_spec(kSharedDeviceModeOption); + const auto legacy_mode = + context.get_runtime_spec(kLegacySharedContextModeOption); + const auto group = context.get_runtime_spec(kSharedGroupIdOption); + const auto missing = [](const auto& result) { + return !result.ok() && result.error() == Error::NotFound; + }; + if (missing(name) && missing(legacy_name) && missing(mode) && + missing(legacy_mode) && missing(group)) { + parsed->device_mode = SharedDeviceMode::kDisabled; + } + return parsed; +} + +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE +namespace { +Error validate_device(const SharedVulkanDevicePtr& device) { + if (!device || !device->is_valid()) { + return Error::DelegateInvalidCompatibility; + } + // We use global instance-loaded Volk dispatch. Never overwrite it with + // volkLoadDevice(), or accept a device from an unrelated instance. + if (device->instance() != vkcompute::vkapi::runtime()->instance()) { + ET_LOG(Error, "VGF shared device must use the ExecuTorch Vulkan instance"); + return Error::DelegateInvalidCompatibility; + } + static constexpr const char* required_extensions[] = { + "VK_ARM_tensors", + "VK_ARM_data_graph", + "VK_KHR_maintenance5", + "VK_KHR_deferred_host_operations"}; + for (const char* extension : required_extensions) { + if (!device->has_device_extension(extension)) { + ET_LOG(Error, "VGF shared device did not enable %s", extension); + return Error::DelegateInvalidCompatibility; + } + } + // An external device must not claim a feature that was not enabled at + // vkCreateDevice. The extension is optional, but its metadata must agree. + if (device->enabled_device_features().shader_replicated_composites && + !device->has_device_extension( + VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME)) { + ET_LOG( + Error, "Replicated-composites feature requires the enabled extension"); + return Error::DelegateInvalidCompatibility; + } + if (!vgf_shared_features_compatible(device->enabled_device_features())) { + ET_LOG( + Error, + "VGF shared device lacks required ENABLED feature metadata; " + "pre-create a VGF-capable device before loading both delegates"); + return Error::DelegateInvalidCompatibility; + } + VkPhysicalDeviceProperties properties{}; + vkGetPhysicalDeviceProperties(device->physical_device(), &properties); + if (properties.apiVersion < VK_API_VERSION_1_3) { + ET_LOG(Error, "VGF shared execution requires Vulkan 1.3"); + return Error::DelegateInvalidCompatibility; + } + uint32_t count = 0; + vkGetPhysicalDeviceQueueFamilyProperties( + device->physical_device(), &count, nullptr); + std::vector families(count); + vkGetPhysicalDeviceQueueFamilyProperties( + device->physical_device(), &count, families.data()); + const uint32_t family = device->queue_family_index(); + constexpr VkQueueFlags required = + VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM; + if (family >= count || families[family].queueCount == 0 || + (families[family].queueFlags & required) != required) { + ET_LOG(Error, "VGF shared queue must support BOTH compute and data graph"); + return Error::DelegateInvalidCompatibility; + } + static constexpr const char* required_commands[] = { + "vkCreateTensorARM", + "vkDestroyTensorARM", + "vkCreateTensorViewARM", + "vkDestroyTensorViewARM", + "vkGetTensorMemoryRequirementsARM", + "vkBindTensorMemoryARM", + "vkCreateDataGraphPipelinesARM", + "vkCmdDispatchDataGraphARM", + "vkCreateDataGraphPipelineSessionARM", + "vkCmdPipelineBarrier2"}; + for (const char* command : required_commands) { + if (vkGetDeviceProcAddr(device->device(), command) == nullptr) { + ET_LOG(Error, "VGF shared device is missing %s", command); + return Error::DelegateInvalidCompatibility; + } + } + const auto& features = device->enabled_device_features(); + if (features.arm_data_graph_neural_statistics && + !device->has_device_extension( + "VK_ARM_data_graph_neural_accelerator_statistics")) { + return Error::DelegateInvalidCompatibility; + } + if (features.shader_bfloat16 && + !device->has_device_extension("VK_KHR_shader_bfloat16")) { + return Error::DelegateInvalidCompatibility; + } + return Error::Ok; +} + +Result create_device(const SharedVulkanDeviceKey& key) { + struct DeviceOwner final { + // Destroy the VkDevice before releasing the owner of its VkInstance. + std::shared_ptr instance_lifetime; + VkDevice device = VK_NULL_HANDLE; + ~DeviceOwner() { + if (device != VK_NULL_HANDLE) { + vkDestroyDevice(device, nullptr); + } + } + }; + auto owner = std::make_shared(); + SharedVulkanDeviceCreateInfo info; + info.key = key; + const auto statistics = get_vgf_neural_statistics_runtime_config(); + bool statistics_enabled = false; + VgfHostMemoryImportCapabilities imports{}; + const VkResult result = vkml_allocate_basics( + &info.instance, + &info.physical_device, + &owner->device, + &info.queue, + nullptr, + &info.queue_family_index, + statistics.requested, + &statistics_enabled, + vgf_zero_copy_io_build_enabled(), + &imports, + &owner->instance_lifetime, + &info); + if (result != VK_SUCCESS) { + ET_LOG(Error, "VGF shared device creation failed: %d", result); + return Error::DelegateInvalidCompatibility; + } + info.device = owner->device; + info.lifetime_anchor = owner; + auto device = std::make_shared(std::move(info)); + const Error compatible = validate_device(device); + if (compatible != Error::Ok) { + return compatible; + } + ET_LOG( + Info, + "VGF shared device created: name=%s group=%d device=%p", + key.device_name.c_str(), + key.group_id, + static_cast(device->device())); + return device; +} + +VgfHostMemoryImportCapabilities import_capabilities( + const SharedVulkanDevice& device) { + VgfHostMemoryImportCapabilities result{}; +#if defined(VK_EXT_external_memory_host) + uint32_t count = 0; + if (vkEnumerateDeviceExtensionProperties( + device.physical_device(), nullptr, &count, nullptr) != VK_SUCCESS) { + return result; + } + std::vector extensions(count); + if (vkEnumerateDeviceExtensionProperties( + device.physical_device(), nullptr, &count, extensions.data()) != + VK_SUCCESS) { + return result; + } + result.physical_device_advertised = std::any_of( + extensions.begin(), extensions.end(), [](const auto& extension) { + return std::strcmp( + extension.extensionName, + VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) == 0; + }); + result.logical_device_enabled = + device.has_device_extension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + if (result.physical_device_advertised) { + VkPhysicalDeviceExternalMemoryHostPropertiesEXT host{}; + host.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT; + VkPhysicalDeviceProperties2 properties{}; + properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + properties.pNext = &host; + vkGetPhysicalDeviceProperties2(device.physical_device(), &properties); + result.min_imported_host_pointer_alignment = + host.minImportedHostPointerAlignment; + } +#endif + return result; +} +} // namespace +#endif + +Result resolve_vgf_shared_device( + const SharedVulkanRuntimeConfig& config) { + if (!config.enabled()) { + return SharedVulkanDevicePtr{}; + } +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + try { + const SharedVulkanDeviceKey key{config.device_name, config.group_id}; + // Reuse single-creator/provenance policy, including create_only. + auto result = vulkan::resolve_vulkan_shared_device( + config, [&]() { return create_device(key); }); + if (!result.ok()) { + return result.error(); + } + const Error compatible = validate_device(result.get()); + if (compatible != Error::Ok) { + return compatible; + } + return result; + } catch (const std::bad_alloc&) { + return Error::MemoryAllocationFailed; + } catch (const std::exception& error) { + ET_LOG(Error, "VGF shared device resolution failed: %s", error.what()); + return Error::DelegateInvalidCompatibility; + } +#else + ET_LOG( + Error, + "VGF shared execution requires EXECUTORCH_BUILD_VULKAN=ON " + "and EXECUTORCH_BUILD_VGF=ON"); + return Error::NotSupported; +#endif +} + +Result> resolve_vgf_shared_device_state( + const runtime::BackendInitContext& context) { + auto config = parse_vgf_shared_device_config(context); + if (!config.ok()) { + return config.error(); + } + auto shared = resolve_vgf_shared_device(config.get()); + if (!shared.ok()) { + return shared.error(); + } + if (!shared.get()) { + return std::shared_ptr{}; + } +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + try { + auto state = std::make_shared(); + state->device = shared.get(); + state->neural_statistics_device_enabled = + state->device->enabled_device_features() + .arm_data_graph_neural_statistics; + state->host_memory_import_capabilities = + import_capabilities(*state->device); + VkCommandPoolCreateInfo pool{}; + pool.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pool.queueFamilyIndex = state->device->queue_family_index(); + VkCommandPool created_pool = VK_NULL_HANDLE; + const auto status = vkCreateCommandPool( + state->device->device(), &pool, nullptr, &created_pool); + if (status != VK_SUCCESS) { + ET_LOG(Error, "VGF shared command pool creation failed: %d", status); + return Error::Internal; + } + state->command_pool = created_pool; + ET_LOG( + Info, + "VGF shared init: name=%s group=%d device=%p pool=%p", + state->device->key().device_name.c_str(), + state->device->key().group_id, + static_cast(state->device->device()), + reinterpret_cast(state->command_pool)); + return state; + } catch (const std::bad_alloc&) { + return Error::MemoryAllocationFailed; + } +#else + return Error::NotSupported; +#endif +} + +} // namespace vgf +} // namespace backends +} // namespace executorch diff --git a/backends/arm/runtime/VGFSharedDevice.h b/backends/arm/runtime/VGFSharedDevice.h new file mode 100644 index 00000000000..30fe73664e0 --- /dev/null +++ b/backends/arm/runtime/VGFSharedDevice.h @@ -0,0 +1,80 @@ +/* + * Copyright 2026 Arm Limited and/or its affiliates. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace executorch { +namespace backends { +namespace vgf { + +// Each representation has its own pool, but retains the registry's device. +struct VgfSharedDeviceState final { + vulkan_shared::SharedVulkanDevicePtr device; + VkCommandPool command_pool = VK_NULL_HANDLE; + bool neural_statistics_device_enabled = false; + VgfHostMemoryImportCapabilities host_memory_import_capabilities{}; + + VgfSharedDeviceState() = default; + VgfSharedDeviceState(const VgfSharedDeviceState&) = delete; + VgfSharedDeviceState& operator=(const VgfSharedDeviceState&) = delete; + ~VgfSharedDeviceState(); +}; + +// This predicate deliberately consumes ENABLED logical-device features, not +// vkGetPhysicalDeviceFeatures2 support bits. It can be tested without a GPU. +inline bool vgf_shared_features_compatible( + const vulkan_shared::SharedVulkanDeviceFeatures& f) { + return f.arm_tensors && f.arm_shader_tensor_access && f.arm_data_graph && + f.arm_data_graph_shader_module && f.shader_int16 && f.shader_int64 && + f.shader_int8 && f.storage_buffer_8bit_access && f.host_query_reset && + f.synchronization2 && f.maintenance4 && + f.pipeline_creation_cache_control && f.vulkan_memory_model; + // Replicated composites are + // optional for models that do not use them. +} + +runtime::Result +parse_vgf_shared_device_config(const runtime::BackendInitContext& context); + +// Public pre-registration seam: create a VGF-capable device BEFORE loading a +// mixed method, then use lookup_only for BOTH delegates. This removes +// dependence on the order in which ExecuTorch initializes its serialized +// delegate table. +runtime::Result resolve_vgf_shared_device( + const vulkan_shared::SharedVulkanRuntimeConfig& config); + +runtime::Result> +resolve_vgf_shared_device_state(const runtime::BackendInitContext& context); + +// Existing VGF device creation, with authoritative metadata and an optional +// instance lifetime owner. A null command_pool skips per-delegate pool +// creation. +VkResult vkml_allocate_basics( + VkInstance* instance, + VkPhysicalDevice* physical_device, + VkDevice* device, + VkQueue* queue, + VkCommandPool* command_pool, + uint32_t* queue_family_index, + bool request_neural_statistics, + bool* neural_statistics_device_enabled, + bool request_host_memory_import, + VgfHostMemoryImportCapabilities* host_memory_import_capabilities, + std::shared_ptr* instance_lifetime, + vulkan_shared::SharedVulkanDeviceCreateInfo* shared_info); + +} // namespace vgf +} // namespace backends +} // namespace executorch diff --git a/backends/arm/runtime/targets.bzl b/backends/arm/runtime/targets.bzl index c6e352df48c..06430101e78 100644 --- a/backends/arm/runtime/targets.bzl +++ b/backends/arm/runtime/targets.bzl @@ -109,6 +109,7 @@ def define_common_targets(): # Linking from a separate static library causes the linker to # drop the symbols when building a shared library. "fbsource//third-party/vulkan-headers-1.4.343/v1.4.343/src:volk_arm_src", + "VGFSharedDevice.cpp", ], exported_headers = [ "VGFBoundaryIO.h", @@ -119,6 +120,7 @@ def define_common_targets(): "VGFShapeUtils.h", "VGFVulkanFeatures.h", "VGFZeroCopy.h", + "VGFSharedDevice.h", ], platforms = CXX, # @lint-ignore BUCKLINT: Avoid `link_whole=True` (https://fburl.com/avoid-link-whole) @@ -132,8 +134,18 @@ def define_common_targets(): "-DUSE_VULKAN_WRAPPER", "-DUSE_VULKAN_VOLK", ], - preprocessor_flags = _vgf_zero_copy_flags(), + # The Buck target compiles the shared-device shim, but VGF/Vulkan + # mixed execution needs a coordinated Vulkan/Volk build. Keep + # this disabled until that path is explicitly integrated. + preprocessor_flags = _vgf_zero_copy_flags() + [ + "-DEXECUTORCH_VGF_SHARED_DEVICE=0", + ], visibility = ["PUBLIC"], + # VGFSharedDevice.h is a public header that includes the shared + # runtime types; re-export the owner for Buck header checking. + exported_deps = [ + "//executorch/backends/vulkan_shared:runtime", + ], deps = [ "//executorch/runtime/backend:interface", "//executorch/runtime/core:core", diff --git a/backends/arm/test/vgf_shared_device_test.cpp b/backends/arm/test/vgf_shared_device_test.cpp new file mode 100644 index 00000000000..e2d0f99b74b --- /dev/null +++ b/backends/arm/test/vgf_shared_device_test.cpp @@ -0,0 +1,123 @@ +/* + * Copyright 2026 Arm Limited and/or its affiliates. + * Licensed under the BSD-style license in the root LICENSE file. + */ +#include +#include +#include + +// Cppcheck may not expand GoogleTest headers. In that analysis-only case, +// supply plain C++ definitions so the test bodies remain parseable. A normal +// compiler sees GoogleTest's TEST macro and does not use these definitions. +#ifndef TEST +#define TEST(suite, name) \ + void cppcheck_##suite##_##name(); \ + [[maybe_unused]] static void (*const \ + cppcheck_test_ref_##suite##_##name[])() = \ + {&cppcheck_##suite##_##name}; \ + void cppcheck_##suite##_##name() +#define ASSERT_TRUE(expr) ((void)(expr)) +#define EXPECT_TRUE(expr) ((void)(expr)) +#define EXPECT_FALSE(expr) ((void)(expr)) +#define ASSERT_EQ(lhs, rhs) ((void)((lhs) == (rhs))) +#define EXPECT_EQ(lhs, rhs) ((void)((lhs) == (rhs))) +#endif + +namespace { +using namespace executorch::backends::vgf; +using namespace executorch::backends::vulkan_shared; +using executorch::runtime::BackendInitContext; +using executorch::runtime::BackendOption; +using executorch::runtime::BackendOptions; +using executorch::runtime::Error; +using executorch::runtime::Span; + +template +BackendInitContext context(BackendOptions& options) { + const auto view = options.view(); + return BackendInitContext( + nullptr, + nullptr, + nullptr, + nullptr, + Span(view.data(), view.size())); +} + +TEST(VgfSharedDevice, NoOptionsPreserveLegacy) { + BackendInitContext init(nullptr); + auto config = parse_vgf_shared_device_config(init); + ASSERT_TRUE(config.ok()); + EXPECT_FALSE(config->enabled()); + auto state = resolve_vgf_shared_device_state(init); + ASSERT_TRUE(state.ok()); + EXPECT_EQ(state.get(), nullptr); +} + +TEST(VgfSharedDevice, InvalidOptionTypeIsRejected) { + BackendOptions<1> options; + ASSERT_EQ(options.set_option("vulkan_shared_device_name", 7), Error::Ok); + EXPECT_EQ( + parse_vgf_shared_device_config(context(options)).error(), + Error::InvalidArgument); +} + +TEST(VgfSharedDevice, RuntimeNameAndGroupOptIn) { + BackendOptions<2> options; + ASSERT_EQ( + options.set_option("vulkan_shared_device_name", "vgf-test"), Error::Ok); + ASSERT_EQ(options.set_option("vulkan_shared_group_id", 9), Error::Ok); + auto config = parse_vgf_shared_device_config(context(options)); + ASSERT_TRUE(config.ok()); + EXPECT_TRUE(config->lookup_or_create()); + EXPECT_EQ(config->device_name, "vgf-test"); + EXPECT_EQ(config->group_id, 9); +} + +TEST(VgfSharedDevice, ExplicitDisableDoesNotCreateDevice) { + BackendOptions<1> options; + ASSERT_EQ( + options.set_option("vulkan_shared_device_mode", "disabled"), Error::Ok); + auto state = resolve_vgf_shared_device_state(context(options)); + ASSERT_TRUE(state.ok()); + EXPECT_EQ(state.get(), nullptr); +} + +TEST(VgfSharedDevice, MissingEnabledMetadataIsNotSupportedFeatures) { + SharedVulkanDeviceFeatures features; + EXPECT_FALSE(vgf_shared_features_compatible(features)); + features.arm_tensors = true; + features.arm_shader_tensor_access = true; + features.arm_data_graph = true; + features.arm_data_graph_shader_module = true; + features.shader_int16 = true; + features.shader_int64 = true; + features.shader_int8 = true; + features.storage_buffer_8bit_access = true; + features.host_query_reset = true; + features.synchronization2 = true; + features.maintenance4 = true; + features.pipeline_creation_cache_control = true; + features.vulkan_memory_model = true; + features.shader_replicated_composites = true; + EXPECT_TRUE(vgf_shared_features_compatible(features)); + // VGF_STAGE4_OPTIONAL_REPLICATED_COMPOSITES_V1 + auto without_replicated_composites = features; + without_replicated_composites.shader_replicated_composites = false; + EXPECT_TRUE(vgf_shared_features_compatible(without_replicated_composites)); + auto without_graph_module = features; + without_graph_module.arm_data_graph_shader_module = false; + EXPECT_FALSE(vgf_shared_features_compatible(without_graph_module)); + auto without_sync = features; + without_sync.synchronization2 = false; + EXPECT_FALSE(vgf_shared_features_compatible(without_sync)); +} + +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE +TEST(VgfSharedDevice, LookupOnlyMissingDoesNotInitializeVulkan) { + SharedVulkanRuntimeConfig config; + config.device_name = "stage4-unit-test-no-such-device"; + config.device_mode = SharedDeviceMode::kLookupOnly; + EXPECT_EQ(resolve_vgf_shared_device(config).error(), Error::NotFound); +} +#endif +} // namespace diff --git a/backends/vulkan/CMakeLists.txt b/backends/vulkan/CMakeLists.txt index c2949ee8105..72cbcf8b40e 100644 --- a/backends/vulkan/CMakeLists.txt +++ b/backends/vulkan/CMakeLists.txt @@ -99,6 +99,11 @@ add_library(vulkan_compute_api ${vulkan_api_cpp} ${vulkan_runtime_utils_cpp}) add_library(executorch::vulkan_compute_api ALIAS vulkan_compute_api) target_include_directories(vulkan_compute_api PUBLIC ${COMMON_INCLUDES}) target_compile_features(vulkan_compute_api PUBLIC cxx_std_17) +if(EXECUTORCH_BUILD_VGF AND EXECUTORCH_BUILD_VULKAN) + target_compile_definitions( + vulkan_compute_api PRIVATE EXECUTORCH_VGF_SHARED_DEVICE=1 + ) +endif() target_compile_definitions( vulkan_compute_api PUBLIC USE_VULKAN_WRAPPER USE_VULKAN_VOLK ) diff --git a/backends/vulkan/runtime/vk_api/Runtime.cpp b/backends/vulkan/runtime/vk_api/Runtime.cpp index 081d7e9fddb..d6083374430 100644 --- a/backends/vulkan/runtime/vk_api/Runtime.cpp +++ b/backends/vulkan/runtime/vk_api/Runtime.cpp @@ -88,7 +88,11 @@ VkInstance create_instance(const RuntimeConfig& config) { 0, // applicationVersion nullptr, // pEngineName 0, // engineVersion - VK_API_VERSION_1_1, // apiVersion +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + VK_API_VERSION_1_3, // VGF uses core 1.3 features and commands. +#else + VK_API_VERSION_1_1, // Preserve Vulkan-only behavior. +#endif }; std::vector enabled_layers; diff --git a/backends/vulkan_shared/runtime/SharedVulkanDevice.h b/backends/vulkan_shared/runtime/SharedVulkanDevice.h index 67c5f958bc7..1e39f735a62 100644 --- a/backends/vulkan_shared/runtime/SharedVulkanDevice.h +++ b/backends/vulkan_shared/runtime/SharedVulkanDevice.h @@ -69,6 +69,20 @@ struct SharedVulkanDeviceFeatures final { bool cooperative_matrix2_tensor_addressing = false; bool subgroup_size_control = false; bool compute_full_subgroups = false; + + bool arm_tensors = false; + bool arm_shader_tensor_access = false; + bool arm_data_graph = false; + bool arm_data_graph_shader_module = false; + bool arm_data_graph_neural_statistics = false; + bool synchronization2 = false; + bool maintenance4 = false; + bool pipeline_creation_cache_control = false; + bool host_query_reset = false; + bool vulkan_memory_model = false; + bool vulkan_memory_model_device_scope = false; + bool shader_replicated_composites = false; + bool shader_bfloat16 = false; }; // The shared layer carries Vulkan handles but deliberately does not call Vulkan diff --git a/examples/arm/vgf_vulkan_shared/CMakeLists.txt b/examples/arm/vgf_vulkan_shared/CMakeLists.txt new file mode 100644 index 00000000000..3428cec22be --- /dev/null +++ b/examples/arm/vgf_vulkan_shared/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# Licensed under the BSD-style license in the root LICENSE file. + +if(NOT EXECUTORCH_BUILD_VULKAN OR NOT EXECUTORCH_BUILD_VGF) + message(FATAL_ERROR "The mixed example requires both Vulkan and VGF backends") +endif() +if(NOT EXECUTORCH_BUILD_EXTENSION_MODULE OR NOT + EXECUTORCH_BUILD_EXTENSION_TENSOR +) + message( + FATAL_ERROR + "Enable EXECUTORCH_BUILD_EXTENSION_MODULE and EXECUTORCH_BUILD_EXTENSION_TENSOR" + ) +endif() + +add_executable(vgf_vulkan_shared_runner runner.cpp) +target_compile_features(vgf_vulkan_shared_runner PRIVATE cxx_std_17) +target_compile_options( + vgf_vulkan_shared_runner + PRIVATE "$<$>:-fexceptions>" + "$<$:/EHsc>" +) +target_link_libraries( + vgf_vulkan_shared_runner + PRIVATE extension_module extension_tensor vgf_backend vulkan_backend + executorch_vulkan_shared_runtime +) +# The exporter rejects CPU KernelCall instructions; no fallback kernels needed. +set_target_properties( + vgf_vulkan_shared_runner PROPERTIES BUILD_RPATH "$ORIGIN" INSTALL_RPATH + "$ORIGIN" +) diff --git a/examples/arm/vgf_vulkan_shared/README.md b/examples/arm/vgf_vulkan_shared/README.md new file mode 100644 index 00000000000..6116fd47ed0 --- /dev/null +++ b/examples/arm/vgf_vulkan_shared/README.md @@ -0,0 +1,408 @@ +# Shared VGF and Vulkan Backend Example + +## Overview + +This example demonstrates how the **VGF and Vulkan backends in ExecuTorch can execute different parts of a single model while sharing the same Vulkan device**. + +The model is partitioned between the two GPU backends: + +- **VGF backend** executes the first operation. +- **Vulkan backend** executes the intermediate operation. +- **VGF backend** executes the final operations. + +Both backends use a shared `VkDevice`, avoiding the need to create separate Vulkan devices for each backend. + +The example demonstrates: + +- Executing a single ExecuTorch program with both VGF and Vulkan delegates. +- Creating and registering a Vulkan device with the features required by VGF. +- Reusing the registered device across both backends. +- Synchronizing access to a shared Vulkan queue. +- Verifying the numerical correctness of the mixed-backend execution. + +The example does not implement zero-copy transfers between delegates or combine their execution into a single GPU submission. Ordinary ExecuTorch delegate boundaries remain. + +## Example Model + +The model is intentionally small to make backend partitioning and shared-device behavior easy to verify. + +It performs the following operations: + +```text +Input tensor [1, 3, 8, 8] + | + v + +-------------+ + | avg_pool2d | VGF backend + | kernel = 2 | + | stride = 2 | + +-------------+ + | + v + +-------------+ + | sigmoid | Vulkan backend + +-------------+ + | + v + +-------------+ + | x * x | VGF backend + | + 0.25 | + +-------------+ + | + v +Output tensor [1, 3, 4, 4] +``` + +The exporter uses `VgfPartitioner` and `VulkanPartitioner` to assign operations to their respective backends. + +For demonstration purposes, `sigmoid` is explicitly excluded from VGF partitioning so that Vulkan executes it. This is an example-specific partitioning policy, not a limitation of VGF's operator support. + +The generated ExecuTorch execution plan must contain exactly three delegate calls: + +```text +VgfBackend -> VulkanBackend -> VgfBackend +``` + +The exporter also verifies that no CPU `KernelCall` instructions remain in the execution plan. + +## How Device Sharing Works + +The example runner creates and registers a Vulkan device before loading the ExecuTorch model. + +The device is created with the Vulkan extensions, features, and queue capabilities required by both backends. + +The runner uses the following shared-device configuration: + +```text +vulkan_shared_device_name = "vgf-vulkan-e2e" +vulkan_shared_group_id = 7 +vulkan_shared_device_mode = "lookup_only" +``` + +The device name and group ID identify the registry entry. Both backends use the same configuration to resolve the shared device. + +The `lookup_only` mode requires the device to exist in the registry. If no compatible device is registered, initialization fails rather than silently creating another device. + +The runner first verifies that the Vulkan adapter resolves the registered device. It then loads and executes the mixed-backend model. + +Conceptually, the execution looks like this: + +```text + Application + | + v + Shared Vulkan Registry + | + v + Shared VkDevice + | + Shared Vulkan Queue + | + +-------------+-------------+ + | | + v v + VGF Backend Vulkan Backend + | | + +-------------+-------------+ + | + v + ExecuTorch Program + | + v + VGF -> Vulkan -> VGF +``` + +### Queue Synchronization + +Because both backends can submit commands to the same Vulkan queue, submissions must be synchronized. + +VGF uses the mutex associated with the shared queue in the Vulkan registry. Queue ownership is established only while a synchronized operation is active. + +The current implementation performs synchronous operations, including the required waits. Sharing the queue does not imply asynchronous execution or automatic overlap between delegates. + +### Device Lifetime + +The shared-device registry maintains references to registered devices. + +Each VGF delegate owns its private resources, such as its command pool, but does not destroy a borrowed Vulkan device. + +The example unregisters the device when its execution resources are released. + +## Prerequisites + +The example requires: + +- An ExecuTorch checkout with VGF and Vulkan backend support. +- Python with the ExecuTorch export dependencies installed. +- Arm Model Converter and compatible VGF tools. +- Vulkan SDK and shader compiler tools. +- CMake and Ninja. +- A Vulkan implementation supporting the required Arm tensor and data-graph extensions, or a compatible ML emulation environment. + +For native compilation, `VGF_ROOT` must reference a VGF installation containing: + +```text +include/vgf/decoder.hpp +lib/libvgf.a +``` + +For Android cross-compilation, use a VGF library built for Android AArch64 rather than the macOS host library. + +A combined VGF/Vulkan build requires Vulkan 1.3 support and a queue family supporting both `VK_QUEUE_COMPUTE_BIT` and `VK_QUEUE_DATA_GRAPH_BIT_ARM`. + +## Export the Model + +From the ExecuTorch repository root, run: + +```bash +python -m examples.arm.vgf_vulkan_shared.export \ + --output-dir ./vgf_vulkan_shared_out +``` + +If required by your Model Converter installation, additional compiler options can be supplied through repeated `--compiler-flag=...` arguments. + +The exporter writes: + +```text +vgf_vulkan_shared_out/ +├── vgf_vulkan_vgf.pte +├── input0.bin +├── expected0.bin +├── edge_graph.txt +└── manifest.json +``` + +These files contain: + +| File | Description | +|---|---| +| `vgf_vulkan_vgf.pte` | ExecuTorch program containing VGF and Vulkan delegates | +| `input0.bin` | FP32 input tensor | +| `expected0.bin` | Reference output calculated using PyTorch | +| `edge_graph.txt` | Lowered graph for inspecting backend partitioning | +| `manifest.json` | Export metadata | + +Input and reference-output tensors use contiguous, little-endian FP32 storage. + +Before writing the `.pte` file, the exporter validates the execution plan and checks that the expected VGF → Vulkan → VGF sequence was generated. + +## Build and Run + +Choose **one** of the two build options below. Native and Android builds are alternatives for the same `vgf_vulkan_shared_runner` executable; neither is a prerequisite for the other. Both options use the model exported above. + +Initialize the Vulkan dependencies for either build option: + +```bash +git submodule update --init \ + backends/vulkan/third-party/Vulkan-Headers \ + backends/vulkan/third-party/volk \ + backends/vulkan/third-party/VulkanMemoryAllocator +``` + +### Option A: Native Build and Run + +Configure the build: + +```bash +export VGF_ROOT=/path/to/vgf/install +export BUILD=cmake-out-vgf-vulkan + +cmake -S . -B "$BUILD" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DEXECUTORCH_BUILD_VGF=ON \ + -DEXECUTORCH_BUILD_VULKAN=ON \ + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ + -DEXECUTORCH_BUILD_VGF_SHARED_EXAMPLE=ON \ + -DEXECUTORCH_ENABLE_LOGGING=ON \ + -DEXECUTORCH_VGF_ZERO_COPY_IO=OFF \ + -DEXECUTORCH_VGF_ROOT="$VGF_ROOT" +``` + +Build the example runner: + +```bash +cmake --build "$BUILD" \ + --target vgf_vulkan_shared_runner \ + --parallel 8 +``` + +Locate the executable: + +```bash +RUNNER="$(find "$BUILD" -type f \ + -name vgf_vulkan_shared_runner -print -quit)" +``` + +Run the model: + +```bash +"$RUNNER" ./vgf_vulkan_shared_out +``` + +On systems without native Arm tensor and data-graph support, configure the Vulkan ML emulation layers before launching the runner. + +### Option B: Android Build and Deployment + +#### Cross-compile for Android + +To execute the example on an Android device with Arm tensor and data-graph support, cross-compile the runner using the Android NDK. + +Set the Android NDK and target VGF installation: + +```bash +export ANDROID_NDK=/path/to/android-ndk +export VGF_ROOT=/path/to/android/vgf/install +export BUILD=cmake-out-android-vgf-vulkan +``` + +Configure CMake: + +```bash +cmake -S . -B "$BUILD" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-28 \ + -DANDROID_STL=c++_static \ + -DBUILD_SHARED_LIBS=OFF \ + -DEXECUTORCH_BUILD_VGF=ON \ + -DEXECUTORCH_BUILD_VULKAN=ON \ + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ + -DEXECUTORCH_BUILD_VGF_SHARED_EXAMPLE=ON \ + -DEXECUTORCH_ENABLE_LOGGING=ON \ + -DEXECUTORCH_VGF_ZERO_COPY_IO=OFF \ + -DEXECUTORCH_VGF_ROOT="$VGF_ROOT" +``` + +Build the runner: + +```bash +cmake --build "$BUILD" \ + --target vgf_vulkan_shared_runner \ + --parallel 8 +``` + +Find the executable and shared runtime library: + +```bash +RUNNER="$(find "$BUILD" -type f \ + -name vgf_vulkan_shared_runner -print -quit)" + +REGISTRY="$(find "$BUILD" -type f \ + -name libexecutorch_vulkan_shared_runtime.so -print -quit)" + +test -n "$RUNNER" && test -n "$REGISTRY" +``` + +The runner must be compiled for Android AArch64. All participating backends must use the same shared Vulkan registry library. + +#### Deploy and Run on Android + +Connect the device using ADB: + +```bash +adb devices +``` + +Create a directory on the device: + +```bash +adb shell mkdir -p /data/local/tmp/vgf-vulkan-shared +``` + +Deploy the runner and the registry library: + +```bash +adb push "$RUNNER" "$REGISTRY" \ + /data/local/tmp/vgf-vulkan-shared/ +``` + +Deploy the exported model and test tensors: + +```bash +adb push ./vgf_vulkan_shared_out/. \ + /data/local/tmp/vgf-vulkan-shared/ +``` + +Run the example: + +```bash +adb shell ' + cd /data/local/tmp/vgf-vulkan-shared && + chmod 755 vgf_vulkan_shared_runner && + LD_LIBRARY_PATH=/data/local/tmp/vgf-vulkan-shared \ + ./vgf_vulkan_shared_runner . +' +``` + +If the runner has additional shared-library dependencies, deploy them to the same directory or make them available through the Android linker configuration. + +## Verify the Results + +The runner checks that the Vulkan adapter resolves the same `VkDevice` registered for the VGF backend. + +Expected output includes messages similar to: + +```text +Shared VkDevice verified: + VGF= + Vulkan= + +PASS iteration=0 max_abs_error=... +PASS iteration=1 max_abs_error=... +PASS iteration=2 max_abs_error=... +``` + +The actual device handles and numerical errors depend on the runtime environment. + +The runner executes the model three times and compares the output with the PyTorch reference using: + +```text +atol = 1e-5 +rtol = 1e-4 +``` + +It also saves the first output to: + +```text +actual0.bin +``` + +To retrieve this output from Android: + +```bash +adb pull \ + /data/local/tmp/vgf-vulkan-shared/actual0.bin \ + ./vgf_vulkan_shared_out/actual0.bin +``` + +Successful execution demonstrates that: + +1. Both backends can operate with the registered Vulkan device. +2. The mixed VGF → Vulkan → VGF execution plan runs without CPU operator fallback. +3. The output matches the PyTorch reference within the specified tolerance. + +## Troubleshooting + +### Model export fails + +Verify that the Python environment contains the required ExecuTorch dependencies and compatible Arm Model Converter tools. + +If the exporter reports unexpected backend partitioning, inspect `edge_graph.txt` and confirm that the expected delegate sequence is present. + +### Shared device cannot be resolved + +Verify that the device is registered before delegate initialization and that both backends use matching registry keys. + +Also check that the registered device has the required Vulkan extensions, enabled feature bits, and queue capabilities. + +### Vulkan extensions are unavailable + +The native GPU must support the Arm tensor and data-graph extensions required by VGF. + +On a development machine without native \ No newline at end of file diff --git a/examples/arm/vgf_vulkan_shared/export.py b/examples/arm/vgf_vulkan_shared/export.py new file mode 100644 index 00000000000..d8613469980 --- /dev/null +++ b/examples/arm/vgf_vulkan_shared/export.py @@ -0,0 +1,133 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +"""Export and assert a deterministic VGF -> Vulkan -> VGF execution plan.""" + +import argparse +import json +from pathlib import Path +from typing import Mapping + +import torch +import torch.nn.functional as F +from executorch.backends.arm.vgf import VgfCompileSpec +from executorch.backends.arm.vgf.partitioner import VgfPartitioner +from executorch.backends.vulkan.partitioner.vulkan_partitioner import VulkanPartitioner +from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.schema import DelegateCall, KernelCall +from torch.fx.passes.operator_support import OperatorSupportBase + +EXPECTED_SEQUENCE = ["VgfBackend", "VulkanBackend", "VgfBackend"] + + +class MixedModel(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = F.avg_pool2d(x, kernel_size=2, stride=2) # VGF prefix + x = torch.sigmoid(x) # Deliberately assigned to Vulkan, not a support claim. + return x * x + 0.25 # VGF suffix + + +class KeepSigmoidForVulkan(OperatorSupportBase): + """Placement policy for this test only; do not change backend support tables.""" + + def is_node_supported( + self, submodules: Mapping[str, torch.nn.Module], node: torch.fx.Node + ) -> bool: + del submodules + return node.target not in { + torch.ops.aten.sigmoid.default, + exir_ops.edge.aten.sigmoid.default, + } + + +def assert_execution_plan(program) -> list[str]: + plans = [ + p for p in program.executorch_program.execution_plan if p.name == "forward" + ] + if len(plans) != 1: + raise RuntimeError(f"Expected one forward execution plan, got {len(plans)}") + plan = plans[0] + if len(plan.chains) != 1: + raise RuntimeError("This linear example must emit exactly one execution chain") + sequence: list[str] = [] + fallback: list[str] = [] + for instruction in plan.chains[0].instructions: + args = instruction.instr_args + if isinstance(args, DelegateCall): + sequence.append(plan.delegates[args.delegate_index].id) + elif isinstance(args, KernelCall): + operator = plan.operators[args.op_index] + fallback.append(f"{operator.name}.{operator.overload}") + if sequence != EXPECTED_SEQUENCE or fallback: + raise RuntimeError( + "Mixed placement changed; refusing to publish a misleading .pte. " + f"Expected {EXPECTED_SEQUENCE}, got {sequence}; CPU kernels={fallback}" + ) + return sequence + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", type=Path, default=Path("vgf_vulkan_shared_out") + ) + parser.add_argument("--compiler-flag", action="append", default=[]) + args = parser.parse_args() + torch.manual_seed(0) + model = MixedModel().eval() + x = torch.linspace(-2.0, 2.0, 1 * 3 * 8 * 8, dtype=torch.float32).reshape( + 1, 3, 8, 8 + ) + with torch.no_grad(): + expected = model(x).contiguous() + exported = torch.export.export(model, (x,), strict=True) + spec = VgfCompileSpec(tosa_spec="TOSA-1.0+FP", compiler_flags=args.compiler_flag) + spec.validate_environment() + edge = to_edge_transform_and_lower( + exported, + partitioner=[ + VgfPartitioner(spec, additional_checks=[KeepSigmoidForVulkan()]), + VulkanPartitioner(), + ], + compile_config=EdgeCompileConfig( + _check_ir_validity=False, _skip_dim_order=True + ), + ) + program = edge.to_executorch() + sequence = assert_execution_plan(program) + args.output_dir.mkdir(parents=True, exist_ok=True) + pte = args.output_dir / "vgf_vulkan_vgf.pte" + with pte.open("wb") as stream: + program.write_to_file(stream) + x.numpy().astype(" ".join(sequence)) + print(f"Wrote {pte}") + + +if __name__ == "__main__": + main() diff --git a/examples/arm/vgf_vulkan_shared/runner.cpp b/examples/arm/vgf_vulkan_shared/runner.cpp new file mode 100644 index 00000000000..a5e1bc1d5f3 --- /dev/null +++ b/examples/arm/vgf_vulkan_shared/runner.cpp @@ -0,0 +1,180 @@ +/* + * Copyright 2026 Arm Limited and/or its affiliates. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using executorch::runtime::Error; +using executorch::runtime::EValue; +using namespace executorch::backends; + +void check(Error error, const char* operation) { + if (error != Error::Ok) { + throw std::runtime_error( + std::string(operation) + " failed with error " + + std::to_string(static_cast(error))); + } +} + +std::vector read_floats(const std::string& path, size_t count) { + std::ifstream stream(path, std::ios::binary | std::ios::ate); + const auto bytes = static_cast(count * sizeof(float)); + if (!stream || stream.tellg() != bytes) { + throw std::runtime_error("Wrong file size or unreadable file: " + path); + } + stream.seekg(0); + std::vector result(count); + std::vector input_bytes(static_cast(bytes)); + if (!stream.read(input_bytes.data(), bytes)) { + throw std::runtime_error("Cannot read " + path); + } + std::memcpy(result.data(), input_bytes.data(), input_bytes.size()); + return result; +} + +// Declared before Module so unregister happens after Module and Adapter +// teardown, including on an exception. Do not clear the process-wide registry +// indiscriminately. +struct Registration final { + vulkan_shared::SharedVulkanDeviceKey key; + vulkan_shared::SharedVulkanDevicePtr device; + ~Registration() { + auto& registry = vulkan_shared::SharedVulkanDeviceRegistry::Get(); + if (registry.lookup(key) == device) { + (void)registry.unregister_device(key); + } + } +}; + +int run(const std::string& directory) { + vulkan_shared::SharedVulkanRuntimeConfig config; + config.device_name = "vgf-vulkan-e2e"; + config.group_id = 7; + config.device_mode = vulkan_shared::SharedDeviceMode::kCreateOnly; + auto prepared = vgf::resolve_vgf_shared_device(config); + if (!prepared.ok()) { + check(prepared.error(), "VGF-capable shared device creation"); + } + Registration registration{ + {config.device_name, config.group_id}, prepared.get()}; + + executorch::runtime::BackendOptions<3> options; + check( + options.set_option("vulkan_shared_device_name", "vgf-vulkan-e2e"), + "name"); + check(options.set_option("vulkan_shared_device_mode", "lookup_only"), "mode"); + check(options.set_option("vulkan_shared_group_id", 7), "group"); + executorch::runtime::LoadBackendOptionsMap backend_options; + check( + backend_options.set_options("VgfBackend", options.view()), "VGF options"); + check( + backend_options.set_options("VulkanBackend", options.view()), + "Vulkan options"); + + // Exercise actual Adapter resolver and keep that Adapter alive. + // The Vulkan delegate subsequently reuses this same cached Adapter. + const auto view = options.view(); + executorch::runtime::BackendInitContext context( + nullptr, + nullptr, + nullptr, + nullptr, + executorch::runtime::Span( + view.data(), view.size())); + auto adapter = vulkan::resolve_vulkan_shared_adapter(context); + if (!adapter.ok()) { + check(adapter.error(), "Vulkan shared Adapter resolution"); + } + if (!adapter.get() || adapter.get()->shared_device() != registration.device || + adapter.get()->device_handle() != registration.device->device()) { + throw std::runtime_error( + "Vulkan Adapter did not reuse the VGF-created device"); + } + std::printf( + "Shared VkDevice verified: VGF=%p Vulkan=%p\n", + static_cast(registration.device->device()), + static_cast(adapter.get()->device_handle())); + + auto input = read_floats(directory + "/input0.bin", 192); + const auto expected = read_floats(directory + "/expected0.bin", 48); + auto tensor = executorch::extension::make_tensor_ptr( + {1, 3, 8, 8}, input.data(), executorch::aten::ScalarType::Float); + executorch::extension::Module module(directory + "/vgf_vulkan_vgf.pte"); + check(module.load(backend_options), "Module::load"); + check(module.load_method("forward"), "Module::load_method"); + + // Re-execution checks pool/fence and representation reuse, not performance. + for (int iteration = 0; iteration != 3; ++iteration) { + auto outputs = module.forward(std::vector{EValue(*tensor)}); + if (!outputs.ok()) { + check(outputs.error(), "Module::forward"); + } + if (outputs->size() != 1 || !outputs->at(0).isTensor()) { + throw std::runtime_error("Expected one output tensor"); + } + const auto& output = outputs->at(0).toTensor(); + if (output.scalar_type() != executorch::aten::ScalarType::Float || + output.dim() != 4 || output.size(0) != 1 || output.size(1) != 3 || + output.size(2) != 4 || output.size(3) != 4 || + static_cast(output.numel()) != expected.size()) { + throw std::runtime_error("Output shape or dtype mismatch"); + } + const float* values = output.const_data_ptr(); + float max_abs_error = 0; + for (size_t i = 0; i != expected.size(); ++i) { + const float error = std::abs(values[i] - expected[i]); + max_abs_error = std::max(max_abs_error, error); + if (!std::isfinite(values[i]) || !std::isfinite(expected[i]) || + error > 1e-5f + 1e-4f * std::abs(expected[i])) { + throw std::runtime_error( + "Numerical mismatch at output element " + std::to_string(i)); + } + } + std::ofstream saved(directory + "/actual0.bin", std::ios::binary); + std::vector output_bytes(output.nbytes()); + std::memcpy(output_bytes.data(), values, output_bytes.size()); + if (!saved.write( + output_bytes.data(), + static_cast(output_bytes.size()))) { + throw std::runtime_error("Cannot write actual0.bin"); + } + std::printf( + "PASS iteration=%d max_abs_error=%.9g\n", iteration, max_abs_error); + } + return 0; +} +} // namespace + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "Usage: %s EXPORTED_DIRECTORY\n", argv[0]); + return 2; + } + executorch::runtime::runtime_init(); + try { + return run(argv[1]); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } +} diff --git a/setup.py b/setup.py index d5f2c593d00..0e7a31e201d 100644 --- a/setup.py +++ b/setup.py @@ -2912,8 +2912,7 @@ def run(self): # noqa C901 if cmake_cache.is_enabled("EXECUTORCH_BUILD_VULKAN"): cmake_build_args += ["--target", "vulkan_backend"] - # VGF can configure the registry target without linking it. - # Only Vulkan requires this DSO in the Python wheel today. + if cmake_cache.is_enabled("EXECUTORCH_VULKAN_SHARED_RUNTIME_REQUIRED"): cmake_build_args += ["--target", "executorch_vulkan_shared_runtime"] if cmake_cache.is_enabled("EXECUTORCH_BUILD_CMSIS_NN_PYBINDS"): @@ -3254,14 +3253,12 @@ def iter_distribution_names(self): "EXECUTORCH_COREML_DELEGATE_LIBRARY_BUILT", ], ), - # The Vulkan delegate requires one shared registry DSO even - # when its own library is static. VGF currently configures this - # target but does not use it, so do not ship it for VGF-only wheels. + # Both delegates need one registry DSO. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/backends/vulkan_shared/%BUILD_TYPE%/", src_name=get_dynamic_lib_name("executorch_vulkan_shared_runtime"), dst="executorch/lib/", - dependent_cmake_flags=["EXECUTORCH_BUILD_VULKAN"], + dependent_cmake_flags=["EXECUTORCH_VULKAN_SHARED_RUNTIME_REQUIRED"], ), BuiltFile( src_dir="%CMAKE_CACHE_DIR%/backends/apple/coreai/", From ea7fcc380880cc434e5c7a764457ee50d68b912e Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Mon, 9 Nov 2026 17:51:44 +0000 Subject: [PATCH 2/7] Fixed README.md --- examples/arm/vgf_vulkan_shared/README.md | 28 +++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/arm/vgf_vulkan_shared/README.md b/examples/arm/vgf_vulkan_shared/README.md index 6116fd47ed0..ad76770163f 100644 --- a/examples/arm/vgf_vulkan_shared/README.md +++ b/examples/arm/vgf_vulkan_shared/README.md @@ -405,4 +405,30 @@ Also check that the registered device has the required Vulkan extensions, enable The native GPU must support the Arm tensor and data-graph extensions required by VGF. -On a development machine without native \ No newline at end of file +On a development machine without native support, use a compatible Vulkan ML emulation environment. + +### Runner cannot load a shared library + +Check that `libexecutorch_vulkan_shared_runtime.so` is deployed and accessible through `LD_LIBRARY_PATH`. + +Ensure that all backends use the same registry library and are built from compatible runtime sources. + +### Numerical output differs + +Confirm that the deployed `.pte`, input tensor, and reference output were produced by the same export. + +Also verify that the selected GPU driver and Model Converter configuration support the operations used by the model. + +## Summary + +This example demonstrates mixed VGF and Vulkan execution using a single shared Vulkan device. + +It provides a small, reproducible model with a known execution sequence: + +```text +VGF -> Vulkan -> VGF +``` + +The example verifies device sharing, queue synchronization, backend interoperability, and numerical correctness. + +It is intended as a starting point for applications that use VGF and Vulkan together in one ExecuTorch program while avoiding separate Vulkan device creation for each backend. From abe8ea6d0a3af189ffd0d828878c825af3e12237 Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Mon, 9 Nov 2026 19:15:49 +0000 Subject: [PATCH 3/7] Fix CI failure with shared libs Change-Id: I18edcfdc2bf445aa5a7ea991fe05e2e0afcc84f1 --- .ci/scripts/wheel/test_shared_libraries.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/wheel/test_shared_libraries.py b/.ci/scripts/wheel/test_shared_libraries.py index bf39abf294a..4c126b34fa3 100644 --- a/.ci/scripts/wheel/test_shared_libraries.py +++ b/.ci/scripts/wheel/test_shared_libraries.py @@ -3016,6 +3016,7 @@ def test_shipped_library_names_are_expected() -> None: "libexecutorch_backend_mlx", "libexecutorch_backend_openvino", "libexecutorch_backend_qnn", + "libexecutorch_vulkan_shared_runtime", "libexecutorch_threadpool", "libexecutorch_etdump", ) From 37328d4ef7ae51aec8235325d2bf6dc65a54a8bc Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Mon, 9 Nov 2026 19:41:34 +0000 Subject: [PATCH 4/7] Fix for CI with shared_device_test. Change-Id: I7793095a25e959507c5ae8a2d4acfe85db28cb30 --- backends/arm/test/test_arm_backend.sh | 1 + backends/vulkan/runtime/vk_api/Runtime.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index e2ff9d85b29..c14f89138d0 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -538,6 +538,7 @@ test_runtime_vgf() { vgf_zero_copy_test \ vgf_execution_stats_enabled_test \ vgf_execution_stats_disabled_test \ + vgf_shared_device_test \ --parallel # --no-tests=error is intentional: discovering zero VGF tests must fail CI. diff --git a/backends/vulkan/runtime/vk_api/Runtime.cpp b/backends/vulkan/runtime/vk_api/Runtime.cpp index d6083374430..56dd113f3e3 100644 --- a/backends/vulkan/runtime/vk_api/Runtime.cpp +++ b/backends/vulkan/runtime/vk_api/Runtime.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) Meta Platforms, Inc. and affiliates. + * Copyright 2026 Arm Limited and/or its affiliates. * All rights reserved. * * This source code is licensed under the BSD-style license found in the From 35b2a98b3b611bfe0064c5e52a8e866efbbd5866 Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Tue, 10 Nov 2026 13:07:09 +0000 Subject: [PATCH 5/7] Fix Runtime.cpp Change-Id: I89bf74c0ae52b98e1abdbbed03f9b0e2d8914b1e --- backends/vulkan/runtime/vk_api/Runtime.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/vulkan/runtime/vk_api/Runtime.cpp b/backends/vulkan/runtime/vk_api/Runtime.cpp index 56dd113f3e3..ce70e89cf17 100644 --- a/backends/vulkan/runtime/vk_api/Runtime.cpp +++ b/backends/vulkan/runtime/vk_api/Runtime.cpp @@ -1,7 +1,7 @@ /* * Copyright (c) Meta Platforms, Inc. and affiliates. - * Copyright 2026 Arm Limited and/or its affiliates. * All rights reserved. + * Copyright 2026 Arm Limited and/or its affiliates. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. From eb40e91fdc8378d8326697c22c6dcca25db4c7d3 Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Tue, 10 Nov 2026 22:52:01 +0000 Subject: [PATCH 6/7] Addressed review's comments. Change-Id: Icd2ef08c627f8e3437156613bb994b6a73b23d6a --- CMakeLists.txt | 6 +++ backends/arm/CMakeLists.txt | 10 +--- backends/arm/runtime/VGFBackend.cpp | 51 ++++++++++++++------ backends/arm/runtime/VGFSharedDevice.cpp | 32 ++++++------ backends/arm/runtime/VGFSharedDevice.h | 23 +++++++++ backends/arm/test/test_arm_backend.sh | 25 ++++++++++ backends/arm/test/vgf_shared_device_test.cpp | 51 ++++++++++++++++++++ backends/vulkan/CMakeLists.txt | 7 +-- backends/vulkan/runtime/vk_api/Runtime.cpp | 33 +++++++++---- backends/vulkan/runtime/vk_api/Runtime.h | 22 ++++++--- examples/arm/vgf_vulkan_shared/README.md | 29 +++++++++++ tools/cmake/preset/default.cmake | 4 ++ 12 files changed, 235 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0dfe85959d0..7e8049331a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1679,6 +1679,12 @@ if(EXECUTORCH_BUILD_VGF) list(APPEND _executorch_backends vgf_backend) endif() +# The example links the Vulkan backend and extension/module targets; these +# targets are all available here, unlike inside backends/arm. +if(EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/arm/vgf_vulkan_shared) +endif() + # Added after the backends so EXECUTORCH_BUILD_WASM_WEBGPU can link # webgpu_backend, which has to already be defined at that point. if(EXECUTORCH_BUILD_WASM) diff --git a/backends/arm/CMakeLists.txt b/backends/arm/CMakeLists.txt index 290bf101ca8..6b595f3fb7a 100644 --- a/backends/arm/CMakeLists.txt +++ b/backends/arm/CMakeLists.txt @@ -477,15 +477,7 @@ if(EXECUTORCH_BUILD_VGF) add_test(NAME vgf_shared_device_test COMMAND vgf_shared_device_test) endif() - option(EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE - "Build the VGF/Vulkan Stage 4 example" OFF - ) - if(EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE) - add_subdirectory( - ${EXECUTORCH_ROOT}/examples/arm/vgf_vulkan_shared - ${CMAKE_CURRENT_BINARY_DIR}/vgf_vulkan_shared - ) - endif() + # The mixed example is configured at the root after dependencies exist. # end config for VGF builds endif() diff --git a/backends/arm/runtime/VGFBackend.cpp b/backends/arm/runtime/VGFBackend.cpp index 60a34148c51..a7d86247ba0 100644 --- a/backends/arm/runtime/VGFBackend.cpp +++ b/backends/arm/runtime/VGFBackend.cpp @@ -576,24 +576,17 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface { ~VGFBackend() override = default; bool is_available() const override { -#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE - // No RuntimeSpec is available during a backend availability probe. In a - // combined build, defer device capability checks to init(), which resolves - // the actual key/device. Probing an unrelated default GPU here can reject - // a valid externally registered device or create a redundant VkDevice. - return true; -#else + // Preserve the real VGF capability probe in combined builds too. + // The Vulkan runtime is created without a redundant default VkDevice + // when VGF probes it first. auto* self = const_cast(this); const std::lock_guard lock(mutex_); ET_LOG(Debug, "Checking VGFBackend is available"); self->ensure_initialized(); const bool available = is_initialized_ && vkml_load_extensions(&vk_device) == VK_SUCCESS; - // An availability-only probe must not retain a device until exit. - // Existing delegates, if any, keep their shared device alive. self->release_basics_if_unused(); return available; -#endif } Result init( @@ -1355,7 +1348,17 @@ VkResult vkml_allocate_basics( return VK_ERROR_INITIALIZATION_FAILED; } try { - auto owner = vkcompute::vkapi::runtime_owner(); + // If VGF is the first runtime user, request Vulkan 1.3 without + // constructing the Vulkan delegate's default VkDevice. + auto owner = vkcompute::vkapi::runtime_owner( + /*init_default_device=*/false, VK_API_VERSION_1_3); + if (owner->instance_api_version() < VK_API_VERSION_1_3) { + ET_LOG( + Error, + "VGF requires Vulkan 1.3: initialize the VGF-capable " + "runtime before Vulkan creates its default runtime"); + return VK_ERROR_INCOMPATIBLE_DRIVER; + } *instance = owner->instance(); *instance_lifetime = std::move(owner); } catch (const std::exception& error) { @@ -1486,9 +1489,15 @@ VkResult vkml_allocate_basics( *physical_device, &qf_count, qps.data()); uint32_t qf = UINT32_MAX; for (uint32_t i = 0; i < qf_count; ++i) { - constexpr VkQueueFlags required = + constexpr VkQueueFlags capabilities = VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM; - if (qps[i].queueCount > 0 && (qps[i].queueFlags & required) == required) { + const VkQueueFlags present = qps[i].queueFlags & capabilities; + // Both capabilities are required when a single VkQueue is shared + // with the Vulkan delegate. Restore legacy VGF queue selection + // for independent, non-shared execution. + const bool suitable = + shared_info != nullptr ? present == capabilities : present != 0; + if (qps[i].queueCount > 0 && suitable) { qf = i; break; } @@ -1814,6 +1823,14 @@ VkResult vkml_allocate_basics( ext, VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME) == 0) { continue; } + // Legacy VGF tolerated these optional extensions; shared devices + // require them because the registered Vulkan contract does. + if (shared_info == nullptr && + (std::strcmp(ext, "VK_KHR_maintenance5") == 0 || + std::strcmp(ext, "VK_KHR_deferred_host_operations") == 0)) { + ET_LOG(Info, "Optional VGF extension unavailable: %s", ext); + continue; + } ET_LOG(Error, "Required VGF extension is missing: %s", ext); return VK_ERROR_EXTENSION_NOT_PRESENT; } @@ -1835,7 +1852,13 @@ VkResult vkml_allocate_basics( *device = VK_NULL_HANDLE; return result; } - // Keep instance-loaded loader dispatch; multiple VkDevices coexist. +#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE + // Combined build: multiple VkDevices share Volk's instance dispatch. + // Do not overwrite it with a per-device function-pointer table. +#else + // VGF-only builds preserve their historical device-level dispatch. + volkLoadDevice(*device); +#endif if (host_memory_import_capabilities != nullptr) { // Only a successful vkCreateDevice makes extension enablement diff --git a/backends/arm/runtime/VGFSharedDevice.cpp b/backends/arm/runtime/VGFSharedDevice.cpp index b8c3cca9257..c61b37c349a 100644 --- a/backends/arm/runtime/VGFSharedDevice.cpp +++ b/backends/arm/runtime/VGFSharedDevice.cpp @@ -15,6 +15,7 @@ #endif #include +#include #include #include #include @@ -73,16 +74,9 @@ Error validate_device(const SharedVulkanDevicePtr& device) { ET_LOG(Error, "VGF shared device must use the ExecuTorch Vulkan instance"); return Error::DelegateInvalidCompatibility; } - static constexpr const char* required_extensions[] = { - "VK_ARM_tensors", - "VK_ARM_data_graph", - "VK_KHR_maintenance5", - "VK_KHR_deferred_host_operations"}; - for (const char* extension : required_extensions) { - if (!device->has_device_extension(extension)) { - ET_LOG(Error, "VGF shared device did not enable %s", extension); - return Error::DelegateInvalidCompatibility; - } + if (const char* extension = vgf_missing_required_device_extension(*device)) { + ET_LOG(Error, "VGF shared device did not enable %s", extension); + return Error::DelegateInvalidCompatibility; } // An external device must not claim a feature that was not enabled at // vkCreateDevice. The extension is optional, but its metadata must agree. @@ -113,10 +107,8 @@ Error validate_device(const SharedVulkanDevicePtr& device) { vkGetPhysicalDeviceQueueFamilyProperties( device->physical_device(), &count, families.data()); const uint32_t family = device->queue_family_index(); - constexpr VkQueueFlags required = - VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM; if (family >= count || families[family].queueCount == 0 || - (families[family].queueFlags & required) != required) { + !vgf_shared_queue_compatible(families[family].queueFlags)) { ET_LOG(Error, "VGF shared queue must support BOTH compute and data graph"); return Error::DelegateInvalidCompatibility; } @@ -176,7 +168,8 @@ Result create_device(const SharedVulkanDeviceKey& key) { &info.queue_family_index, statistics.requested, &statistics_enabled, - vgf_zero_copy_io_build_enabled(), + vgf_zero_copy_io_build_enabled() && + vgf_zero_copy_io_runtime_enabled(std::getenv("ET_VGF_ZERO_COPY")), &imports, &owner->instance_lifetime, &info); @@ -280,6 +273,17 @@ Result> resolve_vgf_shared_device_state( if (!config.ok()) { return config.error(); } + // A mixed method cannot rely on delegate initialization order: + // Vulkan's automatic creator does not enable VGF features. + // Pre-register a VGF-capable device using create_only and load both + // delegates with lookup_only; keep no-options legacy VGF unchanged. + if (config->enabled() && !config->lookup_only()) { + ET_LOG( + Error, + "VGF shared mode requires pre-registration (create_only) and " + "delegate lookup_only; lookup_or_create is order-dependent"); + return Error::InvalidArgument; + } auto shared = resolve_vgf_shared_device(config.get()); if (!shared.ok()) { return shared.error(); diff --git a/backends/arm/runtime/VGFSharedDevice.h b/backends/arm/runtime/VGFSharedDevice.h index 30fe73664e0..d4bde16fa9d 100644 --- a/backends/arm/runtime/VGFSharedDevice.h +++ b/backends/arm/runtime/VGFSharedDevice.h @@ -13,6 +13,7 @@ #include #include +#include #include namespace executorch { @@ -45,6 +46,28 @@ inline bool vgf_shared_features_compatible( // optional for models that do not use them. } +// Pure policy helpers, callable by tests without a Vulkan driver. +inline const char* vgf_missing_required_device_extension( + const vulkan_shared::SharedVulkanDevice& device) { + static constexpr std::array required = { + "VK_ARM_tensors", + "VK_ARM_data_graph", + "VK_KHR_maintenance5", + "VK_KHR_deferred_host_operations"}; + for (const char* extension : required) { + if (!device.has_device_extension(extension)) { + return extension; + } + } + return nullptr; +} + +inline bool vgf_shared_queue_compatible(VkQueueFlags flags) { + constexpr VkQueueFlags required = + VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM; + return (flags & required) == required; +} + runtime::Result parse_vgf_shared_device_config(const runtime::BackendInitContext& context); diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index c14f89138d0..5a050ead479 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -548,6 +548,31 @@ test_runtime_vgf() { --no-tests=error \ -R '^vgf_' + # Phase 4 regression: compile the shared-device branch in addition + # to the legacy VGF-only configuration above. Never silently skip it. + local shared_build_dir="${et_root_dir}/arm_test/vgf_vulkan_shared_tests" + rm -rf "${shared_build_dir}" + cmake -S "${et_root_dir}" -B "${shared_build_dir}" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DEXECUTORCH_BUILD_TESTS=ON \ + -DEXECUTORCH_BUILD_VGF=ON \ + -DEXECUTORCH_BUILD_VULKAN=ON \ + -DEXECUTORCH_BUILD_VGF_SHARED_EXAMPLE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \ + -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ + -DEXECUTORCH_VGF_ZERO_COPY_IO=ON \ + -DPYTHON_EXECUTABLE="${python_executable}" \ + -DPython3_EXECUTABLE="${python_executable}" + cmake --build "${shared_build_dir}" \ + --target vgf_shared_device_test shared_vulkan_runtime_test vgf_vulkan_shared_runner \ + --parallel + ctest --test-dir "${shared_build_dir}" \ + --output-on-failure --no-tests=error \ + -R '^(vgf_shared_device_test|shared_vulkan_runtime_test)$' + echo "${TEST_SUITE_NAME}: PASS" } diff --git a/backends/arm/test/vgf_shared_device_test.cpp b/backends/arm/test/vgf_shared_device_test.cpp index e2d0f99b74b..13ca6d04f56 100644 --- a/backends/arm/test/vgf_shared_device_test.cpp +++ b/backends/arm/test/vgf_shared_device_test.cpp @@ -6,6 +6,8 @@ #include #include +#include + // Cppcheck may not expand GoogleTest headers. In that analysis-only case, // supply plain C++ definitions so the test bodies remain parseable. A normal // compiler sees GoogleTest's TEST macro and does not use these definitions. @@ -112,7 +114,56 @@ TEST(VgfSharedDevice, MissingEnabledMetadataIsNotSupportedFeatures) { EXPECT_FALSE(vgf_shared_features_compatible(without_sync)); } +TEST(VgfSharedDevice, MissingExtensionsAreRejectedByMetadataPolicy) { + SharedVulkanDeviceCreateInfo info; + info.enabled_device_extensions = {"VK_ARM_tensors", "VK_ARM_data_graph"}; + SharedVulkanDevice partial(std::move(info)); + EXPECT_TRUE(vgf_missing_required_device_extension(partial) != nullptr); + SharedVulkanDeviceCreateInfo complete_info; + complete_info.enabled_device_extensions = { + "VK_ARM_tensors", + "VK_ARM_data_graph", + "VK_KHR_maintenance5", + "VK_KHR_deferred_host_operations"}; + SharedVulkanDevice complete(std::move(complete_info)); + EXPECT_TRUE(vgf_missing_required_device_extension(complete) == nullptr); +} + +TEST(VgfSharedDevice, SharedQueueRequiresComputeAndDataGraph) { + EXPECT_FALSE(vgf_shared_queue_compatible(VK_QUEUE_COMPUTE_BIT)); + EXPECT_FALSE(vgf_shared_queue_compatible(VK_QUEUE_DATA_GRAPH_BIT_ARM)); + EXPECT_TRUE(vgf_shared_queue_compatible( + VK_QUEUE_COMPUTE_BIT | VK_QUEUE_DATA_GRAPH_BIT_ARM)); +} + +TEST(VgfSharedDevice, StateReleasesItsDeviceReferenceOnTeardown) { + SharedVulkanDeviceCreateInfo info; + auto owner = std::make_shared(7); + std::weak_ptr owner_weak = owner; + info.lifetime_anchor = owner; + auto device = std::make_shared(std::move(info)); + owner.reset(); + { + auto state = std::make_shared(); + state->device = device; + EXPECT_FALSE(owner_weak.expired()); + } + EXPECT_FALSE(owner_weak.expired()); + device.reset(); + EXPECT_TRUE(owner_weak.expired()); +} + #if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE +TEST(VgfSharedDevice, ImplicitCreationIsRejectedAtDelegateInit) { + BackendOptions<1> options; + ASSERT_EQ( + options.set_option("vulkan_shared_device_name", "vgf-order-test"), + Error::Ok); + EXPECT_EQ( + resolve_vgf_shared_device_state(context(options)).error(), + Error::InvalidArgument); +} + TEST(VgfSharedDevice, LookupOnlyMissingDoesNotInitializeVulkan) { SharedVulkanRuntimeConfig config; config.device_name = "stage4-unit-test-no-such-device"; diff --git a/backends/vulkan/CMakeLists.txt b/backends/vulkan/CMakeLists.txt index 72cbcf8b40e..606b34d5e7e 100644 --- a/backends/vulkan/CMakeLists.txt +++ b/backends/vulkan/CMakeLists.txt @@ -99,11 +99,8 @@ add_library(vulkan_compute_api ${vulkan_api_cpp} ${vulkan_runtime_utils_cpp}) add_library(executorch::vulkan_compute_api ALIAS vulkan_compute_api) target_include_directories(vulkan_compute_api PUBLIC ${COMMON_INCLUDES}) target_compile_features(vulkan_compute_api PUBLIC cxx_std_17) -if(EXECUTORCH_BUILD_VGF AND EXECUTORCH_BUILD_VULKAN) - target_compile_definitions( - vulkan_compute_api PRIVATE EXECUTORCH_VGF_SHARED_DEVICE=1 - ) -endif() +# Instance API version is selected by RuntimeConfig at runtime, not by the +# presence of an unrelated compiled backend. target_compile_definitions( vulkan_compute_api PUBLIC USE_VULKAN_WRAPPER USE_VULKAN_VOLK ) diff --git a/backends/vulkan/runtime/vk_api/Runtime.cpp b/backends/vulkan/runtime/vk_api/Runtime.cpp index ce70e89cf17..1ee528e69e3 100644 --- a/backends/vulkan/runtime/vk_api/Runtime.cpp +++ b/backends/vulkan/runtime/vk_api/Runtime.cpp @@ -89,11 +89,7 @@ VkInstance create_instance(const RuntimeConfig& config) { 0, // applicationVersion nullptr, // pEngineName 0, // engineVersion -#if defined(EXECUTORCH_VGF_SHARED_DEVICE) && EXECUTORCH_VGF_SHARED_DEVICE - VK_API_VERSION_1_3, // VGF uses core 1.3 features and commands. -#else - VK_API_VERSION_1_1, // Preserve Vulkan-only behavior. -#endif + config.instance_api_version, // Selected by the first runtime user. }; std::vector enabled_layers; @@ -311,7 +307,9 @@ uint32_t select_compute_device( // std::unique_ptr init_global_vulkan_runtime( - const std::string& cache_data_path) { + const std::string& cache_data_path, + bool init_default_device, + uint32_t instance_api_version) { // Load Vulkan drivers #if defined(USE_VULKAN_VOLK) if (VK_SUCCESS != volkInitialize()) { @@ -329,7 +327,6 @@ std::unique_ptr init_global_vulkan_runtime( #else false; #endif /* VULKAN_DEBUG */ - const bool init_default_device = true; const uint32_t num_requested_queues = 1; // TODO: raise this value const RuntimeConfig default_config{ @@ -338,6 +335,7 @@ std::unique_ptr init_global_vulkan_runtime( AdapterSelector::Auto, num_requested_queues, cache_data_path, + instance_api_version, }; try { @@ -399,6 +397,17 @@ Runtime::~Runtime() { instance_ = VK_NULL_HANDLE; } +Adapter* Runtime::get_adapter_p() { + std::lock_guard lock(default_adapter_mutex_); + if (default_adapter_i_ == UINT32_MAX) { + default_adapter_i_ = create_adapter(select_compute_device); + } + VK_CHECK_COND( + default_adapter_i_ < adapters_.size(), + "Pytorch Vulkan Runtime: Default device adapter is not set correctly!"); + return adapters_[default_adapter_i_].get(); +} + uint32_t Runtime::create_adapter(const Selector& selector) { VK_CHECK_COND( !device_mappings_.empty(), @@ -445,13 +454,17 @@ std::string& set_and_get_pipeline_cache_data_path( return global_cache_data_path; } -std::shared_ptr runtime_owner() { +std::shared_ptr runtime_owner( + bool init_default_device, + uint32_t instance_api_version) { // The global vulkan runtime is declared as a static local variable within a // non-static function to ensure it has external linkage. If it were a global // static variable there would be one copy per translation unit that includes // Runtime.h as it would have internal linkage. - static const std::shared_ptr p_runtime = - init_global_vulkan_runtime(set_and_get_pipeline_cache_data_path("")); + static const std::shared_ptr p_runtime = init_global_vulkan_runtime( + set_and_get_pipeline_cache_data_path(""), + init_default_device, + instance_api_version); VK_CHECK_COND( p_runtime, diff --git a/backends/vulkan/runtime/vk_api/Runtime.h b/backends/vulkan/runtime/vk_api/Runtime.h index 1780f1c1a96..9d27a8a8a04 100644 --- a/backends/vulkan/runtime/vk_api/Runtime.h +++ b/backends/vulkan/runtime/vk_api/Runtime.h @@ -16,6 +16,7 @@ #include #include +#include namespace vkcompute { namespace vkapi { @@ -42,6 +43,9 @@ struct RuntimeConfig final { AdapterSelector default_selector; uint32_t num_requested_queues; std::string cache_data_path; + // Default Vulkan behavior stays at 1.1. VGF requests 1.3 before + // initializing the process-wide runtime. + uint32_t instance_api_version = VK_API_VERSION_1_1; }; class Runtime final { @@ -69,6 +73,7 @@ class Runtime final { std::vector device_mappings_; std::vector adapters_; uint32_t default_adapter_i_; + std::mutex default_adapter_mutex_; VkDebugReportCallbackEXT debug_report_callback_; @@ -77,13 +82,14 @@ class Runtime final { return instance_; } - inline Adapter* get_adapter_p() { - VK_CHECK_COND( - default_adapter_i_ >= 0 && default_adapter_i_ < adapters_.size(), - "Pytorch Vulkan Runtime: Default device adapter is not set correctly!"); - return adapters_[default_adapter_i_].get(); + inline uint32_t instance_api_version() const { + return config_.instance_api_version; } + // Created on first use when VGF initialized the instance without a + // default Vulkan VkDevice. + Adapter* get_adapter_p(); + inline Adapter* get_adapter_p(uint32_t i) { VK_CHECK_COND( i >= 0 && i < adapters_.size(), @@ -108,7 +114,11 @@ std::string& set_and_get_pipeline_cache_data_path(const std::string& file_path); // a static local variable. // Retain the instance owner while a backend-created shared device exists. // Destroy devices before releasing the last owner of their VkInstance. -std::shared_ptr runtime_owner(); +// First caller selects the process-wide instance configuration. A VGF +// caller must arrive first when Vulkan 1.3 is required. +std::shared_ptr runtime_owner( + bool init_default_device = true, + uint32_t instance_api_version = VK_API_VERSION_1_1); Runtime* runtime(); diff --git a/examples/arm/vgf_vulkan_shared/README.md b/examples/arm/vgf_vulkan_shared/README.md index ad76770163f..657d0aeb1a6 100644 --- a/examples/arm/vgf_vulkan_shared/README.md +++ b/examples/arm/vgf_vulkan_shared/README.md @@ -22,6 +22,35 @@ The example demonstrates: The example does not implement zero-copy transfers between delegates or combine their execution into a single GPU submission. Ordinary ExecuTorch delegate boundaries remain. +**Initialization contract:** Before loading a mixed method, the application must +call `resolve_vgf_shared_device()` with `device_mode=create_only` and the +intended name/group. Pass `device_mode=lookup_only` and the same name/group +as runtime options to **both** delegate backends. No such options means +standalone/legacy execution. `lookup_or_create` during VGF delegate +initialization is rejected: if Vulkan created a device first, it +would normally lack the VGF-specific extensions and enabled features. +These options are supplied at `Module::load`, not stored in the `.pte`. + +**Runtime setup:** The shared VGF device needs a Vulkan 1.3 instance. +Call the VGF pre-registration API before code that initializes the +process-wide Vulkan runtime; a previously created Vulkan 1.1 instance +cannot be upgraded. Vulkan-only initialization keeps requesting 1.1. + +**Synchronization:** The current VGF implementation holds the shared +queue mutex throughout delegate initialization and execution, including +submission and fence wait. This guarantees single-queue host access but +serializes Vulkan work from other threads for longer than necessary. +Fine-grained queue leases require an audited refactor of VGF command +recording, per-delegate fences, and imported-memory cleanup. Do not +interpret this demo as concurrent submissions or overlapped GPU work. + +**Memory:** This example does not share boundary allocations between +delegates: each transition still uses normal ExecuTorch tensors and +may copy data. `ET_VGF_ZERO_COPY=0` disables the optional VGF host-memory +import path before device creation; it does not enable inter-delegate +zero copy. Releasing a VGF delegate may wait for the shared queue to +be idle. + ## Example Model The model is intentionally small to make backend partitioning and shared-device behavior easy to verify. diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 6ffe5dd140a..e3a5c42e2f3 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -247,6 +247,10 @@ define_overridable_option( define_overridable_option( EXECUTORCH_BUILD_VGF "Build the Arm VGF backend" BOOL OFF ) +define_overridable_option( + EXECUTORCH_BUILD_VGF_SHARED_EXAMPLE "Build the mixed VGF/Vulkan example" BOOL + OFF +) define_overridable_option( EXECUTORCH_COREML_BUILD_EXECUTOR_RUNNER "Build CoreML executor runner." BOOL OFF From f3f1280aada766b755429ab25364b49ddfa370a3 Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Tue, 10 Nov 2026 22:58:11 +0000 Subject: [PATCH 7/7] Fixed comment Change-Id: Id06cc50c8e22a7ba3425a4eb8e75f5a2a624aa9f --- backends/arm/test/test_arm_backend.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index 5a050ead479..81c19bcae41 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -548,7 +548,7 @@ test_runtime_vgf() { --no-tests=error \ -R '^vgf_' - # Phase 4 regression: compile the shared-device branch in addition + # Compile the shared-device branch in addition # to the legacy VGF-only configuration above. Never silently skip it. local shared_build_dir="${et_root_dir}/arm_test/vgf_vulkan_shared_tests" rm -rf "${shared_build_dir}"