How can I automatically fallback unsupported operators to the CPU when running a model with the Vulkan backend in ExecuTorch?

[jaeokchoi]

Hello, how can I automatically fallback unsupported operators to the CPU when running a model with the Vulkan backend in ExecuTorch?

This is my CMakeLists.txt for my application to run model on ExecuTorch.

cmake_minimum_required(VERSION 3.17)
project(executorch_demo)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(Vulkan REQUIRED)

option(EXECUTORCH_ENABLE_LOGGING "" ON)
set(EXECUTORCH_LOG_LEVEL "Debug")
option(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER "" ON)
option(EXECUTORCH_BUILD_EXTENSION_MODULE "" ON)
option(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR "" ON)
option(EXECUTORCH_BUILD_EXTENSION_TENSOR "" ON)
option(EXECUTORCH_BUILD_KERNELS_OPTIMIZED "" ON)
option(EXECUTORCH_BUILD_KERNELS_PORTABLE "" ON)
option(EXECUTORCH_BUILD_VULKAN "" ON)
option(EXECUTORCH_BUILD_XNNPACK "" ON)

add_subdirectory(executorch)

add_executable(executorch_demo main.cpp)

target_link_libraries(executorch_demo
  PRIVATE
    executorch
    extension_module_static
    extension_tensor
    optimized_native_cpu_ops_lib
    vulkan_backend
    Vulkan::Vulkan
    portable_kernels
    xnnpack_backend
)

target_include_directories(executorch_demo
  PRIVATE
    ${CMAKE_CURRENT_SOURCE_DIR}/executorch/include
)

configure_file(${CMAKE_CURRENT_SOURCE_DIR}/model.pte ${CMAKE_CURRENT_BINARY_DIR}/model.pte COPYONLY)

And This is my main.cpp code.

#include <executorch/extension/module/module.h>
#include <executorch/extension/tensor/tensor.h>
#include <iostream>
#include <cmath>
#include <iomanip> // for std::setprecision

using namespace ::executorch::extension;

int main(int argc, char* argv[]) {
  std::cout << "ExecuTorch PointNet++ Inference Demo\n";

  // Load the exported PointNet++ model (PTE).
  Module module("model.pte");
  std::cout << "model.pte loaded successfully\n";

  // Create a dummy input: [1, 6, 1024, 1] (Batch, Channels, NumPoints, DummyDim)
  constexpr int B = 1;
  constexpr int C = 6;
  constexpr int N = 1024;
  constexpr int D = 1;
  float input[B * C * N * D];

  // Fill the input with dummy data
  for (int i = 0; i < B * C * N * D; ++i) {
    input[i] = std::sin(i * 0.001f);  // example waveform
  }

  // Create tensor from the raw input buffer
  auto input_tensor = from_blob(input, {B, C, N, D});

  // Print input shape
  std::cout << "Input tensor shape: [";
  for (auto s : input_tensor->sizes()) {
    std::cout << s << " ";
  }
  std::cout << "]\n";

  // Run inference
  std::cout << "Running inference...\n";
  const auto result = module.forward(input_tensor);

  if (!result.ok()) {
    std::cerr << "Inference failed! Error code: " << static_cast<int>(result.error()) << std::endl;
    return 1;
  }

  // Retrieve output tensor
  const auto output_tensor = result->at(0).toTensor();
  const float* output = output_tensor.const_data_ptr<float>();
  const auto& shape = output_tensor.sizes();

  std::cout << "Inference successful!\n";
  std::cout << "Output tensor shape: [";
  for (size_t i = 0; i < shape.size(); ++i) {
    std::cout << shape[i] << (i < shape.size() - 1 ? ", " : "");
  }
  std::cout << "]\n";

  // Print first few logits for inspection
  std::cout << "Class logits (first 10): ";
  int num_logits = shape[1];
  for (int i = 0; i < std::min(num_logits, 10); ++i) {
    std::cout << std::fixed << std::setprecision(4) << output[i] << " ";
  }
  std::cout << "...\n";

  return 0;
}

This is the error result.

D 00:00:00.000048 executorch:operator_registry.cpp:96] Successfully registered all kernels from shared library: NOT_SUPPORTED
D 00:00:00.000387 executorch:operator_registry.cpp:96] Successfully registered all kernels from shared library: NOT_SUPPORTED
Note (XNNPACK): l1_data_cache_bytes=32768, l1_data_cache_line_size=64, l1_data_cache_associativity=4, l1_data_cache_num_sets=128. (init_hardware_config, /mnt/work/workspace/pointnet2_executorch_demo/executorch/backends/xnnpack/third-party/XNNPACK/src/configs/hardware-config.c:413)
Note (XNNPACK): l2_data_cache_bytes=262144, l2_data_cache_line_size=64, l2_data_cache_associativity=8, l2_data_cache_num_sets=512. (init_hardware_config, /mnt/work/workspace/pointnet2_executorch_demo/executorch/backends/xnnpack/third-party/XNNPACK/src/configs/hardware-config.c:432)
Note (XNNPACK): cpuinfo_get_uarchs_count: 1. (init_hardware_config, /mnt/work/workspace/pointnet2_executorch_demo/executorch/backends/xnnpack/third-party/XNNPACK/src/configs/hardware-config.c:449)
Note (XNNPACK): cpu_get_uarch(0): 0x0 (init_hardware_config, /mnt/work/workspace/pointnet2_executorch_demo/executorch/backends/xnnpack/third-party/XNNPACK/src/configs/hardware-config.c:451)
D 00:00:00.004213 executorch:XNNPACKBackend.cpp:56] Creating XNN workspace
D 00:00:00.004272 executorch:XNNPACKBackend.cpp:68] Created XNN workspace: 0x55a146ccf0
ExecuTorch PointNet++ Inference Demo
model.pte loaded successfully
Input tensor shape: [1 6 1024 1 ]
Running inference...
D 00:00:00.015947 executorch:method.cpp:758] Loading method: forward.
terminate called after throwing an instance of 'vkcompute::vkapi::Error'
what(): Exception raised from get_shader_info at /mnt/work/workspace/pointnet2_executorch_demo/executorch/backends/vulkan/runtime/api/ShaderRegistry.cpp:51: (it != listings_.end()) is false! Could not find ShaderInfo with name concat_3_texture3d_int32
Aborted (core dumped)

And this is my code snippet for export .pte.
Please help me.

exported = torch.export.export(model, (sample_input,))

partitioners = [
    VulkanPartitioner(),       # 1순위: GPU
    XnnpackPartitioner(),     # 2순위: CPU fallback
]

et_program = to_edge_transform_and_lower(
    exported,
    partitioner=partitioners,
).to_executorch()

# 6. 저장
with open("model.pte", "wb") as f:
    f.write(et_program.buffer)

[GregoryComer]

@SS-JIA can you take a look at the error?

[SS-JIA]

Unfortunately, there is no way to fall back to the CPU because during export the model has already been converted to a delegate specific IR. However, this is definitely an issue with Vulkan's lowering logic in that it is lowering operators with unsupport input/output dtypes.

The easiest fix here is to add support for running concat with integer inputs; I will put up a PR to enable that shortly.

In the long term, I will also update Vulkan's lowering logic to avoid lowering ops that have unsupported dtypes as inputs.

Apologies for the inconvenience!

@jaeokchoi