Provide the ESP-NN delegate and esp_nn:: op library in the ESP-IDF component

The component now vendors the ESP-NN delegate runtime
(backends/espressif/espnn_runtime) and the esp_nn:: op library
(backends/espressif/ops), depends on espressif/esp-nn (or a project's own
components/esp-nn), and carries the delegate's Kconfig options in its menu. The
op library is a fourth kernel library for gen_kernels.py, after portable,
quantized and Cadence generic. The delegate is linked only when the kernel
bundle records a model that uses EspNNBackend, or when the project calls
executorch_add_backend(EspNNBackend): a small source in the delegate's library
defines an anchor that references register_espnn_backend, so -u on that anchor
pulls in espnn_backend.cpp and its static registration from a library that is
otherwise never linked. Only espnn_backend.h is public, and it now says when
the generated component links it. The kernel library, too, is linked only
when the bundle has kernel libraries or prim ops, so a delegate-only bundle
never builds it, and a project that declares only the backend also defines the
symbol et_model needs to link.

The smoke consumer now embeds any number of models, each with the test case
that split_bundled_program.py extracts from its bundled program, and compares
every output with the reference on the device. CI runs four images on the
emulator. An ESP32-S3 image runs the smoke model on portable kernels, a
quantized mul on esp_nn::quantized_mul_int8 and the conv-act-pool-linear ESP-NN
delegate program together. An ESP32-C6 image runs the first two and must not
contain EspNNBackend. Two more ESP32-S3 images run the delegate alone, one from
a bundle with no kernel libraries, which must not build the kernel library,
and one from executorch_add_backend with no bundle, whose registry must hold
no kernels. A project with its own components/esp-nn must build that one and
not the managed copy. The empty-app size check still passes with esp-nn
compiled in. export_smoke_model.py loses --pte-output, which nothing uses now.

Authored with Claude Code.

Author: rascani

.ci/scripts/test_espressif_idf_component.sh

@@ -38,9 +38,13 @@ struct DelegateStats {
 /**
  * Registers EspNNBackend with the ExecuTorch runtime.
  *
- * Registration also happens from a static initializer when the component is
- * linked whole-archive; calling this is only needed when the linker could
- * drop that initializer. Safe to call more than once.
+ * Registration also happens from a static initializer once the delegate is
+ * linked: when the espnn_runtime component is linked whole-archive or, in the
+ * generated ExecuTorch ESP-IDF component, when the kernel bundle records
+ * EspNNBackend or the project calls executorch_add_backend(EspNNBackend).
+ * Without either, that component does not link the delegate, and calls to
+ * these functions fail to link. Calling this is only needed when the linker
+ * could drop the initializer. Safe to call more than once.
  *
  * @retval true The backend is registered.
  */

backends/espressif/espnn_runtime/include/espnn/espnn_backend.h

@@ -21,7 +21,8 @@ component from sources in this repository:
 
 The output is a registry-shaped component (`idf_component.yml`,
 `CMakeLists.txt`, `Kconfig`, `VERSION`) containing the runtime, the
-portable, quantized and Cadence generic kernel libraries, the shared
+portable, quantized, Cadence generic and `esp_nn::` kernel libraries,
+the ESP-NN delegate, the shared
 ESP-IDF PAL from `platform/esp_pal.cpp` (ESP_LOG + `esp_timer` timing),
 and a C inference API (`et_model.h`, flash-XIP model bytes with
 internal/PSRAM arena placement). Only the kernels a project's models

examples/espressif/README.md

@@ -15,8 +15,9 @@
 #
 # Derived from examples/arduino/build_arduino_library.sh with an ESP-IDF
 # output layout and the shared ESP-IDF PAL. The component vendors every
-# kernel source of its kernel libraries (portable, quantized and the
-# Cadence backend's generic kernels) but registers none of them: a
+# kernel source of its kernel libraries (portable, quantized, the Cadence
+# backend's generic kernels and the esp_nn:: op library) and the ESP-NN
+# delegate, but registers none of them: a
 # project's kernel bundle, made by the component's tools/gen_kernels.py
 # from its models, selects the kernels at build time.
 #
@@ -204,15 +205,24 @@ copy_srcs() { # ET_ROOT-relative paths on stdin; $1 = list file to append to
 
 : > "$SCRATCH/core_srcs.txt"
 : > "$SCRATCH/kernel_srcs.txt"
+: > "$SCRATCH/espnn_srcs.txt"
 bzl_list EXECUTORCH_CORE_SRCS | copy_srcs "$SCRATCH/core_srcs.txt"
 # register_prim_ops.cpp is compiled by the build, with the bundle's prim
 # selection, only when a project has a kernel bundle.
 bzl_list EXECUTORCH_SRCS | grep -v '/register_prim_ops\.cpp$' \
   | copy_srcs "$SCRATCH/kernel_srcs.txt"
 echo "kernels/prim_ops/register_prim_ops.cpp" | copy_srcs "$SCRATCH/prim_src.txt"
 ( cd "$ET_ROOT" && find kernels/portable/cpu kernels/quantized/cpu \
-    backends/cadence/generic -name '*.cpp' -not -path '*/test*/*' | sort ) \
+    backends/cadence/generic backends/espressif/ops -name '*.cpp' \
+    -not -path '*/test*/*' | sort ) \
   | copy_srcs "$SCRATCH/kernel_srcs.txt"
+# The ESP-NN delegate runtime, an ESP-IDF component of its own in-tree.
+ESPNN_RUNTIME="$ET_ROOT/backends/espressif/espnn_runtime"
+mkdir -p "$OUT_DIR/src/espnn"
+for f in "$ESPNN_RUNTIME"/src/*.cpp; do
+  cp "$f" "$OUT_DIR/src/espnn/"
+  echo "src/espnn/$(basename "$f")" >> "$SCRATCH/espnn_srcs.txt"
+done
 
 echo "[2/5] Runtime, extension and kernel sources copied"
 
@@ -271,6 +281,10 @@ cp "$ET_ROOT/kernels/prim_ops/selective_build_prim_ops.h" \
 mkdir -p "$OUT_DIR/priv_include"
 cp -r "$ET_ROOT/third-party/flatbuffers/include/flatbuffers" \
       "$OUT_DIR/priv_include/flatbuffers"
+# Only espnn_backend.h is the delegate's API; the rest is internal to it.
+cp -r "$ESPNN_RUNTIME/include/espnn" "$OUT_DIR/priv_include/espnn"
+mkdir -p "$OUT_DIR/include/espnn"
+mv "$OUT_DIR/priv_include/espnn/espnn_backend.h" "$OUT_DIR/include/espnn/"
 
 echo "[3/5] Headers and third-party code copied"
 
@@ -283,6 +297,7 @@ LIBRARIES=(
   "portable kernels/portable/functions.yaml"
   "quantized kernels/quantized/quantized.yaml"
   "cadence backends/cadence/aot/functions.yaml"
+  "esp_nn backends/espressif/ops/operators.yaml"
 )
 mkdir -p "$OUT_DIR/tools/libraries" "$OUT_DIR/tools/codegen"
 cp "$TEMPLATE_DIR/tools/gen_kernels.py" "$OUT_DIR/tools/"
@@ -315,7 +330,7 @@ cp "$TEMPLATE_DIR/CMakeLists.txt" "$TEMPLATE_DIR/project_include.cmake" \
    "$TEMPLATE_DIR/README.md" "$OUT_DIR/"
 cp "$TEMPLATE_DIR/include/et_model.h" "$OUT_DIR/include/"
 mkdir -p "$OUT_DIR/src/platform"
-cp "$TEMPLATE_DIR/src/et_model.cpp" "$OUT_DIR/src/"
+cp "$TEMPLATE_DIR/src/et_model.cpp" "$TEMPLATE_DIR/src/espnn_anchor.cpp" "$OUT_DIR/src/"
 cp "$ESP_PAL_SRC" "$OUT_DIR/src/platform/"
 cp "$ET_ROOT/LICENSE" "$OUT_DIR/LICENSE"
 
@@ -349,9 +364,18 @@ NOTICE
   sort -u "$SCRATCH/kernel_srcs.txt" | sed 's/.*/    "&"/'
   echo ")"
   echo "set(ET_PRIM_OPS_SRC \"$(cat "$SCRATCH/prim_src.txt")\")"
+  echo "set(ET_ESPNN_SRCS"
+  sed 's/.*/    "&"/' "$SCRATCH/espnn_srcs.txt"
+  echo ")"
 } > "$OUT_DIR/et_generated.cmake"
 
-cat > "$OUT_DIR/Kconfig" <<'KCONFIG'
+sed -n '/^menu /,/^endmenu/p' "$ESPNN_RUNTIME/Kconfig" > "$SCRATCH/espnn_kconfig"
+if [[ ! -s "$SCRATCH/espnn_kconfig" ]]; then
+  echo "ERROR: $ESPNN_RUNTIME/Kconfig has no menu to include." >&2
+  exit 1
+fi
+{
+cat <<'KCONFIG'
 # GENERATED by examples/espressif/build_idf_component.sh -- do not edit.
 menu "ExecuTorch"
 
@@ -413,8 +437,12 @@ menu "ExecuTorch"
             Raise this when the application registers its own kernels.
             Must not be below the kernel bundle's kernel count.
 
-endmenu
 KCONFIG
+# The ESP-NN delegate's options, as a submenu.
+sed 's/^./    &/' "$SCRATCH/espnn_kconfig"
+echo ""
+echo "endmenu"
+} > "$OUT_DIR/Kconfig"
 
 sed "s/@ET_COMPONENT_VERSION@/$ET_COMPONENT_VERSION/" \
   "$TEMPLATE_DIR/idf_component.yml.in" > "$OUT_DIR/idf_component.yml"

examples/espressif/build_idf_component.sh

@@ -26,9 +26,6 @@ def forward(self, x, y):
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--output", type=Path, required=True)
-    parser.add_argument(
-        "--pte-output", type=Path, help="Also write the plain (unbundled) .pte"
-    )
     args = parser.parse_args()
 
     model = SmokeModel().eval()
@@ -46,10 +43,6 @@ def main() -> None:
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_bytes(serialize_from_bundled_program_to_flatbuffer(bundle))
     print(f"Exported {args.output}")
-    if args.pte_output:
-        args.pte_output.parent.mkdir(parents=True, exist_ok=True)
-        args.pte_output.write_bytes(program.buffer)
-        print(f"Exported {args.pte_output}")
 
 
 if __name__ == "__main__":

examples/espressif/export_smoke_model.py

@@ -6,11 +6,14 @@
 
 # Minimal ESP-IDF consumer of the generated ExecuTorch component, used by
 # .ci/scripts/test_espressif_idf_component.sh. Pass the generated component
-# directory as ET_COMPONENT_DIR, the plain smoke .pte from export_smoke_model.py
-# as ET_SMOKE_PTE, and optionally its kernel bundle from the component's
-# tools/gen_kernels.py as ET_SMOKE_KERNELS; without one the build makes it.
-# ET_SMOKE_UNDECLARED runs the model without declaring it. Without ET_SMOKE_PTE
-# it builds an app that does not use ExecuTorch, for size comparisons.
+# directory as ET_COMPONENT_DIR and the .pte files that split_bundled_program.py
+# made as the ET_SMOKE_MODELS list; each runs on its bundled test case. Pass a
+# kernel bundle from the component's tools/gen_kernels.py as ET_SMOKE_KERNELS,
+# or leave it out to have the build make one. With ET_SMOKE_BACKENDS, the models
+# are embedded but not declared and only those delegates are, as for models
+# delivered over the air; with ET_SMOKE_UNDECLARED they are embedded but not
+# declared. Without ET_SMOKE_MODELS it builds an app that does not use
+# ExecuTorch, for size comparisons.
 
 cmake_minimum_required(VERSION 3.16)
 

examples/espressif/idf_component_smoke/CMakeLists.txt

@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Export a quantized mul that runs on the esp_nn:: op library, as a bundled program."""
+
+import argparse
+from pathlib import Path
+
+import torch
+from executorch.backends.espressif.test.tester import EspTester
+from executorch.backends.test.harness.stages import StageType
+from executorch.devtools.bundled_program.config import MethodTestCase, MethodTestSuite
+from executorch.devtools.bundled_program.core import BundledProgram
+from executorch.devtools.bundled_program.serialize import (
+    serialize_from_bundled_program_to_flatbuffer,
+)
+from torch.utils._pytree import tree_flatten
+
+
+class Mul(torch.nn.Module):
+    def forward(self, x, y):
+        return x * y
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    inputs = (torch.linspace(-1.5, 2.5, 17), torch.linspace(3, -4, 17))
+    tester = EspTester(Mul(), inputs)
+    tester.quantize().export().to_edge().run_passes()
+    expected, _ = tree_flatten(
+        tester.get_artifact().exported_program().module()(*inputs)
+    )
+    program = tester.to_executorch().get_artifact(StageType.TO_EXECUTORCH)
+    operators = program.executorch_program.execution_plan[0].operators
+    if "esp_nn::quantized_mul_int8" not in {op.name for op in operators}:
+        raise SystemExit("the mul did not lower to esp_nn::quantized_mul_int8")
+    bundle = BundledProgram(
+        program, [MethodTestSuite("forward", [MethodTestCase(inputs, expected)])]
+    )
+    args.output.write_bytes(serialize_from_bundled_program_to_flatbuffer(bundle))
+
+
+if __name__ == "__main__":
+    main()

examples/espressif/idf_component_smoke/export_op_library_model.py

@@ -4,22 +4,55 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-if(ET_SMOKE_PTE)
-  set(srcs "main.c" "model.S")
+if(ET_SMOKE_MODELS)
+  set(srcs "main.c" "${CMAKE_CURRENT_BINARY_DIR}/models.S")
 else()
   set(srcs "empty.c")
 endif()
 idf_component_register(SRCS ${srcs} PRIV_REQUIRES executorch)
-if(NOT ET_SMOKE_PTE)
+if(NOT ET_SMOKE_MODELS)
   return()
 endif()
 
+# Embeds each model and the .io file split_bundled_program.py wrote next to it,
+# with the table main.c walks.
+set(table "")
+set(data "")
+set(files "")
+set(index 0)
+foreach(pte IN LISTS ET_SMOKE_MODELS)
+  get_filename_component(pte "${pte}" ABSOLUTE)
+  get_filename_component(name "${pte}" NAME_WLE)
+  get_filename_component(dir "${pte}" DIRECTORY)
+  set(io "${dir}/${name}.io")
+  string(
+    APPEND table
+    "    .word name_${index}, pte_${index}, pte_end_${index}, io_${index}\n"
+  )
+  string(
+    APPEND data "name_${index}:\n    .asciz \"${name}\"\n"
+    "    .balign 16\npte_${index}:\n    .incbin \"${pte}\"\npte_end_${index}:\n"
+    "    .balign 16\nio_${index}:\n    .incbin \"${io}\"\n"
+  )
+  list(APPEND files "${pte}" "${io}")
+  if(NOT ET_SMOKE_BACKENDS AND NOT ET_SMOKE_UNDECLARED)
+    executorch_add_model(PTE "${pte}")
+  endif()
+  math(EXPR index "${index} + 1")
+endforeach()
+file(
+  CONFIGURE
+  OUTPUT
+  "${CMAKE_CURRENT_BINARY_DIR}/models.S"
+  CONTENT
+  "    .section .rodata.smoke_models, \"a\"\n    .balign 4\n    .global smoke_models\nsmoke_models:\n${table}    .global smoke_model_count\nsmoke_model_count:\n    .word ${index}\n${data}\n    .section .note.GNU-stack, \"\", @progbits\n"
+)
 set_source_files_properties(
-  model.S PROPERTIES COMPILE_DEFINITIONS "ET_SMOKE_PTE=\"${ET_SMOKE_PTE}\""
+  "${CMAKE_CURRENT_BINARY_DIR}/models.S" PROPERTIES OBJECT_DEPENDS "${files}"
 )
-if(NOT ET_SMOKE_UNDECLARED)
-  executorch_add_model(PTE "${ET_SMOKE_PTE}")
-endif()
 if(ET_SMOKE_KERNELS)
   executorch_add_kernels(DIR "${ET_SMOKE_KERNELS}")
 endif()
+foreach(backend IN LISTS ET_SMOKE_BACKENDS)
+  executorch_add_backend(${backend})
+endforeach()