Make the ESP-IDF kernel bundle at build time when Python has executorch

When a project declares models but no kernel bundle, the component's configure
step now checks whether EXECUTORCH_PYTHON (set with -D or in the environment;
by default the Python ESP-IDF builds with) can import executorch. If it can, it
runs gen_kernels.py on the declared models into the build directory and uses
that bundle, so a project that exports and builds in one environment no longer
has to run gen_kernels.py or keep a bundle in its tree. The result is
byte-identical to a bundle made by hand, which CI checks by building the
ESP32-S3 smoke consumer this way. Without such a Python, or if generation
fails, the configure error names the Python, shows why, and prints the
gen_kernels.py command.

Authored with Claude Code.

Author: rascani

.ci/scripts/test_espressif_idf_component.sh

@@ -6,11 +6,11 @@
 # LICENSE file in the root directory of this source tree.
 
 # Generates the ExecuTorch ESP-IDF component and a kernel bundle for the smoke
-# model, builds the idf_component_smoke consumer for ESP32-S3 and, with the
-# Cadence kernels too, for ESP32-C6, and runs the ESP32-S3 build on the
-# emulator. Also checks that stale or mismatched bundles fail at configure time,
-# and that an app declaring nothing links to the same size as one without the
-# component.
+# model, builds the idf_component_smoke consumer for ESP32-C6 with that bundle,
+# plus the Cadence kernels, and for ESP32-S3 with one the build generates
+# itself, and runs the ESP32-S3 build on the emulator. Also checks that stale or
+# mismatched bundles fail at configure time, and that an app declaring nothing
+# links to the same size as one without the component.
 
 set -euo pipefail
 
@@ -126,7 +126,12 @@ expect_configure_error commit-mismatch component_commit \
     -DET_SMOKE_PTE="${build_dir}/smoke.pte" \
     -DET_SMOKE_KERNELS="${build_dir}/mismatch_kernels"
 
-expect_configure_error no-bundle gen_kernels.py -DET_SMOKE_PTE="${build_dir}/smoke.pte"
+# Without an ExecuTorch Python to generate one, a missing bundle is an error.
+python -m venv --without-pip --clear "${build_dir}/bare_python"
+EXECUTORCH_PYTHON="${build_dir}/bare_python/bin/python" expect_configure_error \
+    no-bundle "cannot import executorch" -DET_SMOKE_PTE="${build_dir}/smoke.pte"
+expect_configure_error relative-python "must be the absolute path" \
+    -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DEXECUTORCH_PYTHON=python
 
 # A kernel registry smaller than the bundle; the emulator run below uses one
 # larger than it.
@@ -172,11 +177,14 @@ if ! tr -s ' \n' '  ' < "${build_dir}/firmware-nano.log" \
     exit 1
 fi
 
-consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \
-    "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/large_registry.defaults" \
-    -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${kernels_dir}" build
 consumer smoke-esp32c6 esp32c6 -DET_COMPONENT_DIR="${component_dir}" \
     -DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${build_dir}/wakeword_kernels" build
+# No bundle declared: the build makes one with its own Python, which has
+# executorch here, and it must match the one gen_kernels.py made above.
+consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \
+    "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/large_registry.defaults" \
+    -DET_SMOKE_PTE="${build_dir}/smoke.pte" build
+diff -r "${build_dir}/firmware-smoke-esp32s3/executorch_kernels" "${kernels_dir}"
 
 firmware_dir="${build_dir}/firmware-smoke-esp32s3"
 idf.py -C "${project}" -B "${firmware_dir}" merge-bin \

examples/espressif/idf_component_smoke/CMakeLists.txt

@@ -7,9 +7,10 @@
 # Minimal ESP-IDF consumer of the generated ExecuTorch component, used by
 # .ci/scripts/test_espressif_idf_component.sh. Pass the generated component
 # directory as ET_COMPONENT_DIR, the plain smoke .pte from export_smoke_model.py
-# as ET_SMOKE_PTE, and its kernel bundle from the component's
-# tools/gen_kernels.py as ET_SMOKE_KERNELS. Without ET_SMOKE_PTE it builds an
-# app that does not use ExecuTorch, for size comparisons.
+# as ET_SMOKE_PTE, and optionally its kernel bundle from the component's
+# tools/gen_kernels.py as ET_SMOKE_KERNELS; without one the build makes it.
+# Without ET_SMOKE_PTE it builds an app that does not use ExecuTorch, for size
+# comparisons.
 
 cmake_minimum_required(VERSION 3.16)
 

examples/espressif/idf_component_template/README.md

@@ -44,6 +44,14 @@ executorch_add_model(PTE "${CMAKE_CURRENT_SOURCE_DIR}/model.pte")
 executorch_add_kernels(DIR "${CMAKE_CURRENT_SOURCE_DIR}/et_kernels")
 ```
 
+If the Python ESP-IDF builds with can import `executorch` (or you
+point `EXECUTORCH_PYTHON` at one that can, with `-D` or in the
+environment, for example
+`idf.py -DEXECUTORCH_PYTHON=$HOME/venvs/et/bin/python build`), you can
+leave out `executorch_add_kernels`: the build then makes the bundle in
+its build directory from the declared models, with the same result as
+running `gen_kernels.py` yourself.
+
 The build fails at configure time if the bundle doesn't match the
 declared models (a model was re-exported, added or removed) or was
 made for another version of this component, and prints the

examples/espressif/idf_component_template/project_include.cmake

@@ -12,7 +12,9 @@
 # executorch_add_kernels(DIR <bundle-dir>)        # from tools/gen_kernels.py
 # executorch_add_backend(<BackendId>)             # force-link a delegate
 #
-# Calls are unioned across the project; a project has one kernel bundle.
+# Calls are unioned across the project; a project has one kernel bundle. With
+# models but no bundle, the build makes one if EXECUTORCH_PYTHON (-D or the
+# environment; default: the build's Python) can import executorch.
 
 function(executorch_add_model)
   if(NOT ARGV0 STREQUAL "PTE" OR ARGC LESS 2)
@@ -80,20 +82,70 @@ function(__executorch_finalize)
 
   list(REMOVE_DUPLICATES _models)
   set(_command "python \"${_dir}/tools/gen_kernels.py\"")
+  set(_pte_args "")
   foreach(_pte IN LISTS _models)
     if(NOT EXISTS "${_pte}")
       message(FATAL_ERROR "ExecuTorch: declared model ${_pte} does not exist")
     endif()
     string(APPEND _command " --pte \"${_pte}\"")
+    list(APPEND _pte_args --pte "${_pte}")
   endforeach()
   if(NOT _bundle)
+    if(NOT EXECUTORCH_PYTHON)
+      set(EXECUTORCH_PYTHON "$ENV{EXECUTORCH_PYTHON}")
+    endif()
+    if(NOT EXECUTORCH_PYTHON)
+      idf_build_get_property(EXECUTORCH_PYTHON PYTHON)
+    elseif(NOT IS_ABSOLUTE "${EXECUTORCH_PYTHON}" OR NOT EXISTS
+                                                     "${EXECUTORCH_PYTHON}"
+    )
+      message(
+        FATAL_ERROR
+          "ExecuTorch: EXECUTORCH_PYTHON must be the absolute path of a "
+          "python, not ${EXECUTORCH_PYTHON}"
+      )
+    endif()
+    string(
+      CONCAT
+        _manual
+        "Make the bundle with the Python environment the models were exported "
+        "with:\n  ${_command} --out <bundle-dir>\nthen call "
+        "executorch_add_kernels(DIR <bundle-dir>), or set EXECUTORCH_PYTHON to "
+        "that environment's python."
+    )
+    execute_process(
+      COMMAND "${EXECUTORCH_PYTHON}" -c
+              "import executorch.codegen.tools.selective_build"
+      RESULT_VARIABLE _no_executorch
+      OUTPUT_QUIET
+      ERROR_VARIABLE _probe_error
+    )
+    if(_no_executorch)
+      message(
+        FATAL_ERROR
+          "ExecuTorch: models are declared but no kernel bundle is, and "
+          "${EXECUTORCH_PYTHON} cannot import executorch to make one:\n"
+          "${_probe_error}${_manual}"
+      )
+    endif()
+    set(_bundle "${CMAKE_BINARY_DIR}/executorch_kernels")
     message(
-      FATAL_ERROR
-        "ExecuTorch: models are declared but no kernel bundle is. Make one "
-        "with the Python environment the models were exported with:\n"
-        "  ${_command} --out <bundle-dir>\n"
-        "then call executorch_add_kernels(DIR <bundle-dir>)."
+      STATUS "ExecuTorch: making the kernel bundle with ${EXECUTORCH_PYTHON}"
+    )
+    execute_process(
+      COMMAND "${EXECUTORCH_PYTHON}" "${_dir}/tools/gen_kernels.py" ${_pte_args}
+              --out "${_bundle}"
+      RESULT_VARIABLE _failed
+      OUTPUT_VARIABLE _output
+      ERROR_VARIABLE _output
     )
+    if(_failed)
+      message(
+        FATAL_ERROR
+          "ExecuTorch: ${EXECUTORCH_PYTHON} failed to make the kernel bundle:\n"
+          "${_output}${_manual}"
+      )
+    endif()
   endif()
   string(APPEND _command " --out \"${_bundle}\"")