Skip to content

Commit 4c21d62

Browse files
rascanirascani
committed
Make the ESP-IDF kernel bundle at build time when Python has executorch
When a project declares models but no kernel bundle, the component's configure step now checks whether EXECUTORCH_PYTHON (set with -D or in the environment; by default the Python ESP-IDF builds with) can import executorch. If it can, it runs gen_kernels.py on the declared models into the build directory and uses that bundle, so a project that exports and builds in one environment no longer has to run gen_kernels.py or keep a bundle in its tree. The result is byte-identical to a bundle made by hand, which CI checks by building the ESP32-S3 smoke consumer this way. Without such a Python, or if generation fails, the configure error names the Python, shows why, and prints the gen_kernels.py command. Authored with Claude Code.
1 parent af61b8d commit 4c21d62

4 files changed

Lines changed: 87 additions & 19 deletions

File tree

‎.ci/scripts/test_espressif_idf_component.sh‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
# LICENSE file in the root directory of this source tree.
77

88
# Generates the ExecuTorch ESP-IDF component and a kernel bundle for the smoke
9-
# model, builds the idf_component_smoke consumer for ESP32-S3 and, with the
10-
# Cadence kernels too, for ESP32-C6, and runs the ESP32-S3 build on the
11-
# emulator. Also checks that stale or mismatched bundles fail at configure time,
12-
# and that an app declaring nothing links to the same size as one without the
13-
# component.
9+
# model, builds the idf_component_smoke consumer for ESP32-C6 with that bundle,
10+
# plus the Cadence kernels, and for ESP32-S3 with one the build generates
11+
# itself, and runs the ESP32-S3 build on the emulator. Also checks that stale or
12+
# mismatched bundles fail at configure time, and that an app declaring nothing
13+
# links to the same size as one without the component.
1414

1515
set -euo pipefail
1616

@@ -126,7 +126,12 @@ expect_configure_error commit-mismatch component_commit \
126126
-DET_SMOKE_PTE="${build_dir}/smoke.pte" \
127127
-DET_SMOKE_KERNELS="${build_dir}/mismatch_kernels"
128128

129-
expect_configure_error no-bundle "no kernel bundle is" -DET_SMOKE_PTE="${build_dir}/smoke.pte"
129+
# Without an ExecuTorch Python to generate one, a missing bundle is an error.
130+
python -m venv --without-pip --clear "${build_dir}/bare_python"
131+
EXECUTORCH_PYTHON="${build_dir}/bare_python/bin/python" expect_configure_error \
132+
no-bundle "cannot import executorch" -DET_SMOKE_PTE="${build_dir}/smoke.pte"
133+
expect_configure_error relative-python "must be the absolute path" \
134+
-DET_SMOKE_PTE="${build_dir}/smoke.pte" -DEXECUTORCH_PYTHON=python
130135

131136
# A model the bundle was not made from, declared instead of the one it was.
132137
mkdir -p "${build_dir}/other"
@@ -210,11 +215,14 @@ if ! tr -s ' \n' ' ' < "${build_dir}/firmware-nano.log" \
210215
exit 1
211216
fi
212217

213-
consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \
214-
"-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/large_registry.defaults" \
215-
-DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${kernels_dir}" build
216218
consumer smoke-esp32c6 esp32c6 -DET_COMPONENT_DIR="${component_dir}" \
217219
-DET_SMOKE_PTE="${build_dir}/smoke.pte" -DET_SMOKE_KERNELS="${build_dir}/wakeword_kernels" build
220+
# No bundle declared: the build makes one with its own Python, which has
221+
# executorch here, and it must match the one gen_kernels.py made above.
222+
consumer smoke-esp32s3 esp32s3 -DET_COMPONENT_DIR="${component_dir}" \
223+
"-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/large_registry.defaults" \
224+
-DET_SMOKE_PTE="${build_dir}/smoke.pte" build
225+
diff -r "${build_dir}/firmware-smoke-esp32s3/executorch_kernels" "${kernels_dir}"
218226

219227
firmware_dir="${build_dir}/firmware-smoke-esp32s3"
220228
idf.py -C "${project}" -B "${firmware_dir}" merge-bin \

‎examples/espressif/idf_component_smoke/CMakeLists.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77
# Minimal ESP-IDF consumer of the generated ExecuTorch component, used by
88
# .ci/scripts/test_espressif_idf_component.sh. Pass the generated component
99
# directory as ET_COMPONENT_DIR, the plain smoke .pte from export_smoke_model.py
10-
# as ET_SMOKE_PTE, and its kernel bundle from the component's
11-
# tools/gen_kernels.py as ET_SMOKE_KERNELS. ET_SMOKE_UNDECLARED runs the model
12-
# without declaring it. Without ET_SMOKE_PTE it builds an app that does not use
13-
# ExecuTorch, for size comparisons.
10+
# as ET_SMOKE_PTE, and optionally its kernel bundle from the component's
11+
# tools/gen_kernels.py as ET_SMOKE_KERNELS; without one the build makes it.
12+
# ET_SMOKE_UNDECLARED runs the model without declaring it. Without ET_SMOKE_PTE
13+
# it builds an app that does not use ExecuTorch, for size comparisons.
1414

1515
cmake_minimum_required(VERSION 3.16)
1616

‎examples/espressif/idf_component_template/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ executorch_add_model(PTE "${CMAKE_CURRENT_SOURCE_DIR}/model.pte")
4444
executorch_add_kernels(DIR "${CMAKE_CURRENT_SOURCE_DIR}/et_kernels")
4545
```
4646

47+
If the Python ESP-IDF builds with can import `executorch` (or you
48+
point `EXECUTORCH_PYTHON` at one that can, with `-D` or in the
49+
environment, for example
50+
`idf.py -DEXECUTORCH_PYTHON=$HOME/venvs/et/bin/python build`), you can
51+
leave out `executorch_add_kernels`: the build then makes the bundle in
52+
its build directory from the declared models, with the same result as
53+
running `gen_kernels.py` yourself.
54+
4755
The build fails at configure time if the bundle doesn't match the
4856
declared models (a model was re-exported, added or removed) or was
4957
made for another version of this component, and prints the

‎examples/espressif/idf_component_template/project_include.cmake‎

Lines changed: 58 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
# executorch_add_kernels(DIR <bundle-dir>) # from tools/gen_kernels.py
1313
# executorch_add_backend(<BackendId>) # force-link a delegate
1414
#
15-
# Calls are unioned across the project; a project has one kernel bundle.
15+
# Calls are unioned across the project; a project has one kernel bundle. With
16+
# models but no bundle, the build makes one if EXECUTORCH_PYTHON (-D or the
17+
# environment; default: the build's Python) can import executorch.
1618

1719
function(executorch_add_model)
1820
cmake_parse_arguments(_arg "" "" "PTE" ${ARGN})
@@ -83,20 +85,70 @@ function(__executorch_finalize)
8385
list(REMOVE_DUPLICATES _models)
8486
set(_command "python \"${_dir}/tools/gen_kernels.py\"")
8587
set(_pte_args "")
88+
set(_gen_args "")
8689
foreach(_pte IN LISTS _models)
8790
if(NOT EXISTS "${_pte}")
8891
message(FATAL_ERROR "ExecuTorch: declared model ${_pte} does not exist")
8992
endif()
9093
string(APPEND _pte_args " --pte \"${_pte}\"")
94+
list(APPEND _gen_args --pte "${_pte}")
9195
endforeach()
9296
if(NOT _bundle)
97+
if(NOT EXECUTORCH_PYTHON)
98+
set(EXECUTORCH_PYTHON "$ENV{EXECUTORCH_PYTHON}")
99+
endif()
100+
if(NOT EXECUTORCH_PYTHON)
101+
idf_build_get_property(EXECUTORCH_PYTHON PYTHON)
102+
elseif(NOT IS_ABSOLUTE "${EXECUTORCH_PYTHON}" OR NOT EXISTS
103+
"${EXECUTORCH_PYTHON}"
104+
)
105+
message(
106+
FATAL_ERROR
107+
"ExecuTorch: EXECUTORCH_PYTHON must be the absolute path of a "
108+
"python, not ${EXECUTORCH_PYTHON}"
109+
)
110+
endif()
111+
string(
112+
CONCAT
113+
_manual
114+
"Make the bundle with the Python environment the models were exported "
115+
"with:\n ${_command}${_pte_args} --out <bundle-dir>\nthen call "
116+
"executorch_add_kernels(DIR <bundle-dir>), or set EXECUTORCH_PYTHON to "
117+
"that environment's python."
118+
)
119+
execute_process(
120+
COMMAND "${EXECUTORCH_PYTHON}" -c
121+
"import executorch.codegen.tools.selective_build"
122+
RESULT_VARIABLE _no_executorch
123+
OUTPUT_QUIET
124+
ERROR_VARIABLE _probe_error
125+
)
126+
if(_no_executorch)
127+
message(
128+
FATAL_ERROR
129+
"ExecuTorch: models are declared but no kernel bundle is, and "
130+
"${EXECUTORCH_PYTHON} cannot import executorch to make one:\n"
131+
"${_probe_error}${_manual}"
132+
)
133+
endif()
134+
set(_bundle "${CMAKE_BINARY_DIR}/executorch_kernels")
93135
message(
94-
FATAL_ERROR
95-
"ExecuTorch: models are declared but no kernel bundle is. Make one "
96-
"with the Python environment the models were exported with:\n"
97-
" ${_command}${_pte_args} --out <bundle-dir>\n"
98-
"then call executorch_add_kernels(DIR <bundle-dir>)."
136+
STATUS "ExecuTorch: making the kernel bundle with ${EXECUTORCH_PYTHON}"
137+
)
138+
execute_process(
139+
COMMAND "${EXECUTORCH_PYTHON}" "${_dir}/tools/gen_kernels.py" ${_gen_args}
140+
--out "${_bundle}"
141+
RESULT_VARIABLE _failed
142+
OUTPUT_VARIABLE _output
143+
ERROR_VARIABLE _output
99144
)
145+
if(_failed)
146+
message(
147+
FATAL_ERROR
148+
"ExecuTorch: ${EXECUTORCH_PYTHON} failed to make the kernel bundle:\n"
149+
"${_output}${_manual}"
150+
)
151+
endif()
100152
endif()
101153

102154
set(_manifest "${_bundle}/kernels.json")

0 commit comments

Comments
 (0)