Test optimized ESP-NN kernels on the S3 emulator (#24244)

### Summary

Espressif CI forces ESP-NN's ANSI C kernels because the pinned emulator
predates S3 SIMD support. Upgrade esp-emulator to 0.51.0 and run the
existing BundleIO suite in separate optimized and ANSI jobs. Each job
verifies its SDK configuration and retains the bundled fixtures,
resolved component versions, SDK configuration, and logs.

Document ESP-NN's automatic target selection and keep tool-version pins
in `setup.sh`.

### Test plan

Ran both complete CI configurations locally with ESP-IDF v6.1 and ESP-NN
1.4.1:

```bash
ESPRESSIF_NN_KERNELS=optimized bash .ci/scripts/test_espressif.sh
ESPRESSIF_NN_KERNELS=ansi bash .ci/scripts/test_espressif.sh
```

Each passed all 16 delegate fixtures and three smoke inferences. The
bundles were byte-identical between runs, with existing tolerances
unchanged. The optimized SDK configuration and link map confirmed S3
assembly with `NN_SKIP_NUDGE` disabled.

ShellCheck, Bash syntax checks, scoped lintrunner, workflow parsing,
invalid-kernel-selection rejection, and `git diff --check` passed. The
pinned installer checksum was verified against the new emulator release.

Authored with AI assistance using Codex.

Author: rascani

.ci/scripts/test_espressif.sh

@@ -9,6 +9,12 @@ set -euo pipefail
 
 et_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 build_dir="${ESPRESSIF_BUILD_DIR:-${et_root}/cmake-out-esp-emulator}"
+espnn_kernels="${ESPRESSIF_NN_KERNELS:-optimized}"
+case "${espnn_kernels}" in
+    optimized) espnn_config=CONFIG_NN_OPTIMIZED ;;
+    ansi) espnn_config=CONFIG_NN_ANSI_C ;;
+    *) echo "ESPRESSIF_NN_KERNELS must be optimized or ansi" >&2; exit 1 ;;
+esac
 export ESPRESSIF_TOOLS_DIR="${ESPRESSIF_TOOLS_DIR:-${RUNNER_TEMP:-/tmp}/executorch-espressif}"
 mkdir -p "${build_dir}"
 build_dir=$(cd "${build_dir}" && pwd)
@@ -18,6 +24,11 @@ collect_artifacts() {
     if [[ -n "${RUNNER_ARTIFACT_DIR:-}" ]]; then
         mkdir -p "${RUNNER_ARTIFACT_DIR}/esp32s3"
         for file in "${build_dir}"/*.log "${build_dir}"/*.bpte \
+            "${build_dir}"/espnn-*-sdkconfig \
+            "${build_dir}/espnn-tests"/*.bpte \
+            "${build_dir}/espnn-tests/manifest.json" \
+            "${build_dir}/espnn-tests/operators.txt" \
+            "${et_root}/examples/espressif/project/dependencies.lock" \
             "${build_dir}/firmware/merged-binary.bin" \
             "${build_dir}/firmware/executorch_esp_runner.elf"; do
             if [[ -f "${file}" ]]; then
@@ -80,9 +91,10 @@ python examples/espressif/test_emulator.py "${build_dir}/emulator.log"
 
 # ESP-NN backend: run every exported program through the ExecuTorch runtime
 # and the ESP-NN delegate, and check it against the bundled host reference.
-# The emulator does not implement the ESP32-S3 SIMD instructions, so ESP-NN is
-# built with its ANSI C kernels.
-printf 'CONFIG_NN_ANSI_C=y\n' > "${build_dir}/espnn-emulator.defaults"
+espnn_firmware_dir="${build_dir}/espnn-${espnn_kernels}-firmware"
+espnn_sdkconfig="${build_dir}/espnn-${espnn_kernels}-sdkconfig"
+espnn_defaults="${build_dir}/espnn-${espnn_kernels}.defaults"
+printf '%s=y\n' "${espnn_config}" > "${espnn_defaults}"
 # Runs a command with its output in a log, and shows the log's tail on failure.
 logged() {
     local log="${build_dir}/$1"
@@ -94,26 +106,31 @@ logged() {
     fi
 }
 while read -r -u 3 name atol rtol; do
-    logged "espnn-${name}-configure.log" \
-        idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" \
+    espnn_log_prefix="espnn-${espnn_kernels}-${name}"
+    logged "${espnn_log_prefix}-configure.log" \
+        idf.py -C examples/espressif/project -B "${espnn_firmware_dir}" \
         -DIDF_TARGET=esp32s3 \
-        -DSDKCONFIG="${build_dir}/espnn-sdkconfig" \
-        "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/espnn-emulator.defaults" \
+        -DSDKCONFIG="${espnn_sdkconfig}" \
+        "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${espnn_defaults}" \
         -DET_ESPNN_BACKEND=ON \
         -DET_BUILD_DIR_PATH="${build_dir}/runtime" \
         -DET_PTE_FILE_PATH="${build_dir}/espnn-tests/${name}.bpte" \
         -DET_BUNDLE_IO=ON -DET_NUM_INFERENCES=1 \
         -DET_ATOL="${atol}" -DET_RTOL="${rtol}" reconfigure
-    logged "espnn-${name}-build.log" cmake --build "${build_dir}/espnn-firmware"
-    logged "espnn-${name}-merge.log" \
-        idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" merge-bin
-    logged "espnn-${name}-emulator.log" \
+    if ! grep -qx "${espnn_config}=y" "${espnn_sdkconfig}"; then
+        echo "Expected ${espnn_config}=y in ${espnn_sdkconfig}" >&2
+        exit 1
+    fi
+    logged "${espnn_log_prefix}-build.log" cmake --build "${espnn_firmware_dir}"
+    logged "${espnn_log_prefix}-merge.log" \
+        idf.py -C examples/espressif/project -B "${espnn_firmware_dir}" merge-bin
+    logged "${espnn_log_prefix}-emulator.log" \
         timeout 90s esp-emu --chip esp32s3 \
-        --firmware "${build_dir}/espnn-firmware/merged-binary.bin" \
-        --elf "${build_dir}/espnn-firmware/executorch_esp_runner.elf" \
+        --firmware "${espnn_firmware_dir}/merged-binary.bin" \
+        --elf "${espnn_firmware_dir}/executorch_esp_runner.elf" \
         --net user --psram-size 8M --timeout 60s --exit-on 'Program complete.'
-    echo -n "${name}: "
-    python examples/espressif/test_emulator.py "${build_dir}/espnn-${name}-emulator.log"
+    echo -n "${espnn_kernels}/${name}: "
+    python examples/espressif/test_emulator.py "${build_dir}/${espnn_log_prefix}-emulator.log"
 done 3< <(python -c '
 import json, sys
 for entry in json.load(open(sys.argv[1])):

.github/workflows/test-espressif.yml

@@ -27,7 +27,12 @@ jobs:
     uses: ./.github/workflows/_docker-image.yml
 
   test-esp32s3:
+    name: ESP32-S3 (${{ matrix.kernels }})
     needs: docker-image
+    strategy:
+      fail-fast: false
+      matrix:
+        kernels: [optimized, ansi]
     uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     permissions:
       id-token: write
@@ -38,11 +43,11 @@ jobs:
       submodules: recursive
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 45
-      upload-artifact: test-report-esp32s3
+      upload-artifact: test-report-esp32s3-${{ matrix.kernels }}
       script: |
         set -euo pipefail
         CONDA_ENV=$(conda env list --json | jq -r '.envs | .[-1]')
         conda activate "${CONDA_ENV}"
         source .ci/scripts/utils.sh
         install_executorch --use-pt-pinned-commit --minimal
-        bash .ci/scripts/test_espressif.sh
+        ESPRESSIF_NN_KERNELS="${{ matrix.kernels }}" bash .ci/scripts/test_espressif.sh

backends/espressif/README.md

@@ -102,7 +102,7 @@ Calibrate on a few hundred real samples; random data gives useless scales.
 
 ## Running on the ESP32-S3
 
-You need ESP-IDF (CI uses v6.1) and an activated ESP-IDF shell.
+You need ESP-IDF and an activated ESP-IDF shell.
 [`examples/espressif/setup.sh`](../../examples/espressif/setup.sh) installs a
 matching ESP-IDF and the emulator.
 
@@ -160,6 +160,10 @@ against the ExecuTorch targets that `executor_runner` imports, and registers
 `EspNNBackend` at startup. It defines no platform layer; the runner's
 `esp_pal.cpp` provides it.
 
+ESP-NN defaults to `CONFIG_NN_OPTIMIZED=y` and selects its implementation from
+ESP-IDF's `IDF_TARGET`. For `esp32s3`, this enables the S3 assembly kernels.
+`CONFIG_NN_ANSI_C=y` selects reference C kernels for verification and debugging.
+
 For a BundleIO program (`.bpte`), add `-DET_BUNDLE_IO=ON` and the tolerances
 `-DET_ATOL=... -DET_RTOL=...`. The runner then checks the outputs against the
 bundled reference and logs `Test_result: PASS` or `FAIL`.
@@ -197,8 +201,10 @@ BundleIO program with the host reference's outputs. CI builds the runner with
 `-DET_ESPNN_BACKEND=ON` for each program and runs it in `esp-emu`. Fully
 delegated programs must match to float precision, which means bit-exact int8
 results. Programs with CPU regions get a tolerance for the portable kernels'
-float rounding. The emulator does not implement the S3 SIMD instructions, so
-this build uses ESP-NN's ANSI C kernels (`CONFIG_NN_ANSI_C=y`).
+float rounding. Separate CI jobs run ESP-NN's optimized S3 kernels and its
+ANSI C reference kernels against the same bundled outputs. Both jobs target
+ESP32-S3. Logs include the kernel configuration in their filenames; artifacts
+retain the bundled fixtures, SDK configuration, and resolved component versions.
 
 The code follows the ExecuTorch coding style and passes the repository's
 `lintrunner` checks:

examples/espressif/README.md

@@ -202,12 +202,13 @@ bash examples/espressif/setup.sh
 source ~/.cache/executorch-espressif/setup_path.sh
 ```
 
-The installer uses ESP-IDF **v6.1** and esp-emulator **0.48.0**, verifying the
-emulator installer's pinned SHA-256 checksum before running it. ESP-IDF's Python
-dependencies are installed into your active environment, so the SDK and
-ExecuTorch share the same Python. The generated `setup_path.sh` adds the installed
-tools to `PATH` and selects the ESP32-S3 target. In a new shell, activate the same
-Python environment and source this file. Installation only needs to be done once.
+The ESP-IDF and esp-emulator versions are pinned in [setup.sh](setup.sh). The
+installer verifies the emulator installer's pinned SHA-256 checksum before
+running it. ESP-IDF's Python dependencies are installed into your active
+environment, so the SDK and ExecuTorch share the same Python. The generated
+`setup_path.sh` adds the installed tools to `PATH` and selects the ESP32-S3 target.
+In a new shell, activate the same Python environment and source this file.
+Installation only needs to be done once.
 
 The default install includes ESP-IDF's standard ESP32-S3 development tools,
 including GDB, OpenOCD, and the ULP toolchains. OpenOCD requires your platform's
@@ -225,7 +226,7 @@ the tools. Rerun setup without `--minimal` to add the full development toolset.
 
 Set `ESPRESSIF_TOOLS_DIR` to choose another installation directory. The installer
 prints the corresponding activation command. It also accepts `IDF_PATH` and
-`IDF_TOOLS_PATH` for an existing ESP-IDF v6.1 installation. The setup script checks
+`IDF_TOOLS_PATH` for an existing ESP-IDF installation. The setup script checks
 the SDK version before installing dependencies into the active environment.
 Use `--skip-idf` or `--skip-emulator` to install only the tools you need.
 
@@ -304,15 +305,17 @@ fails the test. This also detects incomplete runs when the emulator's optional
 without rebuilding or starting the emulator.
 
 The [CI wrapper](../../.ci/scripts/test_espressif.sh) composes these same steps,
-adds a host timeout, and collects logs and firmware. The
+adds a host timeout, and collects logs and firmware. It also runs the ESP-NN
+delegate's BundleIO fixtures with optimized S3 kernels and ANSI C reference
+kernels in separate jobs. The
 [GitHub Actions workflow](../../.github/workflows/test-espressif.yml) runs it on
 pull requests changing the ESP integration, pushes to `main` and `release/*`,
 weekly, and on manual dispatch. Branch and scheduled runs cover changes to
 shared runtime and export code.
 
-This covers CPU execution and the ESP-IDF integration on S3. It does not establish
-performance or peripheral behavior on real boards, and the emulator does not
-support the original ESP32.
+This covers portable and ESP-NN delegate execution and ESP-IDF integration on S3.
+It does not establish performance or peripheral behavior on real boards, and the
+emulator does not support the original ESP32.
 
 ## Memory Considerations
 

examples/espressif/setup.sh

@@ -34,7 +34,7 @@ done
 
 tools_dir="${ESPRESSIF_TOOLS_DIR:-${HOME}/.cache/executorch-espressif}"
 idf_version=v6.1
-emulator_version=0.48.0
+emulator_version=0.51.0
 emulator_installer_sha256=7d116867e9a54b8ab90b12fffec93eb5782b3b01f74191078be5278fa6fd5688
 mkdir -p "${tools_dir}"
 tools_dir=$(cd "${tools_dir}" && pwd)