From a18ed53af14dff7853e6f74c38c952fd56c96583 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 11:52:22 -0800 Subject: [PATCH 01/25] Read the ESP-NN delegate's per-operator timers only when profiling Int8Interpreter::invoke() read esp_timer_get_time() before and after every operator to fill a per-operator profile, whether or not anything printed it. On ESP32-S3 that is two function calls per operator; for a 32-operator streaming keyword-spotting model it was about 55 us of a 540 us call, more than many of the operators themselves. The interpreter now times operators only with Int8RuntimeOptions::profile_operations, which EspNNBackend sets from CONFIG_ESPNN_DELEGATE_PROFILE, and keeps the times in a table allocated only then. Otherwise it reads the timer at the start and end of each call and every 16 operators, for its cooperative yield. last_invoke_us() becomes the wall time of the call. --- backends/espressif/README.md | 2 +- backends/espressif/espnn_runtime/Kconfig | 4 ++ .../include/espnn/int8_interpreter.h | 7 ++- .../espnn_runtime/src/espnn_backend.cpp | 3 + .../espnn_runtime/src/int8_interpreter.cpp | 56 ++++++++++++------- 5 files changed, 49 insertions(+), 23 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 48fcf2cb062..73a140fb9c7 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -174,7 +174,7 @@ Other options, in menuconfig under *ExecuTorch ESP-NN backend*: |---|---|---| | `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | | `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | -| `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | +| `ESPNN_DELEGATE_PROFILE` | n | Times every operator and prints a per-operator time table after a delegate's first run. The timer reads cost about 1 µs per operator on the ESP32-S3, so leave it off in production | ### The original ESP32 diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index 8fa51b176b6..1ec69fa6be2 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -20,5 +20,9 @@ menu "ExecuTorch ESP-NN backend" config ESPNN_DELEGATE_PROFILE bool "Print a per-operator profile after each delegate's first run" default n + help + Times every operator, which adds a timer read per operator to + each execute. Without it, the delegate reads the timer only + every few operators, for its cooperative yield. endmenu diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 9c18ac8e63b..1f4e065f2ab 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -41,6 +41,9 @@ struct Int8RuntimeOptions { bool dual_core = false; /// Internal RAM left free when deciding whether the arena fits there. size_t internal_reserve_bytes = 48 * 1024; + /// Time every operation for op_time_us() and print_profile(). Otherwise + /// invoke() reads the timer only a few times per call. + bool profile_operations = false; }; /** @@ -161,7 +164,8 @@ class Int8Interpreter { int64_t last_invoke_us() const { return last_invoke_us_; } - /// Duration of operation @p op during the last invoke(), or 0. + /// Duration of operation @p op during the last invoke(), or 0 without + /// Int8RuntimeOptions::profile_operations. int64_t op_time_us(uint32_t op) const; /// Size of the activation arena. size_t arena_bytes() const { @@ -229,6 +233,7 @@ class Int8Interpreter { size_t phase_bytes_ = 0; int32_t* params_ = nullptr; size_t params_size_ = 0; + int64_t* op_us_ = nullptr; uint32_t inputs_[MAX_GRAPH_IO]{}; uint32_t outputs_[MAX_GRAPH_IO]{}; size_t input_count_ = 0; diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 06b21783611..df2bcbed420 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -198,6 +198,9 @@ class EspNNBackendInterface final : public BackendInterface { #endif #if CONFIG_ESPNN_DELEGATE_DUAL_CORE options.dual_core = true; +#endif +#if CONFIG_ESPNN_DELEGATE_PROFILE + options.profile_operations = true; #endif if (!state->interpreter.load( static_cast(processed->data()), diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 391574781c2..0d21492d556 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -30,6 +30,10 @@ namespace { constexpr size_t ARENA_ALIGNMENT = 16; constexpr int64_t YIELD_INTERVAL_US = 1000000; +// Without profiling, invoke() checks the yield deadline only every this many +// operations and when it returns: one timer read costs as much as a small +// operation. +constexpr uint32_t YIELD_CHECK_OPERATIONS = 16; SemaphoreHandle_t invoke_mutex() { static StaticSemaphore_t storage; @@ -173,9 +177,6 @@ struct Int8Interpreter::OpState { int32_t dilation_h; int32_t dilation_w; int32_t batches; - int64_t total_us; - int64_t last_us; - uint32_t calls; }; Int8Interpreter::~Int8Interpreter() { @@ -201,6 +202,8 @@ void Int8Interpreter::reset() { heap_caps_free(operations_); if (op_states_ != nullptr) heap_caps_free(op_states_); + if (op_us_ != nullptr) + heap_caps_free(op_us_); if (tensor_offsets_ != nullptr) heap_caps_free(tensor_offsets_); arena_ = nullptr; @@ -210,6 +213,7 @@ void Int8Interpreter::reset() { tensors_ = nullptr; operations_ = nullptr; op_states_ = nullptr; + op_us_ = nullptr; tensor_offsets_ = nullptr; arena_size_ = 0; scratch_size_ = 0; @@ -266,8 +270,13 @@ bool Int8Interpreter::load( heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); tensor_offsets_ = static_cast( heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + if (options_.profile_operations) { + op_us_ = static_cast( + heap_caps_calloc(op_count, sizeof(int64_t), MALLOC_CAP_8BIT)); + } if (tensors_ == nullptr || operations_ == nullptr || op_states_ == nullptr || - tensor_offsets_ == nullptr) { + tensor_offsets_ == nullptr || + (options_.profile_operations && op_us_ == nullptr)) { std::printf("ERROR: unable to allocate interpreter tables\n"); reset(); return false; @@ -926,8 +935,9 @@ int32_t Int8Interpreter::output_argmax(size_t index) const { } int64_t Int8Interpreter::op_time_us(uint32_t op) const { - return loaded_ && op < graph_.header.num_operations ? op_states_[op].last_us - : 0; + return loaded_ && op_us_ != nullptr && op < graph_.header.num_operations + ? op_us_[op] + : 0; } bool Int8Interpreter::run_operation(uint32_t index) { @@ -1680,10 +1690,10 @@ bool Int8Interpreter::invoke() { esp_nn_set_depthwise_conv_scratch_buf(scratch_); esp_nn_set_mean_scratch_buf(scratch_); } - int64_t compute_us = 0; - int64_t last_yield = esp_timer_get_time(); + const int64_t invoke_started = esp_timer_get_time(); + int64_t op_started = invoke_started; + int64_t last_yield = invoke_started; for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { - const int64_t started = esp_timer_get_time(); if (!run_operation(i)) { std::printf( "ERROR: operation %u (%s) failed\n", @@ -1691,18 +1701,22 @@ bool Int8Interpreter::invoke() { opcode_name(op_states_[i].opcode)); return false; } - const int64_t finished = esp_timer_get_time(); - OpState& state = op_states_[i]; - state.last_us = finished - started; - state.total_us += state.last_us; - ++state.calls; - compute_us += state.last_us; - if (finished - last_yield > YIELD_INTERVAL_US) { + if (op_us_ == nullptr && (i + 1) % YIELD_CHECK_OPERATIONS != 0) + continue; + int64_t now = esp_timer_get_time(); + if (op_us_ != nullptr) + op_us_[i] = now - op_started; + if (now - last_yield > YIELD_INTERVAL_US) { vTaskDelay(1); - last_yield = esp_timer_get_time(); + now = esp_timer_get_time(); + last_yield = now; } + op_started = now; } - last_invoke_us_ = compute_us; + const int64_t finished = esp_timer_get_time(); + if (finished - last_yield > YIELD_INTERVAL_US) + vTaskDelay(1); + last_invoke_us_ = finished - invoke_started; return true; } @@ -1750,7 +1764,7 @@ void Int8Interpreter::print_summary() const { } void Int8Interpreter::print_profile() const { - if (!loaded_) + if (!loaded_ || op_us_ == nullptr) return; int64_t per_type[OPCODE_COUNT]{}; uint32_t count_per_type[OPCODE_COUNT]{}; @@ -1758,7 +1772,7 @@ void Int8Interpreter::print_profile() const { const OpState& s = op_states_[i]; const unsigned type = static_cast(s.opcode); if (type < OPCODE_COUNT) { - per_type[type] += s.last_us; + per_type[type] += op_us_[i]; ++count_per_type[type]; } std::printf( @@ -1772,7 +1786,7 @@ void Int8Interpreter::print_profile() const { static_cast( tensors_[s.output].rank == 4 ? tensors_[s.output].shape[1] : static_cast(s.elements)), - static_cast(s.last_us)); + static_cast(op_us_[i])); } for (unsigned t = 0; t < OPCODE_COUNT; ++t) { if (count_per_type[t] == 0) From 90d07e2e78607252d5b251b6f743ad2fcbc3b32b Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 12:11:06 -0800 Subject: [PATCH 02/25] Build and test the ESP-NN delegate on the host The ESP-NN delegate only built inside ESP-IDF, so checking a runtime change needed the ESP32-S3 emulator or a board. Its sources use few ESP-IDF APIs: the heap_caps allocator, esp_timer, a FreeRTOS mutex and a task yield. Outside ESP-IDF, espnn_runtime/CMakeLists.txt now builds the same sources as the espnn_backend library, with host/include standing in for those APIs and ESP-NN's portable C kernels fetched from GitHub at the v1.5.0 commit (or taken from ESPNN_LOCAL_PATH). backends/espressif/test/host builds it with the ExecuTorch runtime into espnn_host_runner, and test_host_runtime.py runs every fully delegated test model through it, over several steps and several loaded instances of a method, with planned and with caller-owned inputs and outputs, checking each output bit-exactly against ProgramEmulator. .ci/scripts/test_espressif_host.sh runs the test on a plain build and on an AddressSanitizer and UndefinedBehaviorSanitizer build, in a new job of the Espressif workflow that needs no ESP-IDF toolchain. The host build exercises the delegate's planner, interpreter and ExecuTorch glue, not ESP-NN's S3 assembly kernels, which the emulator jobs still cover. --- .ci/scripts/test_espressif_host.sh | 30 +++ .github/workflows/test-espressif.yml | 22 ++ backends/espressif/README.md | 25 +- .../espressif/espnn_runtime/CMakeLists.txt | 11 + .../espnn_runtime/host/espnn_host.cmake | 82 ++++++ .../host/include/esp_heap_caps.h | 46 ++++ .../espnn_runtime/host/include/esp_timer.h | 22 ++ .../host/include/freertos/FreeRTOS.h | 27 ++ .../host/include/freertos/semphr.h | 28 ++ .../host/include/freertos/task.h | 15 ++ .../espnn_runtime/host/include/sdkconfig.h | 14 + backends/espressif/test/host/CMakeLists.txt | 61 +++++ .../espressif/test/host/espnn_host_runner.cpp | 254 ++++++++++++++++++ backends/espressif/test/test_host_runtime.py | 154 +++++++++++ 14 files changed, 789 insertions(+), 2 deletions(-) create mode 100755 .ci/scripts/test_espressif_host.sh create mode 100644 backends/espressif/espnn_runtime/host/espnn_host.cmake create mode 100644 backends/espressif/espnn_runtime/host/include/esp_heap_caps.h create mode 100644 backends/espressif/espnn_runtime/host/include/esp_timer.h create mode 100644 backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h create mode 100644 backends/espressif/espnn_runtime/host/include/freertos/semphr.h create mode 100644 backends/espressif/espnn_runtime/host/include/freertos/task.h create mode 100644 backends/espressif/espnn_runtime/host/include/sdkconfig.h create mode 100644 backends/espressif/test/host/CMakeLists.txt create mode 100644 backends/espressif/test/host/espnn_host_runner.cpp create mode 100644 backends/espressif/test/test_host_runtime.py diff --git a/.ci/scripts/test_espressif_host.sh b/.ci/scripts/test_espressif_host.sh new file mode 100755 index 00000000000..1190f9c3d4a --- /dev/null +++ b/.ci/scripts/test_espressif_host.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Builds the ESP-NN delegate for the host, plain and with ASan/UBSan, and checks +# fully delegated programs against the bit-exact Python reference on both. Needs +# no ESP-IDF toolchain or emulator. + +set -euo pipefail + +et_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +build_dir="${ESPRESSIF_HOST_BUILD_DIR:-${et_root}/cmake-out-espnn-host}" +cd "${et_root}" + +for variant in plain sanitized; do + sanitize=OFF + if [[ "${variant}" == sanitized ]]; then + sanitize=ON + fi + cmake -S backends/espressif/test/host -B "${build_dir}/${variant}" -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DPYTHON_EXECUTABLE="$(command -v python)" \ + -DESPNN_HOST_SANITIZE="${sanitize}" + cmake --build "${build_dir}/${variant}" + ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \ + python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py +done diff --git a/.github/workflows/test-espressif.yml b/.github/workflows/test-espressif.yml index 055c5b3a813..e8c5f891ca7 100644 --- a/.github/workflows/test-espressif.yml +++ b/.github/workflows/test-espressif.yml @@ -32,6 +32,7 @@ on: - tools/cmake/preset/esp_baremetal.cmake - CMakePresets.json - .ci/scripts/test_espressif.sh + - .ci/scripts/test_espressif_host.sh - .ci/scripts/test_espressif_idf_component.sh - .ci/scripts/test_espressif_idf_consumer.sh - .ci/scripts/esp_idf_mirror.py @@ -75,6 +76,27 @@ jobs: install_executorch --use-pt-pinned-commit --minimal ESPRESSIF_NN_KERNELS="${{ matrix.kernels }}" bash .ci/scripts/test_espressif.sh + test-host: + name: ESP-NN delegate on the host + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read + with: + runner: mt-l-x86iavx512-8-64 + docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 45 + script: | + set -euo pipefail + CONDA_ENV=$(conda env list --json | jq -r '.envs | .[-1]') + conda activate "${CONDA_ENV}" + source .ci/scripts/utils.sh + install_executorch --use-pt-pinned-commit --minimal + bash .ci/scripts/test_espressif_host.sh + test-idf-component: name: ESP-IDF component needs: docker-image diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 73a140fb9c7..332921ff0d1 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -206,6 +206,27 @@ ANSI C reference kernels against the same bundled outputs. Both jobs target ESP32-S3. Logs include the kernel configuration in their filenames; artifacts retain the bundled fixtures, SDK configuration, and resolved component versions. +### On the host + +The delegate also builds outside ESP-IDF, for Linux and macOS, over ESP-NN's +portable C kernels. `host/include` stands in for the few ESP-IDF APIs it uses, +and ESP-NN is fetched from GitHub (or taken from `-DESPNN_LOCAL_PATH=...`). +`test_host_runtime.py` runs fully delegated programs through the ExecuTorch +runtime and this build, and checks every output against the host reference, +with no board or emulator: + +```bash +cmake -S backends/espressif/test/host -B cmake-out-espnn-host -G Ninja +cmake --build cmake-out-espnn-host +ESPNN_HOST_RUNNER=cmake-out-espnn-host/espnn_host_runner \ + pytest backends/espressif/test/test_host_runtime.py +``` + +`-DESPNN_HOST_SANITIZE=ON` builds with AddressSanitizer and +UndefinedBehaviorSanitizer; `.ci/scripts/test_espressif_host.sh` runs both +builds. The host build checks the delegate's logic, not ESP-NN's S3 assembly +kernels, which only the emulator and the board run. + The code follows the ExecuTorch coding style and passes the repository's `lintrunner` checks: @@ -233,5 +254,5 @@ lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" Every source file in this backend and its examples is released under the BSD 3-Clause license (`SPDX-License-Identifier: BSD-3-Clause`), the same license as ExecuTorch. See the `LICENSE` file at the repository root. ESP-NN -is Apache-2.0 and is fetched from the Espressif component registry, not -vendored. +is Apache-2.0 and is fetched from the Espressif component registry (or, for +the host build, from GitHub), not vendored. diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt index d909069f942..3765d299d79 100644 --- a/backends/espressif/espnn_runtime/CMakeLists.txt +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -10,6 +10,17 @@ # is meant to sit next to examples/espressif/executor_runner, which imports the # cross-compiled ExecuTorch targets this component links against: build # examples/espressif/project with -DET_ESPNN_BACKEND=ON. +# +# Outside ESP-IDF, this file builds the same delegate for the host (Linux, +# macOS) as the espnn_backend library, over esp-nn's portable C kernels, with +# host/include standing in for the few ESP-IDF APIs it uses. Add it after +# ExecuTorch with add_subdirectory(); backends/espressif/test/host uses it to +# check the delegate against the bit-exact Python reference without a board. + +if(NOT ESP_PLATFORM) + include(${CMAKE_CURRENT_LIST_DIR}/host/espnn_host.cmake) + return() +endif() idf_component_register( SRCS diff --git a/backends/espressif/espnn_runtime/host/espnn_host.cmake b/backends/espressif/espnn_runtime/host/espnn_host.cmake new file mode 100644 index 00000000000..a0c1b6f8919 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/espnn_host.cmake @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Host build of the ESP-NN delegate; included from ../CMakeLists.txt outside +# ESP-IDF. Needs the executorch_core target. esp-nn comes from ESPNN_LOCAL_PATH +# or is fetched at ESPNN_GIT_TAG. + +include(FetchContent) + +set(ESPNN_GIT_TAG + "ace42ac7721119cd543fc40ffe719e601fa8f20f" # v1.5.0 + CACHE STRING "esp-nn commit fetched when ESPNN_LOCAL_PATH is not set" +) +set(ESPNN_LOCAL_PATH + "" + CACHE PATH "Existing esp-nn checkout to build instead of fetching one" +) +option(ESPNN_HOST_OPTIMIZED "Use esp-nn's generic optimized C kernels" OFF) +option(ESPNN_DELEGATE_PROFILE "Time every delegate operator" OFF) + +if(ESPNN_LOCAL_PATH) + set(espnn_source_dir "${ESPNN_LOCAL_PATH}") +else() + # esp-nn's own CMakeLists.txt is an ESP-IDF component; SOURCE_SUBDIR keeps + # FetchContent from adding it. + FetchContent_Declare( + esp_nn + GIT_REPOSITORY https://github.com/espressif/esp-nn.git + GIT_TAG ${ESPNN_GIT_TAG} + SOURCE_SUBDIR none + ) + FetchContent_MakeAvailable(esp_nn) + set(espnn_source_dir "${esp_nn_SOURCE_DIR}") +endif() + +file(GLOB espnn_kernel_sources "${espnn_source_dir}/src/*/*_ansi.c") +if(ESPNN_HOST_OPTIMIZED) + file(GLOB espnn_optimized_sources "${espnn_source_dir}/src/*/*_opt.c") + list(APPEND espnn_kernel_sources ${espnn_optimized_sources}) +endif() +add_library(esp_nn_host STATIC ${espnn_kernel_sources}) +target_include_directories( + esp_nn_host + PUBLIC "${espnn_source_dir}/include" + PRIVATE "${espnn_source_dir}/src/common" +) +if(ESPNN_HOST_OPTIMIZED) + target_compile_definitions(esp_nn_host PUBLIC CONFIG_NN_OPTIMIZED=1) +endif() +set_target_properties(esp_nn_host PROPERTIES POSITION_INDEPENDENT_CODE ON) + +get_filename_component( + espnn_executorch_root "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE +) +add_library( + espnn_backend STATIC + ${CMAKE_CURRENT_LIST_DIR}/../src/xts3_graph.cpp + ${CMAKE_CURRENT_LIST_DIR}/../src/int8_interpreter.cpp + ${CMAKE_CURRENT_LIST_DIR}/../src/espnn_backend.cpp +) +target_include_directories( + espnn_backend + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../include + PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include "${espnn_executorch_root}/.." + "${espnn_executorch_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + espnn_backend PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +if(ESPNN_DELEGATE_PROFILE) + target_compile_definitions( + espnn_backend PRIVATE CONFIG_ESPNN_DELEGATE_PROFILE=1 + ) +endif() +target_link_libraries(espnn_backend PUBLIC executorch_core esp_nn_host) +set_target_properties(espnn_backend PROPERTIES POSITION_INDEPENDENT_CODE ON) +executorch_target_link_options_shared_lib(espnn_backend) diff --git a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h new file mode 100644 index 00000000000..1298d9d2c9c --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for the ESP-IDF heap API the ESP-NN delegate uses: one heap, +// all of it "internal", no PSRAM. + +#pragma once + +#include +#include +#include + +#define MALLOC_CAP_8BIT (1U << 2) +#define MALLOC_CAP_SPIRAM (1U << 10) +#define MALLOC_CAP_INTERNAL (1U << 11) + +inline void* +heap_caps_aligned_alloc(size_t alignment, size_t size, uint32_t caps) { + if ((caps & MALLOC_CAP_SPIRAM) != 0 || size == 0) + return nullptr; + return std::aligned_alloc( + alignment, (size + alignment - 1) & ~(alignment - 1)); +} + +inline void* heap_caps_calloc(size_t count, size_t size, uint32_t) { + return std::calloc(count, size); +} + +inline void heap_caps_free(void* memory) { + std::free(memory); +} + +inline size_t heap_caps_get_largest_free_block(uint32_t caps) { + return (caps & MALLOC_CAP_SPIRAM) != 0 ? 0 : SIZE_MAX / 2; +} + +inline size_t heap_caps_get_total_size(uint32_t caps) { + return heap_caps_get_largest_free_block(caps); +} diff --git a/backends/espressif/espnn_runtime/host/include/esp_timer.h b/backends/espressif/espnn_runtime/host/include/esp_timer.h new file mode 100644 index 00000000000..8166fbeccc4 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_timer.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for esp_timer_get_time(): microseconds of a monotonic clock. + +#pragma once + +#include +#include + +inline int64_t esp_timer_get_time() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} diff --git a/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h b/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h new file mode 100644 index 00000000000..5c7e81325b7 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-ins for the FreeRTOS calls the ESP-NN delegate makes: a mutex +// backed by std::mutex, and a no-op task yield. + +#pragma once + +#include +#include + +typedef uint32_t TickType_t; +typedef int BaseType_t; +#define portMAX_DELAY (static_cast(0xffffffffU)) +#define pdTRUE 1 + +struct StaticSemaphore_t { + std::mutex mutex; +}; +typedef StaticSemaphore_t* SemaphoreHandle_t; diff --git a/backends/espressif/espnn_runtime/host/include/freertos/semphr.h b/backends/espressif/espnn_runtime/host/include/freertos/semphr.h new file mode 100644 index 00000000000..8010080f4f8 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/semphr.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +inline SemaphoreHandle_t xSemaphoreCreateMutexStatic( + StaticSemaphore_t* storage) { + return storage; +} + +inline BaseType_t xSemaphoreTake(SemaphoreHandle_t handle, TickType_t) { + handle->mutex.lock(); + return pdTRUE; +} + +inline BaseType_t xSemaphoreGive(SemaphoreHandle_t handle) { + handle->mutex.unlock(); + return pdTRUE; +} diff --git a/backends/espressif/espnn_runtime/host/include/freertos/task.h b/backends/espressif/espnn_runtime/host/include/freertos/task.h new file mode 100644 index 00000000000..7689b82b914 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/task.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +inline void vTaskDelay(TickType_t) {} diff --git a/backends/espressif/espnn_runtime/host/include/sdkconfig.h b/backends/espressif/espnn_runtime/host/include/sdkconfig.h new file mode 100644 index 00000000000..a32ebafedc4 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/sdkconfig.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for ESP-IDF's generated sdkconfig.h. The host CMake build +// passes the delegate's CONFIG_ESPNN_DELEGATE_* options as definitions. + +#pragma once diff --git a/backends/espressif/test/host/CMakeLists.txt b/backends/espressif/test/host/CMakeLists.txt new file mode 100644 index 00000000000..3b3a67aa8d8 --- /dev/null +++ b/backends/espressif/test/host/CMakeLists.txt @@ -0,0 +1,61 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Host build of the ESP-NN delegate and espnn_host_runner, for +# backends/espressif/test/test_host_runtime.py (see "On the host" in +# backends/espressif/README.md). -DESPNN_HOST_SANITIZE=ON builds everything with +# AddressSanitizer and UndefinedBehaviorSanitizer. + +cmake_minimum_required(VERSION 3.24) +project(espnn_host_runner CXX C) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +get_filename_component( + EXECUTORCH_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../.." ABSOLUTE +) + +option(ESPNN_HOST_SANITIZE "Build with ASan and UBSan" OFF) +if(ESPNN_HOST_SANITIZE) + add_compile_options( + -fsanitize=address,undefined -fno-sanitize-recover=all + -fno-omit-frame-pointer + ) + add_link_options(-fsanitize=address,undefined) +endif() + +set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON) +set(EXECUTORCH_BUILD_EXECUTOR_RUNNER OFF) +set(EXECUTORCH_BUILD_PYBIND OFF) +set(EXECUTORCH_BUILD_DEVTOOLS OFF) +set(EXECUTORCH_BUILD_XNNPACK OFF) +set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF) +set(EXECUTORCH_BUILD_KERNELS_QUANTIZED OFF) +set(EXECUTORCH_BUILD_PTHREADPOOL OFF) +set(EXECUTORCH_BUILD_CPUINFO OFF) +set(EXECUTORCH_BUILD_TESTS OFF) +add_subdirectory(${EXECUTORCH_ROOT} executorch EXCLUDE_FROM_ALL) +add_subdirectory( + ${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime espnn_runtime +) +# ESP-IDF's default warnings, as errors. +target_compile_options( + espnn_backend PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-sign-compare + -Werror +) +if(ESPNN_HOST_SANITIZE) + # ESP-NN's C kernels shift negative values left, as TFLite's reference kernels + # do; GCC and Clang define the result. + target_compile_options(esp_nn_host PRIVATE -fno-sanitize=shift-base) +endif() + +add_executable(espnn_host_runner espnn_host_runner.cpp) +target_include_directories(espnn_host_runner PRIVATE "${EXECUTORCH_ROOT}/..") +target_link_libraries( + espnn_host_runner PRIVATE executorch_core extension_data_loader espnn_backend +) diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp new file mode 100644 index 00000000000..8d409524a78 --- /dev/null +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Runs an ExecuTorch program on the host build of the ESP-NN delegate, for + * backends/espressif/test/test_host_runtime.py. + * + * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] + * [--alias INPUT:OUTPUT]... + * + * Step s of instance k reads input i from IO_DIR/in__.bin and writes + * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its + * dim order. With --instances, the method is loaded N times and each step runs + * on every instance in turn. With --share-io (the program's inputs and + * outputs must not be memory planned), the tensors are bound once to runner + * buffers through set_input and set_output_data_ptr, and --alias makes an + * input and an output share one buffer: that input is then read from its file + * at step 0 only and afterwards holds the output of the previous step. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using executorch::aten::ScalarType; +using executorch::aten::TensorImpl; +using executorch::extension::FileDataLoader; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; +using executorch::runtime::TensorInfo; + +namespace { + +[[noreturn]] void fail(const char* what, int code = 0) { + std::fprintf(stderr, "espnn_host_runner: %s (%d)\n", what, code); + std::exit(1); +} + +std::vector read_file(const std::string& path, size_t expected) { + std::FILE* file = std::fopen(path.c_str(), "rb"); + if (file == nullptr) + fail(("cannot open " + path).c_str()); + std::vector data(expected + 1); + const size_t read = std::fread(data.data(), 1, data.size(), file); + std::fclose(file); + if (read != expected) + fail(("wrong size: " + path).c_str(), static_cast(read)); + data.resize(expected); + return data; +} + +void write_file(const std::string& path, const void* data, size_t size) { + std::FILE* file = std::fopen(path.c_str(), "wb"); + if (file == nullptr || std::fwrite(data, 1, size, file) != size) + fail(("cannot write " + path).c_str()); + std::fclose(file); +} + +// A tensor over a runner-owned buffer with the program's shape and dim order. +struct HostTensor { + explicit HostTensor(const TensorInfo& info) + : sizes(info.sizes().begin(), info.sizes().end()), + dim_order(info.dim_order().begin(), info.dim_order().end()), + strides(sizes.size()), + data(info.nbytes()), + impl( + info.scalar_type(), + static_cast(sizes.size()), + sizes.data(), + data.data(), + dim_order.data(), + strides.data()) { + if (executorch::runtime::dim_order_to_stride( + sizes.data(), dim_order.data(), sizes.size(), strides.data()) != + Error::Ok) + fail("invalid dim order"); + } + std::vector sizes; + std::vector dim_order; + std::vector strides; + std::vector data; + TensorImpl impl; +}; + +struct Instance { + std::vector> planned; + std::vector> spans; + std::unique_ptr planned_allocator; + std::vector method_pool; + std::unique_ptr method_allocator; + std::unique_ptr memory; + std::unique_ptr method; + std::vector> inputs; + std::vector> outputs; +}; + +} // namespace + +int main(int argc, char** argv) { + if (argc < 4) + fail("usage: espnn_host_runner PROGRAM IO_DIR STEPS [options]"); + const std::string io_dir = argv[2]; + const int steps = std::atoi(argv[3]); + int instance_count = 1; + bool share_io = false; + std::vector> aliases; + for (int a = 4; a < argc; ++a) { + const std::string arg = argv[a]; + if (arg == "--instances" && a + 1 < argc) { + instance_count = std::atoi(argv[++a]); + } else if (arg == "--share-io") { + share_io = true; + } else if (arg == "--alias" && a + 1 < argc) { + unsigned input = 0, output = 0; + if (std::sscanf(argv[++a], "%u:%u", &input, &output) != 2) + fail("--alias takes INPUT:OUTPUT"); + aliases.emplace_back(input, output); + } else { + fail(("unknown argument " + arg).c_str()); + } + } + if (!aliases.empty() && !share_io) + fail("--alias needs --share-io"); + + executorch::runtime::runtime_init(); + Result loader = FileDataLoader::from(argv[1]); + if (!loader.ok()) + fail("cannot load program", static_cast(loader.error())); + Result program = Program::load(&loader.get()); + if (!program.ok()) + fail("invalid program", static_cast(program.error())); + Result meta = program->method_meta("forward"); + if (!meta.ok()) + fail("no forward method", static_cast(meta.error())); + + std::vector instances(instance_count); + for (Instance& instance : instances) { + for (size_t b = 0; b < meta->num_memory_planned_buffers(); ++b) { + instance.planned.emplace_back(meta->memory_planned_buffer_size(b).get()); + } + for (auto& buffer : instance.planned) + instance.spans.emplace_back(buffer.data(), buffer.size()); + instance.planned_allocator = std::make_unique( + Span>(instance.spans.data(), instance.spans.size())); + instance.method_pool.resize(1 << 20); + instance.method_allocator = std::make_unique( + instance.method_pool.size(), instance.method_pool.data()); + instance.memory = std::make_unique( + instance.method_allocator.get(), instance.planned_allocator.get()); + Result method = + program->load_method("forward", instance.memory.get()); + if (!method.ok()) + fail("cannot load method", static_cast(method.error())); + instance.method = std::make_unique(std::move(method.get())); + + for (size_t i = 0; i < meta->num_inputs(); ++i) { + instance.inputs.push_back( + std::make_unique(meta->input_tensor_meta(i).get())); + } + for (size_t o = 0; o < meta->num_outputs(); ++o) { + instance.outputs.push_back( + std::make_unique(meta->output_tensor_meta(o).get())); + } + for (const auto& [input, output] : aliases) { + if (input >= instance.inputs.size() || + output >= instance.outputs.size() || + instance.inputs[input]->data.size() != + instance.outputs[output]->data.size()) + fail("invalid --alias"); + } + if (!share_io) + continue; + for (size_t i = 0; i < instance.inputs.size(); ++i) { + Error error = instance.method->set_input( + EValue(executorch::aten::Tensor(&instance.inputs[i]->impl)), i); + if (error != Error::Ok) + fail("set_input", static_cast(error)); + } + for (size_t o = 0; o < instance.outputs.size(); ++o) { + void* buffer = instance.outputs[o]->data.data(); + for (const auto& [input, output] : aliases) { + if (output == o) + buffer = instance.inputs[input]->data.data(); + } + Error error = instance.method->set_output_data_ptr( + buffer, instance.outputs[o]->data.size(), o); + if (error != Error::Ok) + fail("set_output_data_ptr", static_cast(error)); + } + } + + for (int step = 0; step < steps; ++step) { + for (int k = 0; k < instance_count; ++k) { + Instance& instance = instances[k]; + for (size_t i = 0; i < instance.inputs.size(); ++i) { + bool carried = false; + for (const auto& alias : aliases) + carried = carried || (alias.first == i && step > 0); + if (carried) + continue; + HostTensor& tensor = *instance.inputs[i]; + const std::vector bytes = read_file( + io_dir + "/in_" + std::to_string(step) + "_" + std::to_string(i) + + ".bin", + tensor.data.size()); + std::memcpy(tensor.data.data(), bytes.data(), bytes.size()); + if (!share_io) { + Error error = instance.method->set_input( + EValue(executorch::aten::Tensor(&tensor.impl)), i); + if (error != Error::Ok) + fail("set_input", static_cast(error)); + } + } + Error error = instance.method->execute(); + if (error != Error::Ok) + fail("execute", static_cast(error)); + for (size_t o = 0; o < instance.outputs.size(); ++o) { + const executorch::aten::Tensor output = + instance.method->get_output(o).toTensor(); + write_file( + io_dir + "/out_" + std::to_string(k) + "_" + std::to_string(step) + + "_" + std::to_string(o) + ".bin", + output.const_data_ptr(), + output.nbytes()); + } + } + } + return 0; +} diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py new file mode 100644 index 00000000000..4ba43244b26 --- /dev/null +++ b/backends/espressif/test/test_host_runtime.py @@ -0,0 +1,154 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Runs fully delegated programs on the host build of the ESP-NN delegate +(espnn_runtime built outside ESP-IDF, over esp-nn's portable C kernels) and +checks every output against ProgramEmulator, the bit-exact Python reference. +Skipped unless ESPNN_HOST_RUNNER points to espnn_host_runner; see +backends/espressif/test/host/CMakeLists.txt. + + ESPNN_HOST_RUNNER=cmake-out-espnn-host/espnn_host_runner \ + pytest backends/espressif/test/test_host_runtime.py +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif import to_espnn_edge +from executorch.backends.espressif.reference import ProgramEmulator +from executorch.backends.espressif.test import ( + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass + +RUNNER = os.environ.get("ESPNN_HOST_RUNNER") +pytestmark = pytest.mark.skipif( + not RUNNER, reason="Set ESPNN_HOST_RUNNER to espnn_host_runner" +) + +# The runner links no CPU kernels, so only fully delegated programs run on it; +# transpose-select-stack leaves a cat on the CPU. +CASES = [ + ("conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)]), + ("concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)]), + ("se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]), +] + [ + (name, model, shapes) + for name, model, shapes, _ in op_cases.CASES + if name != "transpose-select-stack" +] + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def lower(model, shapes, planned_io: bool = True): + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert not report["cpu_ops"] + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=planned_io, alloc_graph_output=planned_io + ) + ) + return edge.exported_program(), edge.to_executorch(config).buffer + + +def as_list(value): + return list(value) if isinstance(value, (tuple, list)) else [value] + + +def run_host(tmp_path: Path, program: bytes, steps, *args) -> list[list[bytes]]: + """Runs one list of input tensors per step; returns raw outputs per step.""" + (tmp_path / "program.pte").write_bytes(program) + for step, inputs in enumerate(steps): + for i, tensor in enumerate(inputs): + (tmp_path / f"in_{step}_{i}.bin").write_bytes( + tensor.contiguous().numpy().tobytes() + ) + result = subprocess.run( + [RUNNER, str(tmp_path / "program.pte"), str(tmp_path), str(len(steps))] + + [str(arg) for arg in args], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + instances = 1 + if "--instances" in args: + instances = int(args[args.index("--instances") + 1]) + return [ + [sorted(tmp_path.glob(f"out_{k}_{step}_*.bin")) for step in range(len(steps))] + for k in range(instances) + ] + + +def assert_matches(paths: list[Path], expected: list[torch.Tensor]): + assert len(paths) == len(expected) + for path, tensor in zip(paths, expected): + actual = torch.frombuffer( + bytearray(path.read_bytes()), dtype=tensor.dtype + ).reshape(tensor.shape) + if tensor.dtype == torch.float32: + torch.testing.assert_close(actual, tensor, atol=1e-5, rtol=1e-5) + else: + assert torch.equal(actual, tensor) + + +def random_steps(shapes, count: int, seed: int = 11): + generator = torch.Generator().manual_seed(seed) + return [ + [torch.randn(shape, generator=generator) for shape in shapes] + for _ in range(count) + ] + + +@pytest.mark.parametrize("name, model, shapes", CASES, ids=[c[0] for c in CASES]) +def test_host_delegate_matches_reference(tmp_path, name, model, shapes): + exported, program = lower(model, shapes) + steps = random_steps(shapes, 2) + (outputs,) = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + + +def test_instances_run_independently(tmp_path): + model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + exported, program = lower(model, shapes) + steps = random_steps(shapes, 3) + first, second = run_host(tmp_path, program, steps, "--instances", 2) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert_matches(first[step], expected) + assert_matches(second[step], expected) + + +def test_unplanned_io_shared_with_the_caller(tmp_path): + model, shapes = op_cases.TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)] + exported, program = lower(model, shapes, planned_io=False) + steps = random_steps(shapes, 2) + (outputs,) = run_host(tmp_path, program, steps, "--share-io") + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) From 3a99bfafb88f64eaa5c9e94007e035c17c9b4573 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 12:16:50 -0800 Subject: [PATCH 03/25] Shrink the ESP-NN delegate's tables, load-time stack and scratch On a 512 KB-SRAM part the delegate's own bookkeeping can cost more internal RAM than a small model's activations. For a 32-operator streaming keyword-spotting model on ESP32-S3 it kept 10.5 KB of per-operator state, 1.4 KB of operation records that are only needed while loading, and a separate 5.8 KB ESP-NN scratch buffer, and graph validation put four 1,024-byte bool arrays on the loading task's stack. Fields of the per-operator state that no operator uses together now share storage, operation records are decoded while loading instead of being kept, the planner frees its tables before allocating the arena, and graph validation uses bitsets. The invoke mutex is created by a static initializer instead of a function-local static, so invoke() no longer goes through a thread-safe static guard. ESPNN_DELEGATE_SCRATCH_IN_ARENA (off by default) plans each operator's ESP-NN scratch as an arena entry that lives only while the operator runs; for a model whose arena stays in internal RAM this replaces the separate buffer. DelegateStats now also reports the scratch, requantization and table bytes. The host test checks the delegate's arena against plan_activation_arena(), its Python mirror, and the sanitized host build runs with scratch in the arena and per-operator profiling on. --- .ci/scripts/test_espressif_host.sh | 11 +- backends/espressif/README.md | 1 + backends/espressif/espnn_runtime/Kconfig | 11 ++ .../espnn_runtime/host/espnn_host.cmake | 13 +- .../include/espnn/espnn_backend.h | 5 + .../include/espnn/int8_interpreter.h | 10 +- .../espnn_runtime/src/espnn_backend.cpp | 6 + .../espnn_runtime/src/int8_interpreter.cpp | 157 ++++++++++++------ .../espnn_runtime/src/xts3_graph.cpp | 34 +++- .../espressif/test/host/espnn_host_runner.cpp | 13 +- backends/espressif/test/test_host_runtime.py | 39 ++++- 11 files changed, 218 insertions(+), 82 deletions(-) diff --git a/.ci/scripts/test_espressif_host.sh b/.ci/scripts/test_espressif_host.sh index 1190f9c3d4a..2561adcebb4 100755 --- a/.ci/scripts/test_espressif_host.sh +++ b/.ci/scripts/test_espressif_host.sh @@ -6,8 +6,9 @@ # LICENSE file in the root directory of this source tree. # Builds the ESP-NN delegate for the host, plain and with ASan/UBSan, and checks -# fully delegated programs against the bit-exact Python reference on both. Needs -# no ESP-IDF toolchain or emulator. +# fully delegated programs against the bit-exact Python reference on both. The +# sanitized build also turns on the delegate's non-default options. Needs no +# ESP-IDF toolchain or emulator. set -euo pipefail @@ -20,10 +21,12 @@ for variant in plain sanitized; do if [[ "${variant}" == sanitized ]]; then sanitize=ON fi + options=(-DESPNN_HOST_SANITIZE="${sanitize}" + -DESPNN_DELEGATE_SCRATCH_IN_ARENA="${sanitize}" + -DESPNN_DELEGATE_PROFILE="${sanitize}") cmake -S backends/espressif/test/host -B "${build_dir}/${variant}" -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DPYTHON_EXECUTABLE="$(command -v python)" \ - -DESPNN_HOST_SANITIZE="${sanitize}" + -DPYTHON_EXECUTABLE="$(command -v python)" "${options[@]}" cmake --build "${build_dir}/${variant}" ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \ python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 332921ff0d1..810ca275cda 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -174,6 +174,7 @@ Other options, in menuconfig under *ExecuTorch ESP-NN backend*: |---|---|---| | `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | | `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | +| `ESPNN_DELEGATE_SCRATCH_IN_ARENA` | n | Plans each operator's ESP-NN scratch in the activation arena instead of a separate buffer sized for the largest operator. Saves internal RAM when the arena fits there; for a large model it moves the scratch to PSRAM with the arena | | `ESPNN_DELEGATE_PROFILE` | n | Times every operator and prints a per-operator time table after a delegate's first run. The timer reads cost about 1 µs per operator on the ESP32-S3, so leave it off in production | ### The original ESP32 diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index 1ec69fa6be2..6cff21ee7f0 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -17,6 +17,17 @@ menu "ExecuTorch ESP-NN backend" bool "Split ESP-NN kernels across both cores inside delegates" default n + config ESPNN_DELEGATE_SCRATCH_IN_ARENA + bool "Plan ESP-NN scratch inside each delegate's activation arena" + default n + help + Gives each operator's ESP-NN scratch a slot in the activation + arena that is live only while the operator runs, instead of a + separate buffer sized for the largest operator. This saves + internal RAM for small models whose arena fits there; for a model + whose arena goes to PSRAM, the scratch moves to PSRAM too, which + is slower. + config ESPNN_DELEGATE_PROFILE bool "Print a per-operator profile after each delegate's first run" default n diff --git a/backends/espressif/espnn_runtime/host/espnn_host.cmake b/backends/espressif/espnn_runtime/host/espnn_host.cmake index a0c1b6f8919..c4bda908409 100644 --- a/backends/espressif/espnn_runtime/host/espnn_host.cmake +++ b/backends/espressif/espnn_runtime/host/espnn_host.cmake @@ -22,6 +22,9 @@ set(ESPNN_LOCAL_PATH ) option(ESPNN_HOST_OPTIMIZED "Use esp-nn's generic optimized C kernels" OFF) option(ESPNN_DELEGATE_PROFILE "Time every delegate operator" OFF) +option(ESPNN_DELEGATE_SCRATCH_IN_ARENA + "Plan ESP-NN scratch in the activation arena" OFF +) if(ESPNN_LOCAL_PATH) set(espnn_source_dir "${ESPNN_LOCAL_PATH}") @@ -72,11 +75,11 @@ target_include_directories( target_compile_definitions( espnn_backend PRIVATE C10_USING_CUSTOM_GENERATED_MACROS ) -if(ESPNN_DELEGATE_PROFILE) - target_compile_definitions( - espnn_backend PRIVATE CONFIG_ESPNN_DELEGATE_PROFILE=1 - ) -endif() +foreach(option ESPNN_DELEGATE_PROFILE ESPNN_DELEGATE_SCRATCH_IN_ARENA) + if(${option}) + target_compile_definitions(espnn_backend PRIVATE CONFIG_${option}=1) + endif() +endforeach() target_link_libraries(espnn_backend PUBLIC executorch_core esp_nn_host) set_target_properties(espnn_backend PROPERTIES POSITION_INDEPENDENT_CODE ON) executorch_target_link_options_shared_lib(espnn_backend) diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index d5293be3bfa..5c8810683b1 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -33,6 +33,11 @@ struct DelegateStats { int64_t total_us; /// Sum of the activation arenas held by the initialized delegates. size_t arena_bytes; + /// Sums of the separate ESP-NN scratch buffers, the requantization tables + /// and the per-operation and per-tensor tables of the initialized delegates. + size_t scratch_bytes; + size_t parameter_bytes; + size_t table_bytes; }; /** diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 1f4e065f2ab..2a046b69333 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -44,6 +44,11 @@ struct Int8RuntimeOptions { /// Time every operation for op_time_us() and print_profile(). Otherwise /// invoke() reads the timer only a few times per call. bool profile_operations = false; + /// Plan each operation's ESP-NN scratch in the activation arena, live only + /// while the operation runs, instead of one buffer sized for the largest + /// operation. Saves memory, but the scratch then lives wherever the arena + /// does (PSRAM for a large model). + bool scratch_in_arena = false; }; /** @@ -175,10 +180,12 @@ class Int8Interpreter { bool arena_in_internal_ram() const { return arena_internal_; } - /// Size of the shared ESP-NN scratch buffer. + /// Size of the separate ESP-NN scratch buffer (0 with scratch_in_arena). size_t scratch_bytes() const { return scratch_size_; } + /// Size of the per-operation and per-tensor tables kept after load(). + size_t table_bytes() const; /// Size of the precomputed requantization tables. size_t parameter_bytes() const { return params_size_; @@ -221,7 +228,6 @@ class Int8Interpreter { bool loaded_ = false; uint8_t* blob_copy_ = nullptr; TensorRecord* tensors_ = nullptr; - OperationRecord* operations_ = nullptr; OpState* op_states_ = nullptr; size_t* tensor_offsets_ = nullptr; uint8_t* arena_ = nullptr; diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index df2bcbed420..b9876f7a59c 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -201,6 +201,9 @@ class EspNNBackendInterface final : public BackendInterface { #endif #if CONFIG_ESPNN_DELEGATE_PROFILE options.profile_operations = true; +#endif +#if CONFIG_ESPNN_DELEGATE_SCRATCH_IN_ARENA + options.scratch_in_arena = true; #endif if (!state->interpreter.load( static_cast(processed->data()), @@ -213,6 +216,9 @@ class EspNNBackendInterface final : public BackendInterface { processed->Free(); ++g_stats.delegates; g_stats.arena_bytes += state->interpreter.arena_bytes(); + g_stats.scratch_bytes += state->interpreter.scratch_bytes(); + g_stats.parameter_bytes += state->interpreter.parameter_bytes(); + g_stats.table_bytes += state->interpreter.table_bytes(); return state; } diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 0d21492d556..0c6950cfeba 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -35,15 +35,15 @@ constexpr int64_t YIELD_INTERVAL_US = 1000000; // operation. constexpr uint32_t YIELD_CHECK_OPERATIONS = 16; -SemaphoreHandle_t invoke_mutex() { - static StaticSemaphore_t storage; - static SemaphoreHandle_t handle = xSemaphoreCreateMutexStatic(&storage); - return handle; -} +// Created by a static initializer rather than on first use, so that invoke() +// needs no thread-safe static guard (and no C++ runtime support for one). +StaticSemaphore_t g_invoke_mutex_storage; +const SemaphoreHandle_t g_invoke_mutex = + xSemaphoreCreateMutexStatic(&g_invoke_mutex_storage); class InvokeLock { public: - InvokeLock() : handle_(invoke_mutex()) { + InvokeLock() : handle_(g_invoke_mutex) { xSemaphoreTake(handle_, portMAX_DELAY); } ~InvokeLock() { @@ -145,25 +145,34 @@ int8_t quantize_value(float value, float inverse_scale, int32_t zero_point) { } // namespace +// Fields that no operation uses together share storage: the input list +// (CONCAT, COPY) with the pooling parameters, and the convolution, +// depthwise-convolution and add parameters with each other. struct Int8Interpreter::OpState { OpCode opcode; uint32_t input0; uint32_t input1; uint32_t output; uint8_t input_count; - uint32_t inputs[6]; - int32_t pool[6]; + union { + uint32_t inputs[6]; + int32_t pool[6]; + }; const int8_t* weights; const int32_t* bias; data_dims_t input_dims; data_dims_t filter_dims; data_dims_t output_dims; - conv_params_t conv; - dw_conv_params_t dw; + union { + conv_params_t conv; + dw_conv_params_t dw; + Int8AddParams add; + }; quant_data_t quant; int32_t* multipliers; int32_t* shifts; - Int8AddParams add; + uint32_t scratch_bytes; + uint32_t scratch_offset; int32_t value0; int32_t value1; int32_t input_zero_point; @@ -198,8 +207,6 @@ void Int8Interpreter::reset() { heap_caps_free(blob_copy_); if (tensors_ != nullptr) heap_caps_free(tensors_); - if (operations_ != nullptr) - heap_caps_free(operations_); if (op_states_ != nullptr) heap_caps_free(op_states_); if (op_us_ != nullptr) @@ -211,7 +218,6 @@ void Int8Interpreter::reset() { params_ = nullptr; blob_copy_ = nullptr; tensors_ = nullptr; - operations_ = nullptr; op_states_ = nullptr; op_us_ = nullptr; tensor_offsets_ = nullptr; @@ -264,8 +270,6 @@ bool Int8Interpreter::load( const uint32_t op_count = graph_.header.num_operations; tensors_ = static_cast( heap_caps_calloc(tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); - operations_ = static_cast( - heap_caps_calloc(op_count, sizeof(OperationRecord), MALLOC_CAP_8BIT)); op_states_ = static_cast( heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); tensor_offsets_ = static_cast( @@ -274,7 +278,7 @@ bool Int8Interpreter::load( op_us_ = static_cast( heap_caps_calloc(op_count, sizeof(int64_t), MALLOC_CAP_8BIT)); } - if (tensors_ == nullptr || operations_ == nullptr || op_states_ == nullptr || + if (tensors_ == nullptr || op_states_ == nullptr || tensor_offsets_ == nullptr || (options_.profile_operations && op_us_ == nullptr)) { std::printf("ERROR: unable to allocate interpreter tables\n"); @@ -296,10 +300,9 @@ bool Int8Interpreter::load( inputs_[i] = graph_input_id(graph_, i); for (uint32_t i = 0; i < output_count_; ++i) outputs_[i] = graph_output_id(graph_, i); - for (uint32_t i = 0; i < op_count; ++i) - read_operation(graph_, i, operations_[i]); - if (!plan_memory() || !prepare_operations()) { + // Operations first: the planner places each operation's ESP-NN scratch. + if (!prepare_operations() || !plan_memory()) { reset(); return false; } @@ -339,25 +342,36 @@ bool Int8Interpreter::load( bool Int8Interpreter::plan_memory() { const uint32_t tensor_count = graph_.header.num_tensors; const int32_t op_count = static_cast(graph_.header.num_operations); + // Entries below tensor_count are tensors. With scratch_in_arena, entry + // tensor_count + op is that operation's ESP-NN scratch, live only while the + // operation runs. + const uint32_t entries = tensor_count + + (options_.scratch_in_arena ? static_cast(op_count) : 0U); int32_t* first = static_cast( - heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(int32_t), MALLOC_CAP_8BIT)); int32_t* last = static_cast( - heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(int32_t), MALLOC_CAP_8BIT)); size_t* bytes = static_cast( - heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(size_t), MALLOC_CAP_8BIT)); + size_t* offsets = static_cast( + heap_caps_calloc(entries, sizeof(size_t), MALLOC_CAP_8BIT)); bool* planned = static_cast( - heap_caps_calloc(tensor_count, sizeof(bool), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(bool), MALLOC_CAP_8BIT)); struct PlanBuffers { - void* buffers[4]; - ~PlanBuffers() { - for (void* buffer : buffers) { + void* buffers[5]; + void release() { + for (void*& buffer : buffers) { if (buffer != nullptr) heap_caps_free(buffer); + buffer = nullptr; } } - } plan_buffers{{first, last, bytes, planned}}; + ~PlanBuffers() { + release(); + } + } plan_buffers{{first, last, bytes, offsets, planned}}; if (first == nullptr || last == nullptr || bytes == nullptr || - planned == nullptr) { + offsets == nullptr || planned == nullptr) { std::printf("ERROR: unable to allocate memory-planner tables\n"); return false; } @@ -367,10 +381,20 @@ bool Int8Interpreter::plan_memory() { last[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); - tensor_offsets_[i] = SIZE_MAX; + planned[i] = (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0; + offsets[i] = SIZE_MAX; + } + for (uint32_t e = tensor_count; e < entries; ++e) { + const int32_t op = static_cast(e - tensor_count); + const size_t scratch = op_states_[op].scratch_bytes; + first[e] = scratch != 0 ? op : INT32_MAX; + last[e] = op; + bytes[e] = align_up(scratch, ARENA_ALIGNMENT); + offsets[e] = SIZE_MAX; } for (int32_t op = 0; op < op_count; ++op) { - const OperationRecord& record = operations_[op]; + OperationRecord record{}; + read_operation(graph_, static_cast(op), record); for (uint8_t j = 0; j < record.num_inputs; ++j) { const uint32_t id = record.inputs[j]; if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) == 0 && last[id] < op) @@ -386,12 +410,10 @@ bool Int8Interpreter::plan_memory() { } size_t arena_size = 0; - for (uint32_t step = 0; step < tensor_count; ++step) { + for (uint32_t step = 0; step < entries; ++step) { int32_t best = -1; - for (uint32_t i = 0; i < tensor_count; ++i) { - if (planned[i] || (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0) - continue; - if (first[i] == INT32_MAX) + for (uint32_t i = 0; i < entries; ++i) { + if (planned[i] || first[i] == INT32_MAX) continue; if (best < 0 || bytes[i] > bytes[best]) best = static_cast(i); @@ -403,14 +425,14 @@ bool Int8Interpreter::plan_memory() { bool moved = true; while (moved) { moved = false; - for (uint32_t j = 0; j < tensor_count; ++j) { - if (j == static_cast(best) || tensor_offsets_[j] == SIZE_MAX) + for (uint32_t j = 0; j < entries; ++j) { + if (j == static_cast(best) || offsets[j] == SIZE_MAX) continue; const bool lifetime_overlap = first[j] <= last[best] && first[best] <= last[j]; if (!lifetime_overlap) continue; - const size_t other_begin = tensor_offsets_[j]; + const size_t other_begin = offsets[j]; const size_t other_end = other_begin + bytes[j]; if (candidate < other_end && other_begin < candidate + bytes[best]) { candidate = other_end; @@ -418,19 +440,26 @@ bool Int8Interpreter::plan_memory() { } } } - tensor_offsets_[best] = candidate; + offsets[best] = candidate; if (candidate + bytes[best] > arena_size) arena_size = candidate + bytes[best]; } for (uint32_t i = 0; i < tensor_count; ++i) { + tensor_offsets_[i] = offsets[i]; if ((tensors_[i].flags & TENSOR_FLAG_CONSTANT) == 0 && - tensor_offsets_[i] == SIZE_MAX) { + offsets[i] == SIZE_MAX) { std::printf( "ERROR: tensor %u is never produced or consumed\n", static_cast(i)); return false; } } + for (uint32_t e = tensor_count; e < entries; ++e) { + op_states_[e - tensor_count].scratch_offset = + offsets[e] == SIZE_MAX ? 0 : static_cast(offsets[e]); + } + // Loading then never needs the planner's tables and the arena at once. + plan_buffers.release(); arena_size_ = arena_size; arena_ = static_cast(allocate_preferring_internal( @@ -452,7 +481,8 @@ bool Int8Interpreter::prepare_operations() { const uint32_t op_count = graph_.header.num_operations; size_t param_words = 0; for (uint32_t i = 0; i < op_count; ++i) { - const OperationRecord& op = operations_[i]; + OperationRecord op{}; + read_operation(graph_, i, op); const OpCode code = static_cast(op.opcode); if (code == OpCode::CONV2D || code == OpCode::DEPTHWISE_CONV2D || code == OpCode::LINEAR || code == OpCode::TRANSPOSE_CONV2D) { @@ -473,8 +503,18 @@ bool Int8Interpreter::prepare_operations() { int32_t* cursor = params_; size_t scratch_size = 0; + auto require_scratch = [&scratch_size](OpState& state, int32_t needed) { + if (needed <= 0) + return; + const uint32_t bytes = static_cast(needed); + if (bytes > state.scratch_bytes) + state.scratch_bytes = bytes; + if (bytes > scratch_size) + scratch_size = bytes; + }; for (uint32_t i = 0; i < op_count; ++i) { - const OperationRecord& op = operations_[i]; + OperationRecord op{}; + read_operation(graph_, i, op); OpState& state = op_states_[i]; state = {}; state.opcode = static_cast(op.opcode); @@ -556,8 +596,7 @@ bool Int8Interpreter::prepare_operations() { params.activation_min, params.activation_max}; const int needed = esp_nn_get_conv_scratch_size( &tile_in, &state.filter_dims, &tile_out, &state.conv); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } else { state.filter_dims = { params.kernel_w, params.kernel_h, output.shape[1], 1}; @@ -571,8 +610,7 @@ bool Int8Interpreter::prepare_operations() { params.activation_min, params.activation_max}; const int needed = esp_nn_get_depthwise_conv_scratch_size( &tile_in, &state.filter_dims, &tile_out, &state.dw); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } break; } @@ -591,8 +629,7 @@ bool Int8Interpreter::prepare_operations() { &state.filter_dims, &state.output_dims, &state.conv); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } else { state.filter_dims = { params.kernel_w, params.kernel_h, output.shape[1], 1}; @@ -608,8 +645,7 @@ bool Int8Interpreter::prepare_operations() { &state.filter_dims, &state.output_dims, &state.dw); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } break; } @@ -669,8 +705,7 @@ bool Int8Interpreter::prepare_operations() { state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; const int32_t needed = esp_nn_get_mean_scratch_size( input.shape[2], input.shape[3], input.shape[1]); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); break; } case OpCode::MAX_POOL2D: @@ -760,7 +795,7 @@ bool Int8Interpreter::prepare_operations() { return false; } } - scratch_size_ = scratch_size; + scratch_size_ = options_.scratch_in_arena ? 0 : scratch_size; if (scratch_size_ != 0) { scratch_ = allocate_preferring_internal( scratch_size_, true, options_.internal_reserve_bytes, nullptr); @@ -934,6 +969,13 @@ int32_t Int8Interpreter::output_argmax(size_t index) const { return best; } +size_t Int8Interpreter::table_bytes() const { + if (!loaded_) + return 0; + return graph_.header.num_operations * sizeof(OpState) + + graph_.header.num_tensors * (sizeof(TensorRecord) + sizeof(size_t)); +} + int64_t Int8Interpreter::op_time_us(uint32_t op) const { return loaded_ && op_us_ != nullptr && op < graph_.header.num_operations ? op_us_[op] @@ -1694,6 +1736,13 @@ bool Int8Interpreter::invoke() { int64_t op_started = invoke_started; int64_t last_yield = invoke_started; for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + const OpState& state = op_states_[i]; + if (options_.scratch_in_arena && state.scratch_bytes != 0) { + void* scratch = arena_ + state.scratch_offset; + esp_nn_set_conv_scratch_buf(scratch); + esp_nn_set_depthwise_conv_scratch_buf(scratch); + esp_nn_set_mean_scratch_buf(scratch); + } if (!run_operation(i)) { std::printf( "ERROR: operation %u (%s) failed\n", diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index af6b6c38453..abec03bdb4f 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -524,24 +524,39 @@ bool validate_int8_operation( return false; } +// Set of tensor IDs, so that validation keeps 128 bytes per set on the stack +// instead of a 1,024-byte bool array. +class TensorSet { + public: + bool operator[](uint32_t id) const { + return ((bits_[id / 32] >> (id % 32)) & 1U) != 0; + } + void set(uint32_t id) { + bits_[id / 32] |= 1U << (id % 32); + } + + private: + uint32_t bits_[MAX_GRAPH_TENSORS / 32]{}; +}; + bool validate_int8_dataflow(const GraphView& graph) { - bool produced[MAX_GRAPH_TENSORS]{}; - bool copy_produced[MAX_GRAPH_TENSORS]{}; + TensorSet produced; + TensorSet copy_produced; uint32_t input_count = 0; uint32_t output_count = 0; for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; read_tensor(graph, i, tensor); if ((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0) { - produced[i] = true; + produced.set(i); ++input_count; } if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) ++output_count; } if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { - bool listed_input[MAX_GRAPH_TENSORS]{}; - bool listed_output[MAX_GRAPH_TENSORS]{}; + TensorSet listed_input; + TensorSet listed_output; for (uint32_t i = 0; i < graph.num_inputs; ++i) { const uint32_t id = graph_input_id(graph, i); TensorRecord tensor{}; @@ -552,7 +567,7 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - listed_input[id] = true; + listed_input.set(id); } for (uint32_t i = 0; i < graph.num_outputs; ++i) { const uint32_t id = graph_output_id(graph, i); @@ -564,7 +579,7 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - listed_output[id] = true; + listed_output.set(id); } for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; @@ -608,8 +623,9 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - produced[id] = true; - copy_produced[id] = is_copy; + produced.set(id); + if (is_copy) + copy_produced.set(id); } } for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index 8d409524a78..f520d2f992b 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -22,7 +22,8 @@ * outputs must not be memory planned), the tensors are bound once to runner * buffers through set_input and set_output_data_ptr, and --alias makes an * input and an output share one buffer: that input is then read from its file - * at step 0 only and afterwards holds the output of the previous step. + * at step 0 only and afterwards holds the output of the previous step. After + * loading, the runner prints the delegates' memory as a STATS line. */ #include @@ -33,6 +34,7 @@ #include #include +#include #include #include #include @@ -214,6 +216,15 @@ int main(int argc, char** argv) { } } + const auto stats = executorch::backends::espressif::delegate_stats(); + std::printf( + "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu\n", + static_cast(stats.delegates), + stats.arena_bytes, + stats.scratch_bytes, + stats.parameter_bytes, + stats.table_bytes); + for (int step = 0; step < steps; ++step) { for (int k = 0; k < instance_count; ++k) { Instance& instance = instances[k]; diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 4ba43244b26..ce7dd7ee156 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -30,7 +30,10 @@ import torch from executorch.backends.espressif import to_espnn_edge -from executorch.backends.espressif.reference import ProgramEmulator +from executorch.backends.espressif.reference import ( + esp_nn_reference as reference, + ProgramEmulator, +) from executorch.backends.espressif.test import ( test_int8_lowering as lowering_cases, test_int8_ops as op_cases, @@ -78,8 +81,11 @@ def as_list(value): return list(value) if isinstance(value, (tuple, list)) else [value] -def run_host(tmp_path: Path, program: bytes, steps, *args) -> list[list[bytes]]: - """Runs one list of input tensors per step; returns raw outputs per step.""" +def run_host(tmp_path: Path, program: bytes, steps, *args): + """ + Runs one list of input tensors per step. Returns the output files per + instance and step, and the delegates' memory statistics. + """ (tmp_path / "program.pte").write_bytes(program) for step, inputs in enumerate(steps): for i, tensor in enumerate(inputs): @@ -93,13 +99,21 @@ def run_host(tmp_path: Path, program: bytes, steps, *args) -> list[list[bytes]]: text=True, ) assert result.returncode == 0, result.stdout + result.stderr + (stats_line,) = [ + line for line in result.stdout.splitlines() if line.startswith("STATS ") + ] + stats = { + key: int(value) + for key, value in (field.split("=") for field in stats_line.split()[1:]) + } instances = 1 if "--instances" in args: instances = int(args[args.index("--instances") + 1]) - return [ + outputs = [ [sorted(tmp_path.glob(f"out_{k}_{step}_*.bin")) for step in range(len(steps))] for k in range(instances) ] + return outputs, stats def assert_matches(paths: list[Path], expected: list[torch.Tensor]): @@ -126,17 +140,28 @@ def random_steps(shapes, count: int, seed: int = 11): def test_host_delegate_matches_reference(tmp_path, name, model, shapes): exported, program = lower(model, shapes) steps = random_steps(shapes, 2) - (outputs,) = run_host(tmp_path, program, steps) + (outputs,), stats = run_host(tmp_path, program, steps) emulator = ProgramEmulator(exported) for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs))) + # With ESP-NN scratch planned in the arena, the arena also holds scratch. + planned = sum( + reference.plan_activation_arena(delegate.graph)[0] + for delegate in emulator.delegates.values() + ) + assert stats["delegates"] == len(emulator.delegates) + if stats["scratch"]: + assert stats["arena"] == planned + else: + assert stats["arena"] >= planned def test_instances_run_independently(tmp_path): model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] exported, program = lower(model, shapes) steps = random_steps(shapes, 3) - first, second = run_host(tmp_path, program, steps, "--instances", 2) + (first, second), stats = run_host(tmp_path, program, steps, "--instances", 2) + assert stats["delegates"] == 2 emulator = ProgramEmulator(exported) for step, inputs in enumerate(steps): expected = as_list(emulator(*inputs)) @@ -148,7 +173,7 @@ def test_unplanned_io_shared_with_the_caller(tmp_path): model, shapes = op_cases.TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)] exported, program = lower(model, shapes, planned_io=False) steps = random_steps(shapes, 2) - (outputs,) = run_host(tmp_path, program, steps, "--share-io") + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io") emulator = ProgramEmulator(exported) for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs))) From 017793959de3fcd49da9e03b6b3fc5ae2a5ad536 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 12:30:21 -0800 Subject: [PATCH 04/25] Use XTS3 tables and requantization parameters in place The delegate copied every tensor record and every per-channel requantization multiplier and shift out of its graph into RAM, although the graph already sits in memory-mapped flash or in PSRAM. For a 32-operator streaming keyword-spotting model on ESP32-S3 the two copies took 4.4 KB of internal RAM. They were needed because XTS3 v6 places those tables at offsets that are not word-aligned: the 22-byte header and the input/output table leave the tensor records, and so everything after them, 2 bytes off a word boundary. XTS3 v7 is v6 with the input/output table padded to a 4-byte boundary and each operation's parameter block starting on one. The serializer now writes v7. On a little-endian target the runtime uses a v7 graph's tensor records and requantization arrays (and softmax tables) where they are, and still copies them for v5 and v6 graphs, which it keeps accepting. When the delegate copies the graph's constants into PSRAM, the in-place pointers follow the copy. A program exported with this version needs a runtime from the same version or later; the runtime rejects unknown graph versions at init with DelegateInvalidCompatibility. espnn_host_interpreter runs a graph on the host build of the interpreter from addresses of every alignment, and test_host_interpreter.py checks that v6 and v7 graphs give bit-exact results from all of them and that v7 graphs keep no copy of their requantization parameters. --- .ci/scripts/test_espressif_host.sh | 4 +- .../include/espnn/int8_interpreter.h | 4 +- .../espnn_runtime/include/espnn/xts3_graph.h | 10 +- .../espnn_runtime/src/int8_interpreter.cpp | 190 ++++++++++++++---- .../espnn_runtime/src/xts3_graph.cpp | 28 ++- backends/espressif/serialization/xts3.py | 17 +- backends/espressif/test/host/CMakeLists.txt | 12 +- .../test/host/espnn_host_interpreter.cpp | 108 ++++++++++ .../espressif/test/test_host_interpreter.py | 95 +++++++++ backends/espressif/test/test_host_runtime.py | 24 ++- .../espressif/test/test_xts3_parser_host.py | 2 + 11 files changed, 432 insertions(+), 62 deletions(-) create mode 100644 backends/espressif/test/host/espnn_host_interpreter.cpp create mode 100644 backends/espressif/test/test_host_interpreter.py diff --git a/.ci/scripts/test_espressif_host.sh b/.ci/scripts/test_espressif_host.sh index 2561adcebb4..0b66db45043 100755 --- a/.ci/scripts/test_espressif_host.sh +++ b/.ci/scripts/test_espressif_host.sh @@ -29,5 +29,7 @@ for variant in plain sanitized; do -DPYTHON_EXECUTABLE="$(command -v python)" "${options[@]}" cmake --build "${build_dir}/${variant}" ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \ - python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py + ESPNN_HOST_INTERPRETER="${build_dir}/${variant}/espnn_host_interpreter" \ + python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py \ + backends/espressif/test/test_host_interpreter.py done diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 2a046b69333..7881ca8dae6 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -227,7 +227,9 @@ class Int8Interpreter { GraphView graph_{}; bool loaded_ = false; uint8_t* blob_copy_ = nullptr; - TensorRecord* tensors_ = nullptr; + const TensorRecord* tensors_ = nullptr; + /// The graph's tensor table and requantization arrays are used in place. + bool tables_in_place_ = false; OpState* op_states_ = nullptr; size_t* tensor_offsets_ = nullptr; uint8_t* arena_ = nullptr; diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h index 4fac0173531..a275c626463 100644 --- a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -19,17 +19,21 @@ * (backends/espressif/serialization/xts3.py). All multi-byte fields are * little-endian. A graph is a header, an optional input/output table * (version 6), fixed-size tensor and operation records, a parameter region - * and a constant region. parse_graph() bounds-checks every region and record - * before any accessor reads it. + * and a constant region. Version 7 pads the input/output table to a 4-byte + * boundary, so that the records and the parameter region are word-aligned and + * the runtime can use them where they are. parse_graph() bounds-checks every + * region and record before any accessor reads it. */ namespace executorch { namespace backends { namespace espressif { -/// Graph versions accepted by this runtime; 6 adds the input/output table. +/// Graph versions accepted by this runtime; 6 adds the input/output table and +/// 7 aligns the tables that follow it. static constexpr uint32_t GRAPH_ESP_NN_INT8_VERSION = 5; static constexpr uint32_t GRAPH_ESP_NN_INT8_IO_VERSION = 6; +static constexpr uint32_t GRAPH_ESP_NN_INT8_ALIGNED_VERSION = 7; static constexpr size_t GRAPH_HEADER_SIZE = 22; static constexpr size_t GRAPH_IO_TABLE_HEADER_SIZE = 4; static constexpr size_t MAX_GRAPH_IO = 16; diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 0c6950cfeba..36f0c288d7a 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -56,6 +57,71 @@ class InvokeLock { SemaphoreHandle_t handle_; }; +// A version 7 graph keeps its tensor table and parameter region word-aligned. +// On a little-endian target its tensor records then have TensorRecord's +// layout, and per-channel multipliers and shifts are int32 arrays. +static_assert( + sizeof(TensorRecord) == TENSOR_RECORD_SIZE && + offsetof(TensorRecord, dtype) == 8 && + offsetof(TensorRecord, rank) == 10 && + offsetof(TensorRecord, shape) == 12 && + offsetof(TensorRecord, quant_scale) == 28 && + offsetof(TensorRecord, quant_zero_point) == 32, + "TensorRecord must have the XTS3 tensor record layout"); + +bool graph_tables_aligned(const GraphView& graph) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION && + (reinterpret_cast(graph.data) & 3U) == 0; +#else + return false; +#endif +} + +// An int32 array of the parameter region where it sits in the graph, or +// nullptr when it must be copied. +const int32_t* int32s_in_place(bool tables_in_place, const uint8_t* data) { + if (!tables_in_place || (reinterpret_cast(data) & 3U) != 0) + return nullptr; + return reinterpret_cast(data); +} + +// The per-channel requantization multipliers and shifts of a convolution or +// linear operation, little-endian int32 arrays in its parameter block. +bool requantization_arrays( + const GraphView& graph, + const TensorRecord* tensors, + const OperationRecord& op, + const uint8_t*& multipliers, + const uint8_t*& shifts, + uint32_t& count) { + const uint8_t* p = graph.parameter_data + op.parameter_offset; + switch (static_cast(op.opcode)) { + case OpCode::CONV2D: + case OpCode::DEPTHWISE_CONV2D: { + count = static_cast(tensors[op.outputs[0]].shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph, op, count, params)) + return false; + multipliers = params.multipliers; + shifts = params.shifts; + return true; + } + case OpCode::TRANSPOSE_CONV2D: + count = static_cast(tensors[op.outputs[0]].shape[1]); + multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + shifts = multipliers + 4 * count; + return true; + case OpCode::LINEAR: + count = static_cast(tensors[op.inputs[1]].shape[0]); + multipliers = p + INT8_PAIR_PARAMS_SIZE; + shifts = multipliers + 4 * count; + return true; + default: + return false; + } +} + size_t align_up(size_t value, size_t alignment) { return (value + alignment - 1) & ~(alignment - 1); } @@ -205,8 +271,9 @@ void Int8Interpreter::reset() { heap_caps_free(params_); if (blob_copy_ != nullptr) heap_caps_free(blob_copy_); - if (tensors_ != nullptr) - heap_caps_free(tensors_); + if (tensors_ != nullptr && !tables_in_place_) + heap_caps_free(const_cast(tensors_)); + tables_in_place_ = false; if (op_states_ != nullptr) heap_caps_free(op_states_); if (op_us_ != nullptr) @@ -242,10 +309,10 @@ bool Int8Interpreter::load( } if (!is_int8_graph_version(graph_.header.version)) { std::printf( - "ERROR: model is XTS3 v%u; the int8 runtime requires v%u or v%u\n", + "ERROR: model is XTS3 v%u; the int8 runtime requires v%u to v%u\n", static_cast(graph_.header.version), static_cast(GRAPH_ESP_NN_INT8_VERSION), - static_cast(GRAPH_ESP_NN_INT8_IO_VERSION)); + static_cast(GRAPH_ESP_NN_INT8_ALIGNED_VERSION)); return false; } const bool misaligned = @@ -268,8 +335,12 @@ bool Int8Interpreter::load( const uint32_t tensor_count = graph_.header.num_tensors; const uint32_t op_count = graph_.header.num_operations; - tensors_ = static_cast( - heap_caps_calloc(tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); + tables_in_place_ = graph_tables_aligned(graph_); + tensors_ = tables_in_place_ + ? reinterpret_cast( + graph_.data + graph_.tensor_table_offset) + : static_cast(heap_caps_calloc( + tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); op_states_ = static_cast( heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); tensor_offsets_ = static_cast( @@ -285,8 +356,11 @@ bool Int8Interpreter::load( reset(); return false; } - for (uint32_t i = 0; i < tensor_count; ++i) - read_tensor(graph_, i, tensors_[i]); + if (!tables_in_place_) { + TensorRecord* tensors = const_cast(tensors_); + for (uint32_t i = 0; i < tensor_count; ++i) + read_tensor(graph_, i, tensors[i]); + } input_count_ = graph_input_count(graph_); output_count_ = graph_output_count(graph_); if (input_count_ > MAX_GRAPH_IO || output_count_ > MAX_GRAPH_IO) { @@ -326,6 +400,10 @@ bool Int8Interpreter::load( return false; } rebase_constants(old_base, size); + if (tables_in_place_) { + tensors_ = reinterpret_cast( + graph_.data + graph_.tensor_table_offset); + } } } if (options_.dual_core) { @@ -479,15 +557,25 @@ bool Int8Interpreter::plan_memory() { bool Int8Interpreter::prepare_operations() { const uint32_t op_count = graph_.header.num_operations; + // Requantization parameters are used where they sit in a version 7 graph, + // and copied into params_ otherwise. size_t param_words = 0; for (uint32_t i = 0; i < op_count; ++i) { OperationRecord op{}; read_operation(graph_, i, op); - const OpCode code = static_cast(op.opcode); - if (code == OpCode::CONV2D || code == OpCode::DEPTHWISE_CONV2D || - code == OpCode::LINEAR || code == OpCode::TRANSPOSE_CONV2D) { - param_words += 2U * tensor_element_count(tensors_[op.inputs[2]]); - } else if (code == OpCode::SOFTMAX) { + const uint8_t* multipliers = nullptr; + const uint8_t* shifts = nullptr; + uint32_t channels = 0; + if (requantization_arrays( + graph_, tensors_, op, multipliers, shifts, channels)) { + if (int32s_in_place(tables_in_place_, multipliers) == nullptr || + int32s_in_place(tables_in_place_, shifts) == nullptr) + param_words += 2U * channels; + } else if ( + static_cast(op.opcode) == OpCode::SOFTMAX && + int32s_in_place( + tables_in_place_, graph_.parameter_data + op.parameter_offset) == + nullptr) { param_words += 256U; } } @@ -512,6 +600,31 @@ bool Int8Interpreter::prepare_operations() { if (bytes > scratch_size) scratch_size = bytes; }; + auto bind_requantization = [this, &cursor]( + OpState& state, const OperationRecord& op) { + const uint8_t* multipliers = nullptr; + const uint8_t* shifts = nullptr; + uint32_t channels = 0; + if (!requantization_arrays( + graph_, tensors_, op, multipliers, shifts, channels)) + return false; + const int32_t* multipliers_in_place = + int32s_in_place(tables_in_place_, multipliers); + const int32_t* shifts_in_place = int32s_in_place(tables_in_place_, shifts); + if (multipliers_in_place != nullptr && shifts_in_place != nullptr) { + state.multipliers = const_cast(multipliers_in_place); + state.shifts = const_cast(shifts_in_place); + return true; + } + state.multipliers = cursor; + state.shifts = cursor + channels; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(multipliers + c * 4); + state.shifts[c] = read_le_i32(shifts + c * 4); + } + cursor += 2U * channels; + return true; + }; for (uint32_t i = 0; i < op_count; ++i) { OperationRecord op{}; read_operation(graph_, i, op); @@ -541,13 +654,8 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, weight)); state.bias = reinterpret_cast( tensor_constant_data(graph_, bias)); - state.multipliers = cursor; - state.shifts = cursor + channels; - for (uint32_t c = 0; c < channels; ++c) { - state.multipliers[c] = read_le_i32(params.multipliers + c * 4); - state.shifts[c] = read_le_i32(params.shifts + c * 4); - } - cursor += 2U * channels; + if (!bind_requantization(state, op)) + return false; state.quant.mult = state.multipliers; state.quant.shift = state.shifts; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; @@ -674,7 +782,6 @@ bool Int8Interpreter::prepare_operations() { case OpCode::TRANSPOSE_CONV2D: { const TensorRecord& weight = tensors_[op.inputs[1]]; const TensorRecord& bias = tensors_[op.inputs[2]]; - const uint32_t channels = static_cast(output.shape[1]); state.weights = reinterpret_cast( tensor_constant_data(graph_, weight)); state.bias = reinterpret_cast( @@ -685,15 +792,8 @@ bool Int8Interpreter::prepare_operations() { state.dilation_w = read_le_i32(p + 36); state.value0 = read_le_i32(p + 44); state.value1 = read_le_i32(p + 48); - state.multipliers = cursor; - state.shifts = cursor + channels; - const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; - const uint8_t* shifts = multipliers + channels * 4U; - for (uint32_t c = 0; c < channels; ++c) { - state.multipliers[c] = read_le_i32(multipliers + c * 4); - state.shifts[c] = read_le_i32(shifts + c * 4); - } - cursor += 2U * channels; + if (!bind_requantization(state, op)) + return false; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; state.output_dims = { output.shape[3], output.shape[2], output.shape[1], 1}; @@ -733,10 +833,14 @@ bool Int8Interpreter::prepare_operations() { state.rows = static_cast(state.elements) / state.cols; break; case OpCode::SOFTMAX: - state.multipliers = cursor; - for (int k = 0; k < 256; ++k) - state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); - cursor += 256; + if (const int32_t* table = int32s_in_place(tables_in_place_, p)) { + state.multipliers = const_cast(table); + } else { + state.multipliers = cursor; + for (int k = 0; k < 256; ++k) + state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); + cursor += 256; + } state.cols = input.shape[1]; state.rows = static_cast(state.elements) / state.cols; break; @@ -768,16 +872,8 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, bias)); state.value0 = read_le_i32(p); state.value1 = read_le_i32(p + 4); - state.multipliers = cursor; - state.shifts = cursor + state.rows; - const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; - const uint8_t* shifts = - multipliers + static_cast(state.rows) * 4U; - for (int32_t r = 0; r < state.rows; ++r) { - state.multipliers[r] = read_le_i32(multipliers + r * 4); - state.shifts[r] = read_le_i32(shifts + r * 4); - } - cursor += 2 * state.rows; + if (!bind_requantization(state, op)) + return false; break; } default: @@ -825,6 +921,12 @@ void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { state.bias = reinterpret_cast(rebase(state.bias)); if (state.table != nullptr) state.table = rebase(state.table); + state.multipliers = const_cast( + reinterpret_cast(rebase(state.multipliers))); + state.shifts = const_cast( + reinterpret_cast(rebase(state.shifts))); + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; } } diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index abec03bdb4f..0dd4aa6de5d 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -42,7 +42,7 @@ bool valid_dtype(uint8_t dtype) { } bool valid_opcode(uint8_t opcode, uint32_t version) { - if (version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (version >= GRAPH_ESP_NN_INT8_IO_VERSION) { return opcode <= static_cast(OpCode::TRANSPOSE_CONV2D); } return opcode <= static_cast(OpCode::SOFTMAX); @@ -554,7 +554,7 @@ bool validate_int8_dataflow(const GraphView& graph) { if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) ++output_count; } - if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION) { TensorSet listed_input; TensorSet listed_output; for (uint32_t i = 0; i < graph.num_inputs; ++i) { @@ -653,7 +653,8 @@ int32_t read_le_i32(const uint8_t* p) { bool is_int8_graph_version(uint32_t version) { return version == GRAPH_ESP_NN_INT8_VERSION || - version == GRAPH_ESP_NN_INT8_IO_VERSION; + version == GRAPH_ESP_NN_INT8_IO_VERSION || + version == GRAPH_ESP_NN_INT8_ALIGNED_VERSION; } namespace { @@ -868,7 +869,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.data = data; graph.size = size; graph.tensor_table_offset = GRAPH_HEADER_SIZE; - if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION) { if (!range_valid(GRAPH_HEADER_SIZE, GRAPH_IO_TABLE_HEADER_SIZE, size)) return false; graph.num_inputs = read_u16(data + GRAPH_HEADER_SIZE); @@ -892,6 +893,21 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.tensor_table_offset = GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE + io_bytes; } + if (graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION) { + const size_t padded = (graph.tensor_table_offset + 3U) & ~size_t{3}; + if (!range_valid( + graph.tensor_table_offset, + padded - graph.tensor_table_offset, + size)) + return false; + for (size_t i = graph.tensor_table_offset; i < padded; ++i) { + if (data[i] != 0) { + std::printf("ERROR: graph IO table padding is not zero\n"); + return false; + } + } + graph.tensor_table_offset = padded; + } graph.tensor_record_size = TENSOR_RECORD_SIZE; const size_t tensor_bytes = static_cast(graph.header.num_tensors) * graph.tensor_record_size; @@ -953,7 +969,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; } const bool quantized_constant_allowed = - graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION && + graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION && tensor.dtype == static_cast(DataType::INT8); if ((is_constant ? ((tensor.dtype != static_cast(DataType::INT8) && @@ -990,6 +1006,8 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { for (uint32_t i = 0; i < graph.header.num_operations; ++i) { OperationRecord op{}; read_operation(graph, i, op); + if (graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION) + expected_parameter_offset = (expected_parameter_offset + 3U) & ~size_t{3}; if (op.op_id != (i & 0xFFU) || !valid_opcode(op.opcode, graph.header.version) || op.num_inputs == 0 || op.num_inputs > 6 || op.num_outputs == 0 || op.num_outputs > 2 || diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index 7ec71b95a1d..e16c545b6ef 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -19,8 +19,10 @@ ) MAGIC = b"XTS3" -XTS3_VERSION = 6 -XTS3_VERSIONS = (5, 6) +# Version 7 is version 6 with the IO table padded to 4 bytes, so that the +# runtime can use the tensor records and parameter blocks in place. +XTS3_VERSION = 7 +XTS3_VERSIONS = (5, 6, 7) IO_TABLE_HEADER = " bytes: parameter_data = bytearray() operation_data = bytearray() for operation in graph.operations: + if graph.version >= 7: + parameter_data += b"\x00" * (-len(parameter_data) % 4) operation_data += _serialize_operation(operation, len(parameter_data)) parameter_data += operation.params io_table = b"" @@ -1959,6 +1963,8 @@ def serialize_graph(graph: XTS3Graph) -> bytes: ) + struct.pack( f"<{len(graph.inputs) + len(graph.outputs)}I", *graph.inputs, *graph.outputs ) + if graph.version >= 7: + io_table += b"\x00" * ((-(HEADER_SIZE + len(io_table))) % 4) prefix = ( HEADER_SIZE + len(io_table) @@ -1987,7 +1993,7 @@ def serialize_graph(graph: XTS3Graph) -> bytes: def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 - """Parse and bounds-check an XTS3 v5 or v6 int8 graph.""" + """Parse and bounds-check an XTS3 v5, v6 or v7 int8 graph.""" if len(data) < HEADER_SIZE: raise ValueError("Graph blob is smaller than the header") magic, version, tensor_count, operation_count, parameter_size, constant_size = ( @@ -2012,6 +2018,11 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 ids = struct.unpack_from(f"<{input_count + output_count}I", data, offset) graph_inputs, graph_outputs = list(ids[:input_count]), list(ids[input_count:]) offset += 4 * (input_count + output_count) + if version >= 7: + padding = (-offset) % 4 + if data[offset : offset + padding] != b"\x00" * padding: + raise ValueError("Graph IO table padding is not zero") + offset += padding if offset + tensor_count * TENSOR_SIZE > len(data): raise ValueError("Graph tensor region exceeds blob") diff --git a/backends/espressif/test/host/CMakeLists.txt b/backends/espressif/test/host/CMakeLists.txt index 3b3a67aa8d8..b11108d466c 100644 --- a/backends/espressif/test/host/CMakeLists.txt +++ b/backends/espressif/test/host/CMakeLists.txt @@ -6,10 +6,11 @@ # # SPDX-License-Identifier: BSD-3-Clause -# Host build of the ESP-NN delegate and espnn_host_runner, for -# backends/espressif/test/test_host_runtime.py (see "On the host" in -# backends/espressif/README.md). -DESPNN_HOST_SANITIZE=ON builds everything with -# AddressSanitizer and UndefinedBehaviorSanitizer. +# Host build of the ESP-NN delegate, espnn_host_runner and +# espnn_host_interpreter, for backends/espressif/test/test_host_runtime.py and +# test_host_interpreter.py (see "On the host" in backends/espressif/README.md). +# -DESPNN_HOST_SANITIZE=ON builds everything with AddressSanitizer and +# UndefinedBehaviorSanitizer. cmake_minimum_required(VERSION 3.24) project(espnn_host_runner CXX C) @@ -59,3 +60,6 @@ target_include_directories(espnn_host_runner PRIVATE "${EXECUTORCH_ROOT}/..") target_link_libraries( espnn_host_runner PRIVATE executorch_core extension_data_loader espnn_backend ) + +add_executable(espnn_host_interpreter espnn_host_interpreter.cpp) +target_link_libraries(espnn_host_interpreter PRIVATE espnn_backend) diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp new file mode 100644 index 00000000000..b023c066253 --- /dev/null +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Runs one XTS3 graph on Int8Interpreter directly, for + * backends/espressif/test/test_host_interpreter.py: + * + * espnn_host_interpreter GRAPH OFFSET INPUT... -- OUTPUT... + * + * The graph is placed OFFSET bytes past a 16-byte boundary, so that a test can + * load it from addresses with any alignment. Inputs and outputs are raw + * device-layout (NHWC) int8 files. + */ + +#include +#include +#include +#include +#include +#include + +#include + +using executorch::backends::espressif::Int8Interpreter; +using executorch::backends::espressif::Int8TensorInfo; + +namespace { + +std::vector read_file(const char* path) { + std::vector data; + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) + return data; + uint8_t chunk[4096]; + size_t read = 0; + while ((read = std::fread(chunk, 1, sizeof(chunk), file)) > 0) + data.insert(data.end(), chunk, chunk + read); + std::fclose(file); + return data; +} + +int fail(const std::string& what) { + std::fprintf(stderr, "espnn_host_interpreter: %s\n", what.c_str()); + return 1; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 4) + return fail("usage: GRAPH OFFSET INPUT... -- OUTPUT..."); + const std::vector graph = read_file(argv[1]); + const size_t offset = std::strtoul(argv[2], nullptr, 10); + std::vector inputs; + std::vector outputs; + bool past_separator = false; + for (int a = 3; a < argc; ++a) { + if (std::strcmp(argv[a], "--") == 0) + past_separator = true; + else + (past_separator ? outputs : inputs).push_back(argv[a]); + } + + // A 16-byte aligned buffer, holding the graph at the requested offset. + std::vector storage((graph.size() + offset) / 8 + 3); + uint8_t* base = reinterpret_cast(storage.data()); + base += (16 - reinterpret_cast(base) % 16) % 16; + std::memcpy(base + offset, graph.data(), graph.size()); + + Int8Interpreter interpreter; + if (!interpreter.load(base + offset, graph.size())) + return fail("load failed"); + if (inputs.size() != interpreter.input_count() || + outputs.size() != interpreter.output_count()) + return fail("wrong number of inputs or outputs"); + for (size_t i = 0; i < inputs.size(); ++i) { + Int8TensorInfo info{}; + interpreter.input_info(i, info); + const std::vector data = read_file(inputs[i]); + if (data.size() != info.bytes) + return fail(std::string("wrong input size: ") + inputs[i]); + std::memcpy(interpreter.input_data(i), data.data(), data.size()); + } + if (!interpreter.invoke()) + return fail("invoke failed"); + for (size_t o = 0; o < outputs.size(); ++o) { + Int8TensorInfo info{}; + interpreter.output_info(o, info); + std::FILE* file = std::fopen(outputs[o], "wb"); + if (file == nullptr || + std::fwrite(interpreter.output_data(o), 1, info.bytes, file) != + info.bytes) + return fail(std::string("cannot write ") + outputs[o]); + std::fclose(file); + } + std::printf( + "STATS parameters=%zu blob_copied=%d\n", + interpreter.parameter_bytes(), + interpreter.constants_copied() ? 1 : 0); + return 0; +} diff --git a/backends/espressif/test/test_host_interpreter.py b/backends/espressif/test/test_host_interpreter.py new file mode 100644 index 00000000000..d43c53289f3 --- /dev/null +++ b/backends/espressif/test/test_host_interpreter.py @@ -0,0 +1,95 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Runs XTS3 graphs on the host build of Int8Interpreter from addresses of every +alignment and checks the outputs against the bit-exact reference interpreter. +Skipped unless ESPNN_HOST_INTERPRETER points to espnn_host_interpreter; see +backends/espressif/test/host/CMakeLists.txt. + + ESPNN_HOST_INTERPRETER=cmake-out-espnn-host/espnn_host_interpreter \ + pytest backends/espressif/test/test_host_interpreter.py +""" + +from __future__ import annotations + +import os +import subprocess + +import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization import xts3 +from executorch.backends.espressif.test import ( + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) + +INTERPRETER = os.environ.get("ESPNN_HOST_INTERPRETER") +pytestmark = pytest.mark.skipif( + not INTERPRETER, reason="Set ESPNN_HOST_INTERPRETER to espnn_host_interpreter" +) + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def run(tmp_path, graph: xts3.XTS3Graph, offset: int, inputs) -> tuple[list, dict]: + blob = tmp_path / "graph.xts3" + blob.write_bytes(xts3.serialize_graph(graph)) + input_paths, output_paths = [], [] + for i, array in enumerate(inputs): + path = tmp_path / f"in_{i}.bin" + path.write_bytes(array.astype(np.int8).tobytes()) + input_paths.append(str(path)) + for o in range(len(graph.outputs)): + output_paths.append(str(tmp_path / f"out_{offset}_{o}.bin")) + result = subprocess.run( + [INTERPRETER, str(blob), str(offset), *input_paths, "--", *output_paths], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + (stats,) = [line for line in result.stdout.splitlines() if line.startswith("STATS")] + outputs = [ + np.frombuffer(open(path, "rb").read(), dtype=np.int8) for path in output_paths + ] + return outputs, dict(field.split("=") for field in stats.split()[1:]) + + +@pytest.mark.parametrize("version", [6, 7]) +def test_graph_at_any_alignment(tmp_path, version): + _, program = op_cases.quantize(lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]) + graph = xts3.build_graph(program) + graph.version = version + graph = xts3.parse_graph(xts3.serialize_graph(graph)) + interpreter = reference.ReferenceInterpreter(graph) + generator = np.random.default_rng(0) + inputs = [ + generator.integers(-128, 128, reference.device_shape(graph.tensors[i])) + for i in interpreter.inputs + ] + expected = [ + value.astype(np.int8).reshape(-1) for value in interpreter.run_outputs(inputs) + ] + for offset in (0, 1, 2, 3, 4, 8): + outputs, stats = run(tmp_path, graph, offset, inputs) + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + # A misaligned graph is copied to aligned memory first. + assert stats["blob_copied"] == ("0" if offset == 0 else "1") + # Version 7 graphs keep their requantization parameters in place. + assert (stats["parameters"] == "0") == (version == 7) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index ce7dd7ee156..ac481d8330e 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -34,11 +34,13 @@ esp_nn_reference as reference, ProgramEmulator, ) +from executorch.backends.espressif.serialization import xts3 from executorch.backends.espressif.test import ( test_int8_lowering as lowering_cases, test_int8_ops as op_cases, ) from executorch.exir import ExecutorchBackendConfig +from executorch.exir.lowered_backend_module import LoweredBackendModule from executorch.exir.passes import MemoryPlanningPass RUNNER = os.environ.get("ESPNN_HOST_RUNNER") @@ -65,10 +67,16 @@ def disable_grad(): yield -def lower(model, shapes, planned_io: bool = True): +def lower(model, shapes, planned_io: bool = True, xts3_version: int | None = None): _, program = op_cases.quantize(model, shapes) edge, report = to_espnn_edge(program, allow_cpu_fallback=False) assert not report["cpu_ops"] + if xts3_version is not None: + for _, module in edge.exported_program().graph_module.named_children(): + if isinstance(module, LoweredBackendModule): + graph = xts3.parse_graph(module.processed_bytes) + graph.version = xts3_version + module._processed_bytes = xts3.serialize_graph(graph) config = ExecutorchBackendConfig( memory_planning_pass=MemoryPlanningPass( alloc_graph_input=planned_io, alloc_graph_output=planned_io @@ -150,6 +158,8 @@ def test_host_delegate_matches_reference(tmp_path, name, model, shapes): for delegate in emulator.delegates.values() ) assert stats["delegates"] == len(emulator.delegates) + # Version 7 graphs keep every requantization table in place. + assert stats["parameters"] == 0 if stats["scratch"]: assert stats["arena"] == planned else: @@ -177,3 +187,15 @@ def test_unplanned_io_shared_with_the_caller(tmp_path): emulator = ProgramEmulator(exported) for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs))) + + +def test_version_6_graphs_still_run(tmp_path): + model, shapes = lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] + exported, program = lower(model, shapes, xts3_version=6) + steps = random_steps(shapes, 2) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert all(d.graph.version == 6 for d in emulator.delegates.values()) + assert stats["parameters"] > 0 diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py index 00077357b9a..52022281a70 100644 --- a/backends/espressif/test/test_xts3_parser_host.py +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -89,6 +89,8 @@ def _layout(graph: xts3.XTS3Graph): len(graph.inputs) + len(graph.outputs) ) tensors = xts3.HEADER_SIZE + io_table + if graph.version >= 7: + tensors += -tensors % 4 operations = tensors + xts3.TENSOR_SIZE * len(graph.tensors) parameters = operations + xts3.OP_SIZE * len(graph.operations) offsets, cursor = [], parameters From f7e0a3de36d3ff6eab7c95ed1f9a52aac4ecd075 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 12:51:05 -0800 Subject: [PATCH 05/25] Use ExecuTorch tensors in place for ESP-NN delegate inputs and outputs The delegate copied every input into its activation arena before running and every output out of it afterwards, transposing rank-4 tensors between NCHW and its NHWC device layout element by element. For a streaming keyword-spotting model on ESP32-S3 whose state tensors cross the delegate boundary on every step, that was both the largest share of the delegate's non-kernel time and the reason the arena held a copy of every input and output. With Int8RuntimeOptions::external_io, which EspNNBackend now sets, the planner leaves graph inputs and outputs out of the arena and the caller binds a buffer to each before invoke(). The backend binds the ExecuTorch tensor itself when its memory is 16-byte aligned, like the arena, and already in the device layout: an int8 tensor that is not rank 4, has one channel or one pixel, or has a channels-last dim order. Any other input or output (float, NCHW or misaligned) goes through a zero-initialized staging buffer that the first execute() needing one allocates, which keeps existing programs working unchanged. Because operators may read an input after writing an output, an output whose memory overlaps an input is always staged and copied out after the last operator; this makes binding one buffer as both an input and an output (an application updating a state tensor in place) safe instead of depending on the order of the delegate's copies. A graph tensor that the delegate returns as several outputs (when its value feeds several CPU operators) is written once and copied to the other outputs, and a graph input that is also returned as an output is staged and copied from there. The first COPY writing an output bound to caller memory, unless it fills it, clears it first, so the buffer's previous contents never show through; a long cat is written by several COPY operations. Pooling and convolution outputs wider or taller than 65535, which ESP-NN's kernels would truncate to uint16_t and so leave unwritten, are now rejected at load. A graph whose every activation is an input or output needs no arena at all. The interpreter keeps a pointer per tensor instead of an arena offset, so binding costs nothing per operator. plan_activation_arena() mirrors the external-I/O plan, the README describes how to export int8 channels-last I/O that is used in place, and the tests cover channels-last I/O without staging, misaligned caller buffers, an output bound to the memory of an input that operators read after the output is written, an input returned as an output (also with another output bound over it), a partial COPY into a bound output, a seven-way cat as an output, those limits, a graph with no arena, and, with the portable and quantized kernels now linked into the host runner, the programs that leave operators on the CPU between delegates. --- backends/espressif/README.md | 55 +++- .../include/espnn/espnn_backend.h | 3 + .../include/espnn/int8_interpreter.h | 24 +- .../espnn_runtime/src/espnn_backend.cpp | 214 ++++++++++++-- .../espnn_runtime/src/int8_interpreter.cpp | 85 +++++- .../espnn_runtime/src/xts3_graph.cpp | 7 +- .../espressif/reference/esp_nn_reference.py | 17 +- backends/espressif/test/host/CMakeLists.txt | 6 +- .../test/host/espnn_host_interpreter.cpp | 27 +- .../espressif/test/host/espnn_host_runner.cpp | 67 +++-- .../espressif/test/test_host_interpreter.py | 68 ++++- backends/espressif/test/test_host_runtime.py | 273 +++++++++++++++++- .../espressif/test/test_xts3_parser_host.py | 54 ++++ 13 files changed, 797 insertions(+), 103 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 810ca275cda..e9f8013a0a2 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -100,6 +100,49 @@ wrote mobilenet_v2_espnn.pte Calibrate on a few hundred real samples; random data gives useless scales. +### Inputs and outputs + +The delegate keeps rank-4 tensors in NHWC order. It reads an int8 input and +writes an int8 output directly in the ExecuTorch tensor's memory when that +memory is 16-byte aligned and already in this layout: the tensor is not rank +4, has one channel or one pixel, or has a channels-last dim order. Anything +else (float, NCHW with several channels and pixels, or a misaligned buffer) +goes through a delegate-owned staging buffer and is quantized or transposed +on the way. The staging buffer is allocated by the first `execute()` that +needs it. + +For the smallest and fastest program, give the delegate int8, channels-last +inputs and outputs and leave them out of ExecuTorch's memory plan, so that +the application's own buffers are used: + +```python +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass +from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output + +# Example inputs (and calibration data) in channels-last memory format. An +# output that PyTorch would make contiguous, such as a slice of a cat, needs +# .clone(memory_format=torch.channels_last) at the end of forward(). +example_inputs = (torch.randn(1, 3, 96, 96).to(memory_format=torch.channels_last),) +# ... quantize and calibrate as above ... +program = torch.export.export(quantized, example_inputs) +quantize_input(program, 0) +quantize_output(program, 0) +edge, report = to_espnn_edge(program) +config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) +) +pte = edge.to_executorch(config).buffer +``` + +The application then binds its buffers once, with `Method::set_input()` and +`Method::set_output_data_ptr()`, and each step is a call to +`Method::execute()`. An output may share memory with an input (for example, +to update a state tensor in place): the delegate detects the overlap and +writes that output through its staging buffer after all operators have run. + ## Running on the ESP32-S3 You need ESP-IDF and an activated ESP-IDF shell. @@ -212,9 +255,10 @@ retain the bundled fixtures, SDK configuration, and resolved component versions. The delegate also builds outside ESP-IDF, for Linux and macOS, over ESP-NN's portable C kernels. `host/include` stands in for the few ESP-IDF APIs it uses, and ESP-NN is fetched from GitHub (or taken from `-DESPNN_LOCAL_PATH=...`). -`test_host_runtime.py` runs fully delegated programs through the ExecuTorch -runtime and this build, and checks every output against the host reference, -with no board or emulator: +`test_host_runtime.py` runs programs through the ExecuTorch runtime, this +build and the portable kernels, and checks every output against the host +reference (bit-exactly for fully delegated programs), with no board or +emulator: ```bash cmake -S backends/espressif/test/host -B cmake-out-espnn-host -G Ninja @@ -247,8 +291,9 @@ lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" - The delegate handle comes from ExecuTorch's runtime allocator, but each delegate allocates its activation arena, scratch buffers and optional PSRAM weight copy with ESP-IDF's `heap_caps` allocator at `init()`, so it can - choose between internal RAM and PSRAM. Nothing is allocated during - `execute()`. + choose between internal RAM and PSRAM. After that, only the first + `execute()` that needs a staging buffer (see "Inputs and outputs") + allocates. ## License diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index 5c8810683b1..b966fe61c98 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -38,6 +38,9 @@ struct DelegateStats { size_t scratch_bytes; size_t parameter_bytes; size_t table_bytes; + /// Sum of the buffers that delegates allocated on their first execute() + /// for inputs and outputs they cannot use in place. + size_t staging_bytes; }; /** diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 7881ca8dae6..f1d79513600 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -49,6 +49,11 @@ struct Int8RuntimeOptions { /// operation. Saves memory, but the scratch then lives wherever the arena /// does (PSRAM for a large model). bool scratch_in_arena = false; + /// Leave graph inputs and outputs out of the activation arena: the caller + /// binds a device-layout (NHWC) buffer to each of them with bind_input() + /// and bind_output() before invoke(), and the operations read and write + /// those buffers directly. + bool external_io = false; }; /** @@ -123,6 +128,21 @@ class Int8Interpreter { bool input_info(size_t index, Int8TensorInfo& info) const; /// Describes output @p index. @retval false @p index is out of range. bool output_info(size_t index, Int8TensorInfo& info) const; + /** + * With Int8RuntimeOptions::external_io, makes @p data, a device-layout + * buffer of input_info().bytes, the storage of input @p index until it is + * bound again. invoke() only reads it. + * + * @retval false external_io is off or @p index is out of range. + */ + bool bind_input(size_t index, const int8_t* data); + /** + * Same for output @p index. invoke() writes the buffer while it may still + * read the inputs, so it must not overlap a bound input. An output that is + * also a graph input cannot be bound: output_data() returns the input's + * buffer. + */ + bool bind_output(size_t index, int8_t* data); /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. int8_t* input_data(size_t index); /// Device-layout (NHWC) int8 buffer of output @p index, or nullptr. @@ -184,7 +204,7 @@ class Int8Interpreter { size_t scratch_bytes() const { return scratch_size_; } - /// Size of the per-operation and per-tensor tables kept after load(). + /// Size of the per-operation and per-tensor tables kept in RAM after load(). size_t table_bytes() const; /// Size of the precomputed requantization tables. size_t parameter_bytes() const { @@ -231,7 +251,7 @@ class Int8Interpreter { /// The graph's tensor table and requantization arrays are used in place. bool tables_in_place_ = false; OpState* op_states_ = nullptr; - size_t* tensor_offsets_ = nullptr; + int8_t** tensor_data_ = nullptr; uint8_t* arena_ = nullptr; size_t arena_size_ = 0; bool arena_internal_ = false; diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index b9876f7a59c..dc88b74ca77 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -73,29 +74,54 @@ int8_t quantize(float value, float inverse_scale, int32_t zero_point) { return static_cast(q); } -bool same_layout(const Tensor& tensor, const Int8TensorInfo& info) { +// How an ExecuTorch tensor's bytes relate to the delegate's device layout +// (NHWC for rank 4). kDevice: identical, because the tensor is not rank 4, its +// transpose is trivial (one channel, or one pixel), or it is channels-last. +// kNchw: a contiguous rank-4 tensor that needs transposing. +enum class Layout : uint8_t { kInvalid, kNchw, kDevice }; + +Layout tensor_layout(const Tensor& tensor, const Int8TensorInfo& info) { if (tensor.dim() != info.rank || static_cast(tensor.numel()) != info.bytes) - return false; + return Layout::kInvalid; for (uint8_t d = 0; d < info.rank; ++d) { if (tensor.size(d) != info.shape[d]) - return false; + return Layout::kInvalid; + } + const auto dim_order = tensor.dim_order(); + if (executorch::runtime::is_contiguous_dim_order( + dim_order.data(), dim_order.size())) { + const bool trivial = info.rank != 4 || info.channels == 1 || + static_cast(info.height) * info.width == 1; + return trivial ? Layout::kDevice : Layout::kNchw; } - return executorch::runtime::is_contiguous_dim_order( - tensor.dim_order().data(), tensor.dim_order().size()); + if (info.rank == 4 && + executorch::runtime::is_channels_last_dim_order( + dim_order.data(), dim_order.size())) + return Layout::kDevice; + return Layout::kInvalid; +} + +// The kernels may rely on the activation arena's 16-byte alignment, so only +// buffers aligned like it are used in place. +bool arena_aligned(const void* data) { + return reinterpret_cast(data) % 16 == 0; +} + +bool overlaps(const void* a, size_t a_bytes, const void* b, size_t b_bytes) { + const uintptr_t a_begin = reinterpret_cast(a); + const uintptr_t b_begin = reinterpret_cast(b); + return a_begin < b_begin + b_bytes && b_begin < a_begin + a_bytes; } bool load_input( - Int8Interpreter& interpreter, - size_t index, - const Tensor& tensor) { - Int8TensorInfo info{}; - if (!interpreter.input_info(index, info) || !same_layout(tensor, info)) - return false; - int8_t* destination = interpreter.input_data(index); + const Int8TensorInfo& info, + Layout layout, + const Tensor& tensor, + int8_t* destination) { const size_t plane = static_cast(info.height) * info.width; const size_t channels = static_cast(info.channels); - const bool nchw = info.rank == 4; + const bool nchw = layout == Layout::kNchw; if (tensor.scalar_type() == ScalarType::Float) { const float* source = tensor.const_data_ptr(); const float inverse = @@ -129,16 +155,13 @@ bool load_input( } bool store_output( - const Int8Interpreter& interpreter, - size_t index, + const Int8TensorInfo& info, + Layout layout, + const int8_t* source, Tensor& tensor) { - Int8TensorInfo info{}; - if (!interpreter.output_info(index, info) || !same_layout(tensor, info)) - return false; - const int8_t* source = interpreter.output_data(index); const size_t plane = static_cast(info.height) * info.width; const size_t channels = static_cast(info.channels); - const bool nchw = info.rank == 4; + const bool nchw = layout == Layout::kNchw; if (tensor.scalar_type() == ScalarType::Float) { float* destination = tensor.mutable_data_ptr(); if (!nchw) { @@ -160,7 +183,8 @@ bool store_output( if (tensor.scalar_type() == ScalarType::Char) { int8_t* destination = tensor.mutable_data_ptr(); if (!nchw) { - std::memcpy(destination, source, info.bytes); + if (destination != source) + std::memcpy(destination, source, info.bytes); return true; } for (size_t c = 0; c < channels; ++c) { @@ -175,6 +199,52 @@ bool store_output( struct DelegateState { Int8Interpreter interpreter; bool profiled = false; + // Device-layout copies of the inputs and outputs whose tensors the + // interpreter cannot use directly (float, NCHW, misaligned, an input that is + // also an output, or an output that overlaps an input), one slot per input + // then per output. Allocated, zeroed, by the first execute() that needs one. + int8_t* staging = nullptr; + + ~DelegateState() { + if (staging != nullptr) + heap_caps_free(staging); + } + + int8_t* staging_slot(size_t slot) { + const size_t inputs = interpreter.input_count(); + const size_t slots = inputs + interpreter.output_count(); + size_t offset = 0; + size_t total = 0; + for (size_t k = 0; k < slots; ++k) { + Int8TensorInfo info{}; + if (k < inputs ? !interpreter.input_info(k, info) + : !interpreter.output_info(k - inputs, info)) + return nullptr; + if (k == slot) + offset = total; + const size_t bytes = (info.bytes + 15) & ~static_cast(15); + if (info.bytes > SIZE_MAX - 15 || bytes > SIZE_MAX - total) + return nullptr; + total += bytes; + } + if (staging == nullptr) { + staging = static_cast(heap_caps_aligned_alloc( + 16, total, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); + if (staging == nullptr) { + staging = static_cast( + heap_caps_aligned_alloc(16, total, MALLOC_CAP_8BIT)); + } + if (staging == nullptr) { + std::printf( + "EspNN delegate: cannot allocate %u staging bytes\n", + static_cast(total)); + return nullptr; + } + std::memset(staging, 0, total); + g_stats.staging_bytes += total; + } + return staging + offset; + } }; class EspNNBackendInterface final : public BackendInterface { @@ -193,6 +263,7 @@ class EspNNBackendInterface final : public BackendInterface { return Error::MemoryAllocationFailed; DelegateState* state = new (memory) DelegateState(); Int8RuntimeOptions options{}; + options.external_io = true; #if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM options.copy_constants_to_psram = true; #endif @@ -238,28 +309,113 @@ class EspNNBackendInterface final : public BackendInterface { return Error::InvalidArgument; } const int64_t started = esp_timer_get_time(); + // An aligned int8 tensor already in the device layout is used in place; + // any other goes through a staging slot. + Int8TensorInfo info{}; for (size_t i = 0; i < inputs; ++i) { - if (!args[i]->isTensor() || - !load_input(interpreter, i, args[i]->toTensor())) { + const Layout layout = + args[i]->isTensor() && interpreter.input_info(i, info) + ? tensor_layout(args[i]->toTensor(), info) + : Layout::kInvalid; + bool ok = layout != Layout::kInvalid; + if (ok) { + const Tensor& tensor = args[i]->toTensor(); + // An input that is also returned as an output is staged, so that + // outputs written over the caller's input buffers cannot change it. + bool returned = false; + for (size_t o = 0; o < outputs; ++o) { + Int8TensorInfo output{}; + interpreter.output_info(o, output); + returned = returned || output.tensor_id == info.tensor_id; + } + if (tensor.scalar_type() == ScalarType::Char && + layout == Layout::kDevice && + arena_aligned(tensor.const_data_ptr()) && !returned) { + ok = interpreter.bind_input(i, tensor.const_data_ptr()); + } else { + int8_t* slot = state->staging_slot(i); + if (slot == nullptr) + return Error::MemoryAllocationFailed; + ok = load_input(info, layout, tensor, slot) && + interpreter.bind_input(i, slot); + } + } + if (!ok) { std::printf( "EspNN delegate: input %u has an unsupported dtype, shape or dim order\n", static_cast(i)); return Error::InvalidArgument; } } - if (!interpreter.invoke()) - return Error::Internal; + uint32_t staged_outputs = 0; for (size_t o = 0; o < outputs; ++o) { - if (!args[inputs + o]->isTensor()) - return Error::InvalidArgument; - Tensor tensor = args[inputs + o]->toTensor(); - if (!store_output(interpreter, o, tensor)) { + const Layout layout = + args[inputs + o]->isTensor() && interpreter.output_info(o, info) + ? tensor_layout(args[inputs + o]->toTensor(), info) + : Layout::kInvalid; + bool ok = layout != Layout::kInvalid; + // A graph tensor returned more than once is written where its first + // output is bound, and a graph input returned as an output stays where + // the input is bound; both are copied to their outputs afterwards. + bool repeated = false; + for (size_t earlier = 0; ok && earlier < o; ++earlier) { + Int8TensorInfo first{}; + interpreter.output_info(earlier, first); + repeated = repeated || first.tensor_id == info.tensor_id; + } + for (size_t i = 0; ok && i < inputs; ++i) { + Int8TensorInfo input{}; + interpreter.input_info(i, input); + repeated = repeated || input.tensor_id == info.tensor_id; + } + if (ok && repeated) { + staged_outputs |= 1U << o; + } else if (ok) { + Tensor tensor = args[inputs + o]->toTensor(); + // The operations may still read an input after writing an output, so + // an output that shares memory with an input is staged. + bool direct = tensor.scalar_type() == ScalarType::Char && + layout == Layout::kDevice && arena_aligned(tensor.const_data_ptr()); + for (size_t i = 0; direct && i < inputs; ++i) { + Int8TensorInfo input{}; + interpreter.input_info(i, input); + direct = !overlaps( + tensor.const_data_ptr(), + info.bytes, + interpreter.input_data(i), + input.bytes); + } + if (direct) { + ok = interpreter.bind_output(o, tensor.mutable_data_ptr()); + } else { + int8_t* slot = state->staging_slot(inputs + o); + if (slot == nullptr) + return Error::MemoryAllocationFailed; + ok = interpreter.bind_output(o, slot); + staged_outputs |= 1U << o; + } + } + if (!ok) { std::printf( "EspNN delegate: output %u has an unsupported dtype, shape or dim order\n", static_cast(o)); return Error::InvalidArgument; } } + if (!interpreter.invoke()) + return Error::Internal; + for (size_t o = 0; o < outputs; ++o) { + if ((staged_outputs & (1U << o)) == 0) + continue; + Tensor tensor = args[inputs + o]->toTensor(); + interpreter.output_info(o, info); + if (!store_output( + info, + tensor_layout(tensor, info), + interpreter.output_data(o), + tensor)) + return Error::InvalidArgument; + } ++g_stats.calls; g_stats.total_us += esp_timer_get_time() - started; #if CONFIG_ESPNN_DELEGATE_PROFILE diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 36f0c288d7a..1e8418661a3 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -220,6 +220,8 @@ struct Int8Interpreter::OpState { uint32_t input1; uint32_t output; uint8_t input_count; + // COPY: the first writer of an output in caller memory clears it first. + bool clear_output; union { uint32_t inputs[6]; int32_t pool[6]; @@ -278,8 +280,8 @@ void Int8Interpreter::reset() { heap_caps_free(op_states_); if (op_us_ != nullptr) heap_caps_free(op_us_); - if (tensor_offsets_ != nullptr) - heap_caps_free(tensor_offsets_); + if (tensor_data_ != nullptr) + heap_caps_free(tensor_data_); arena_ = nullptr; scratch_ = nullptr; params_ = nullptr; @@ -287,7 +289,7 @@ void Int8Interpreter::reset() { tensors_ = nullptr; op_states_ = nullptr; op_us_ = nullptr; - tensor_offsets_ = nullptr; + tensor_data_ = nullptr; arena_size_ = 0; scratch_size_ = 0; params_size_ = 0; @@ -343,14 +345,13 @@ bool Int8Interpreter::load( tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); op_states_ = static_cast( heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); - tensor_offsets_ = static_cast( - heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + tensor_data_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(int8_t*), MALLOC_CAP_8BIT)); if (options_.profile_operations) { op_us_ = static_cast( heap_caps_calloc(op_count, sizeof(int64_t), MALLOC_CAP_8BIT)); } - if (tensors_ == nullptr || op_states_ == nullptr || - tensor_offsets_ == nullptr || + if (tensors_ == nullptr || op_states_ == nullptr || tensor_data_ == nullptr || (options_.profile_operations && op_us_ == nullptr)) { std::printf("ERROR: unable to allocate interpreter tables\n"); reset(); @@ -436,7 +437,7 @@ bool Int8Interpreter::plan_memory() { bool* planned = static_cast( heap_caps_calloc(entries, sizeof(bool), MALLOC_CAP_8BIT)); struct PlanBuffers { - void* buffers[5]; + void* buffers[4]; void release() { for (void*& buffer : buffers) { if (buffer != nullptr) @@ -447,19 +448,29 @@ bool Int8Interpreter::plan_memory() { ~PlanBuffers() { release(); } - } plan_buffers{{first, last, bytes, offsets, planned}}; + } plan_buffers{{first, last, bytes, planned}}; + struct OffsetBuffer { + size_t* offsets; + ~OffsetBuffer() { + heap_caps_free(offsets); + } + } offset_buffer{offsets}; if (first == nullptr || last == nullptr || bytes == nullptr || offsets == nullptr || planned == nullptr) { std::printf("ERROR: unable to allocate memory-planner tables\n"); return false; } + // With external_io, graph inputs and outputs live in the caller's buffers. + const uint8_t unplanned = TENSOR_FLAG_CONSTANT | + (options_.external_io ? TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT + : 0); for (uint32_t i = 0; i < tensor_count; ++i) { first[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_INPUT) != 0 ? -1 : INT32_MAX; last[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); - planned[i] = (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0; + planned[i] = (tensors_[i].flags & unplanned) != 0; offsets[i] = SIZE_MAX; } for (uint32_t e = tensor_count; e < entries; ++e) { @@ -523,9 +534,7 @@ bool Int8Interpreter::plan_memory() { arena_size = candidate + bytes[best]; } for (uint32_t i = 0; i < tensor_count; ++i) { - tensor_offsets_[i] = offsets[i]; - if ((tensors_[i].flags & TENSOR_FLAG_CONSTANT) == 0 && - offsets[i] == SIZE_MAX) { + if ((tensors_[i].flags & unplanned) == 0 && offsets[i] == SIZE_MAX) { std::printf( "ERROR: tensor %u is never produced or consumed\n", static_cast(i)); @@ -536,10 +545,14 @@ bool Int8Interpreter::plan_memory() { op_states_[e - tensor_count].scratch_offset = offsets[e] == SIZE_MAX ? 0 : static_cast(offsets[e]); } - // Loading then never needs the planner's tables and the arena at once. + // Loading then needs at most the tensor offsets alongside the arena. plan_buffers.release(); + // With external_io, a graph whose every activation is an input or output + // needs no arena. arena_size_ = arena_size; + if (arena_size_ == 0) + return true; arena_ = static_cast(allocate_preferring_internal( arena_size_, options_.prefer_internal_arena, @@ -552,6 +565,10 @@ bool Int8Interpreter::plan_memory() { return false; } std::memset(arena_, 0, arena_size_); + for (uint32_t i = 0; i < tensor_count; ++i) { + if (offsets[i] != SIZE_MAX) + tensor_data_[i] = reinterpret_cast(arena_ + offsets[i]); + } return true; } @@ -771,6 +788,12 @@ bool Int8Interpreter::prepare_operations() { for (uint8_t k = 0; k < op.num_inputs; ++k) state.inputs[k] = op.inputs[k]; state.table = p; + // Several COPY operations may write parts of one output (a long cat). + state.clear_output = options_.external_io && + (output.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && + (read_le_i32(p + 4) & COPY_FLAG_FILL) == 0; + for (uint32_t j = 0; state.clear_output && j < i; ++j) + state.clear_output = op_states_[j].output != state.output; break; case OpCode::UPSAMPLE: state.table = p; @@ -931,7 +954,22 @@ void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { } int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { - return reinterpret_cast(arena_ + tensor_offsets_[tensor_id]); + return tensor_data_[tensor_id]; +} + +bool Int8Interpreter::bind_input(size_t index, const int8_t* data) { + if (!loaded_ || !options_.external_io || index >= input_count_) + return false; + tensor_data_[inputs_[index]] = const_cast(data); + return true; +} + +bool Int8Interpreter::bind_output(size_t index, int8_t* data) { + if (!loaded_ || !options_.external_io || index >= output_count_ || + (tensors_[outputs_[index]].flags & TENSOR_FLAG_GRAPH_INPUT) != 0) + return false; + tensor_data_[outputs_[index]] = data; + return true; } const int8_t* Int8Interpreter::operand(uint32_t tensor_id) const { @@ -1074,8 +1112,9 @@ int32_t Int8Interpreter::output_argmax(size_t index) const { size_t Int8Interpreter::table_bytes() const { if (!loaded_) return 0; + const size_t records = tables_in_place_ ? 0 : sizeof(TensorRecord); return graph_.header.num_operations * sizeof(OpState) + - graph_.header.num_tensors * (sizeof(TensorRecord) + sizeof(size_t)); + graph_.header.num_tensors * (records + sizeof(int8_t*)); } int64_t Int8Interpreter::op_time_us(uint32_t op) const { @@ -1675,6 +1714,10 @@ void Int8Interpreter::run_copy(const OpState& s, int8_t* output) const { if ((flags & COPY_FLAG_FILL) != 0) { std::memset( output, read_le_i32(p + 8), tensor_data_size(tensors_[s.output])); + } else if (s.clear_output) { + // Segments that do not cover a caller's output buffer must not leave its + // previous contents there. + std::memset(output, 0, tensor_data_size(tensors_[s.output])); } for (int32_t k = 0; k < segments; ++k) { const uint8_t* q = @@ -1828,6 +1871,16 @@ void Int8Interpreter::run_transposed_conv( bool Int8Interpreter::invoke() { if (!loaded_) return false; + if (options_.external_io) { + for (size_t i = 0; i < input_count_; ++i) { + if (tensor_data_[inputs_[i]] == nullptr) + return false; + } + for (size_t o = 0; o < output_count_; ++o) { + if (tensor_data_[outputs_[o]] == nullptr) + return false; + } + } InvokeLock lock; if (scratch_ != nullptr) { esp_nn_set_conv_scratch_buf(scratch_); diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index 0dd4aa6de5d..92626a8a6c0 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -154,8 +154,10 @@ bool validate_int8_conv( 1; if (output.shape[2] != expected_h || output.shape[3] != expected_w) return false; + // ESP-NN's convolution kernels take these as uint16_t. if (input.shape[1] > 65535 || input.shape[2] > 65535 || - input.shape[3] > 65535 || output.shape[1] > 65535) + input.shape[3] > 65535 || output.shape[1] > 65535 || + output.shape[2] > 65535 || output.shape[3] > 65535) return false; for (uint32_t i = 0; i < channels; ++i) { if (!valid_requantization( @@ -407,7 +409,8 @@ bool validate_int8_operation( !is_int8_activation(out) || !same_quantization(a, out) || a.rank != 4 || out.rank != 4 || a.shape[0] != 1 || out.shape[0] != 1 || a.shape[1] != out.shape[1] || - a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535) + a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535 || + out.shape[2] > 65535 || out.shape[3] > 65535) return false; const int32_t kh = p[0]; const int32_t kw = p[1]; diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py index 291ccca80f7..2a1874e1ba4 100644 --- a/backends/espressif/reference/esp_nn_reference.py +++ b/backends/espressif/reference/esp_nn_reference.py @@ -577,17 +577,21 @@ def _run_operation(self, operation, values) -> np.ndarray: # noqa: C901 raise ValueError(f"Unsupported opcode {opcode}") -def plan_activation_arena( - graph: XTS3Graph, alignment: int = 16 +def plan_activation_arena( # noqa: C901 + graph: XTS3Graph, alignment: int = 16, external_io: bool = False ) -> tuple[int, dict[int, int]]: - """Mirror of Int8Interpreter::plan_memory in the firmware.""" + """ + Mirror of Int8Interpreter::plan_memory in the firmware. With external_io + (as EspNNBackend loads graphs), graph inputs and outputs are not planned. + """ op_count = len(graph.operations) big = 1 << 30 first: dict[int, int] = {} last: dict[int, int] = {} sizes: dict[int, int] = {} + io_flags = TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT for tensor in graph.tensors: - if tensor.is_constant: + if tensor.is_constant or (external_io and tensor.flags & io_flags): continue first[tensor.tensor_id] = -1 if tensor.flags & TENSOR_FLAG_GRAPH_INPUT else big last[tensor.tensor_id] = ( @@ -600,8 +604,9 @@ def plan_activation_arena( if tensor_id in last: last[tensor_id] = max(last[tensor_id], index) for tensor_id in operation.outputs: - first[tensor_id] = min(first[tensor_id], index) - last[tensor_id] = max(last[tensor_id], index) + if tensor_id in first: + first[tensor_id] = min(first[tensor_id], index) + last[tensor_id] = max(last[tensor_id], index) order = sorted( (tensor_id for tensor_id in sizes if first[tensor_id] != big), key=lambda tensor_id: (-sizes[tensor_id], tensor_id), diff --git a/backends/espressif/test/host/CMakeLists.txt b/backends/espressif/test/host/CMakeLists.txt index b11108d466c..3f95c38708b 100644 --- a/backends/espressif/test/host/CMakeLists.txt +++ b/backends/espressif/test/host/CMakeLists.txt @@ -36,7 +36,7 @@ set(EXECUTORCH_BUILD_PYBIND OFF) set(EXECUTORCH_BUILD_DEVTOOLS OFF) set(EXECUTORCH_BUILD_XNNPACK OFF) set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF) -set(EXECUTORCH_BUILD_KERNELS_QUANTIZED OFF) +set(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON) set(EXECUTORCH_BUILD_PTHREADPOOL OFF) set(EXECUTORCH_BUILD_CPUINFO OFF) set(EXECUTORCH_BUILD_TESTS OFF) @@ -55,10 +55,14 @@ if(ESPNN_HOST_SANITIZE) target_compile_options(esp_nn_host PRIVATE -fno-sanitize=shift-base) endif() +# The portable and quantized kernels run what a program leaves on the CPU. +executorch_target_link_options_shared_lib(portable_ops_lib) +executorch_target_link_options_shared_lib(quantized_ops_lib) add_executable(espnn_host_runner espnn_host_runner.cpp) target_include_directories(espnn_host_runner PRIVATE "${EXECUTORCH_ROOT}/..") target_link_libraries( espnn_host_runner PRIVATE executorch_core extension_data_loader espnn_backend + portable_ops_lib quantized_ops_lib ) add_executable(espnn_host_interpreter espnn_host_interpreter.cpp) diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp index b023c066253..e3914d01a08 100644 --- a/backends/espressif/test/host/espnn_host_interpreter.cpp +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -12,11 +12,12 @@ * Runs one XTS3 graph on Int8Interpreter directly, for * backends/espressif/test/test_host_interpreter.py: * - * espnn_host_interpreter GRAPH OFFSET INPUT... -- OUTPUT... + * espnn_host_interpreter GRAPH OFFSET [--external-io] INPUT... -- OUTPUT... * * The graph is placed OFFSET bytes past a 16-byte boundary, so that a test can * load it from addresses with any alignment. Inputs and outputs are raw - * device-layout (NHWC) int8 files. + * device-layout (NHWC) int8 files. With --external-io, inputs and outputs are + * bound to runner buffers, the outputs prefilled with 0x5a. */ #include @@ -29,6 +30,7 @@ #include using executorch::backends::espressif::Int8Interpreter; +using executorch::backends::espressif::Int8RuntimeOptions; using executorch::backends::espressif::Int8TensorInfo; namespace { @@ -61,9 +63,12 @@ int main(int argc, char** argv) { std::vector inputs; std::vector outputs; bool past_separator = false; + Int8RuntimeOptions options{}; for (int a = 3; a < argc; ++a) { if (std::strcmp(argv[a], "--") == 0) past_separator = true; + else if (std::strcmp(argv[a], "--external-io") == 0) + options.external_io = true; else (past_separator ? outputs : inputs).push_back(argv[a]); } @@ -75,11 +80,27 @@ int main(int argc, char** argv) { std::memcpy(base + offset, graph.data(), graph.size()); Int8Interpreter interpreter; - if (!interpreter.load(base + offset, graph.size())) + if (!interpreter.load(base + offset, graph.size(), options)) return fail("load failed"); if (inputs.size() != interpreter.input_count() || outputs.size() != interpreter.output_count()) return fail("wrong number of inputs or outputs"); + std::vector> bound(inputs.size() + outputs.size()); + for (size_t k = 0; options.external_io && k < bound.size(); ++k) { + Int8TensorInfo info{}; + if (k < inputs.size()) + interpreter.input_info(k, info); + else + interpreter.output_info(k - inputs.size(), info); + bound[k].resize(info.bytes / 8 + 2); + int8_t* buffer = reinterpret_cast(bound[k].data()); + if (k < inputs.size()) { + interpreter.bind_input(k, buffer); + } else { + std::memset(buffer, 0x5a, info.bytes); + interpreter.bind_output(k - inputs.size(), buffer); + } + } for (size_t i = 0; i < inputs.size(); ++i) { Int8TensorInfo info{}; interpreter.input_info(i, info); diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index f520d2f992b..edc36dffe5f 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -13,7 +13,7 @@ * backends/espressif/test/test_host_runtime.py. * * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] - * [--alias INPUT:OUTPUT]... + * [--alias INPUT:OUTPUT]... [--misalign] * * Step s of instance k reads input i from IO_DIR/in__.bin and writes * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its @@ -22,8 +22,9 @@ * outputs must not be memory planned), the tensors are bound once to runner * buffers through set_input and set_output_data_ptr, and --alias makes an * input and an output share one buffer: that input is then read from its file - * at step 0 only and afterwards holds the output of the previous step. After - * loading, the runner prints the delegates' memory as a STATS line. + * at step 0 only and afterwards holds the output of the previous step. + * --misalign places every runner buffer one byte past a 16-byte boundary. At + * the end, the runner prints the delegates' memory as a STATS line. */ #include @@ -83,18 +84,24 @@ void write_file(const std::string& path, const void* data, size_t size) { std::fclose(file); } +uint8_t* aligned_buffer(uint8_t* base, size_t misalign) { + return base + (16 - reinterpret_cast(base) % 16) % 16 + misalign; +} + // A tensor over a runner-owned buffer with the program's shape and dim order. struct HostTensor { - explicit HostTensor(const TensorInfo& info) + HostTensor(const TensorInfo& info, size_t misalign) : sizes(info.sizes().begin(), info.sizes().end()), dim_order(info.dim_order().begin(), info.dim_order().end()), strides(sizes.size()), - data(info.nbytes()), + storage(info.nbytes() + 32), + data(aligned_buffer(storage.data(), misalign)), + nbytes(info.nbytes()), impl( info.scalar_type(), static_cast(sizes.size()), sizes.data(), - data.data(), + data, dim_order.data(), strides.data()) { if (executorch::runtime::dim_order_to_stride( @@ -105,7 +112,9 @@ struct HostTensor { std::vector sizes; std::vector dim_order; std::vector strides; - std::vector data; + std::vector storage; + uint8_t* data; + size_t nbytes; TensorImpl impl; }; @@ -130,6 +139,7 @@ int main(int argc, char** argv) { const int steps = std::atoi(argv[3]); int instance_count = 1; bool share_io = false; + size_t misalign = 0; std::vector> aliases; for (int a = 4; a < argc; ++a) { const std::string arg = argv[a]; @@ -137,6 +147,8 @@ int main(int argc, char** argv) { instance_count = std::atoi(argv[++a]); } else if (arg == "--share-io") { share_io = true; + } else if (arg == "--misalign") { + misalign = 1; } else if (arg == "--alias" && a + 1 < argc) { unsigned input = 0, output = 0; if (std::sscanf(argv[++a], "%u:%u", &input, &output) != 2) @@ -181,18 +193,17 @@ int main(int argc, char** argv) { instance.method = std::make_unique(std::move(method.get())); for (size_t i = 0; i < meta->num_inputs(); ++i) { - instance.inputs.push_back( - std::make_unique(meta->input_tensor_meta(i).get())); + instance.inputs.push_back(std::make_unique( + meta->input_tensor_meta(i).get(), misalign)); } for (size_t o = 0; o < meta->num_outputs(); ++o) { - instance.outputs.push_back( - std::make_unique(meta->output_tensor_meta(o).get())); + instance.outputs.push_back(std::make_unique( + meta->output_tensor_meta(o).get(), misalign)); } for (const auto& [input, output] : aliases) { if (input >= instance.inputs.size() || output >= instance.outputs.size() || - instance.inputs[input]->data.size() != - instance.outputs[output]->data.size()) + instance.inputs[input]->nbytes != instance.outputs[output]->nbytes) fail("invalid --alias"); } if (!share_io) @@ -204,27 +215,18 @@ int main(int argc, char** argv) { fail("set_input", static_cast(error)); } for (size_t o = 0; o < instance.outputs.size(); ++o) { - void* buffer = instance.outputs[o]->data.data(); + void* buffer = instance.outputs[o]->data; for (const auto& [input, output] : aliases) { if (output == o) - buffer = instance.inputs[input]->data.data(); + buffer = instance.inputs[input]->data; } Error error = instance.method->set_output_data_ptr( - buffer, instance.outputs[o]->data.size(), o); + buffer, instance.outputs[o]->nbytes, o); if (error != Error::Ok) fail("set_output_data_ptr", static_cast(error)); } } - const auto stats = executorch::backends::espressif::delegate_stats(); - std::printf( - "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu\n", - static_cast(stats.delegates), - stats.arena_bytes, - stats.scratch_bytes, - stats.parameter_bytes, - stats.table_bytes); - for (int step = 0; step < steps; ++step) { for (int k = 0; k < instance_count; ++k) { Instance& instance = instances[k]; @@ -238,8 +240,8 @@ int main(int argc, char** argv) { const std::vector bytes = read_file( io_dir + "/in_" + std::to_string(step) + "_" + std::to_string(i) + ".bin", - tensor.data.size()); - std::memcpy(tensor.data.data(), bytes.data(), bytes.size()); + tensor.nbytes); + std::memcpy(tensor.data, bytes.data(), bytes.size()); if (!share_io) { Error error = instance.method->set_input( EValue(executorch::aten::Tensor(&tensor.impl)), i); @@ -261,5 +263,16 @@ int main(int argc, char** argv) { } } } + const auto stats = executorch::backends::espressif::delegate_stats(); + std::printf( + "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu " + "staging=%zu\n", + static_cast(stats.delegates), + stats.arena_bytes, + stats.scratch_bytes, + stats.parameter_bytes, + stats.table_bytes, + stats.staging_bytes); + return 0; } diff --git a/backends/espressif/test/test_host_interpreter.py b/backends/espressif/test/test_host_interpreter.py index d43c53289f3..9b84e00c1b2 100644 --- a/backends/espressif/test/test_host_interpreter.py +++ b/backends/espressif/test/test_host_interpreter.py @@ -19,6 +19,7 @@ from __future__ import annotations import os +import struct import subprocess import pytest @@ -47,7 +48,9 @@ def disable_grad(): yield -def run(tmp_path, graph: xts3.XTS3Graph, offset: int, inputs) -> tuple[list, dict]: +def run( + tmp_path, graph: xts3.XTS3Graph, offset: int, inputs, *options +) -> tuple[list, dict]: blob = tmp_path / "graph.xts3" blob.write_bytes(xts3.serialize_graph(graph)) input_paths, output_paths = [], [] @@ -58,7 +61,15 @@ def run(tmp_path, graph: xts3.XTS3Graph, offset: int, inputs) -> tuple[list, dic for o in range(len(graph.outputs)): output_paths.append(str(tmp_path / f"out_{offset}_{o}.bin")) result = subprocess.run( - [INTERPRETER, str(blob), str(offset), *input_paths, "--", *output_paths], + [ + INTERPRETER, + str(blob), + str(offset), + *options, + *input_paths, + "--", + *output_paths, + ], capture_output=True, text=True, ) @@ -93,3 +104,56 @@ def test_graph_at_any_alignment(tmp_path, version): assert stats["blob_copied"] == ("0" if offset == 0 else "1") # Version 7 graphs keep their requantization parameters in place. assert (stats["parameters"] == "0") == (version == 7) + + +def test_partial_copy_does_not_leak_into_a_bound_output(tmp_path): + # One COPY segment writes 4 of the output's 16 elements; the rest of a + # caller's output buffer must not keep its previous contents. + def tensor(tensor_id, size, flags): + return xts3.XTS3Tensor( + tensor_id, + (1, size), + xts3.XTS3DType.INT8, + flags=flags | xts3.TENSOR_FLAG_QUANTIZED, + quant_scale=1.0, + ) + + segment = struct.pack( + xts3.COPY_SEGMENT, + 0, + 1, + 0, + 0, + 4, + 1, + 1, + 1, + 1, + 1, + *([1] + [0] * 5) * 2, + 0, + 0, + 0, + 0, + 0, + ) + graph = xts3.XTS3Graph( + tensors=[ + tensor(0, 4, xts3.TENSOR_FLAG_GRAPH_INPUT), + tensor(1, 16, xts3.TENSOR_FLAG_GRAPH_OUTPUT), + ], + operations=[ + xts3.XTS3Operation( + 0, + xts3.XTS3OpCode.COPY, + [0], + [1], + struct.pack(xts3.COPY_HEADER, 1, 0, 0) + segment, + ) + ], + inputs=[0], + outputs=[1], + ) + values = np.arange(1, 5, dtype=np.int8) + (output,), _ = run(tmp_path, graph, 0, [values], "--external-io") + np.testing.assert_array_equal(output, np.concatenate([values, np.zeros(12)])) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index ac481d8330e..3f0dc7a934e 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -30,27 +30,59 @@ import torch from executorch.backends.espressif import to_espnn_edge +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer from executorch.backends.espressif.reference import ( esp_nn_reference as reference, ProgramEmulator, ) from executorch.backends.espressif.serialization import xts3 from executorch.backends.espressif.test import ( + test_cpu_fallback as fallback_cases, test_int8_lowering as lowering_cases, test_int8_ops as op_cases, ) from executorch.exir import ExecutorchBackendConfig from executorch.exir.lowered_backend_module import LoweredBackendModule from executorch.exir.passes import MemoryPlanningPass +from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e RUNNER = os.environ.get("ESPNN_HOST_RUNNER") pytestmark = pytest.mark.skipif( not RUNNER, reason="Set ESPNN_HOST_RUNNER to espnn_host_runner" ) -# The runner links no CPU kernels, so only fully delegated programs run on it; -# transpose-select-stack leaves a cat on the CPU. + +class RepeatedOutput(torch.nn.Module): + """Returns one tensor twice, so two delegate outputs share a graph tensor.""" + + def __init__(self): + super().__init__() + self.conv = torch.nn.Conv2d(3, 4, 1) + + def forward(self, x): + y = self.conv(x) + return y, y + + +class LongCat(torch.nn.Module): + """A cat of seven tensors, which two COPY operations write into one output.""" + + def __init__(self): + super().__init__() + self.convs = torch.nn.ModuleList(torch.nn.Conv2d(3, 2, 1) for _ in range(7)) + + def forward(self, x): + return torch.cat([conv(x) for conv in self.convs], dim=2) + + +# Fully delegated programs, checked bit-exactly; transpose-select-stack leaves a +# cat on the CPU and runs with the CPU-fallback programs below. single-conv has +# no activation besides its input and output. CASES = [ + ("single-conv", torch.nn.Conv2d(3, 4, 1), [(1, 3, 8, 8)]), + ("repeated-output", RepeatedOutput(), [(1, 3, 8, 8)]), + ("seven-way-cat", LongCat(), [(1, 3, 4, 4)]), ("conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)]), ("concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)]), ("se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]), @@ -89,6 +121,20 @@ def as_list(value): return list(value) if isinstance(value, (tuple, list)) else [value] +def to_bytes(tensor: torch.Tensor) -> bytes: + """The tensor's bytes in memory order, which may be channels-last.""" + if tensor.dim() == 4 and tensor.is_contiguous(memory_format=torch.channels_last): + tensor = tensor.permute(0, 2, 3, 1) + return tensor.contiguous().numpy().tobytes() + + +def from_bytes(data: bytes, like: torch.Tensor, channels_last: bool) -> torch.Tensor: + channels_last = channels_last and like.dim() == 4 + shape = tuple(like.permute(0, 2, 3, 1).shape) if channels_last else like.shape + tensor = torch.frombuffer(bytearray(data), dtype=like.dtype).reshape(shape) + return tensor.permute(0, 3, 1, 2) if channels_last else tensor + + def run_host(tmp_path: Path, program: bytes, steps, *args): """ Runs one list of input tensors per step. Returns the output files per @@ -97,9 +143,7 @@ def run_host(tmp_path: Path, program: bytes, steps, *args): (tmp_path / "program.pte").write_bytes(program) for step, inputs in enumerate(steps): for i, tensor in enumerate(inputs): - (tmp_path / f"in_{step}_{i}.bin").write_bytes( - tensor.contiguous().numpy().tobytes() - ) + (tmp_path / f"in_{step}_{i}.bin").write_bytes(to_bytes(tensor)) result = subprocess.run( [RUNNER, str(tmp_path / "program.pte"), str(tmp_path), str(len(steps))] + [str(arg) for arg in args], @@ -124,12 +168,13 @@ def run_host(tmp_path: Path, program: bytes, steps, *args): return outputs, stats -def assert_matches(paths: list[Path], expected: list[torch.Tensor]): +def assert_matches( + paths: list[Path], expected: list[torch.Tensor], channels_last: bool = False +): + """Compares output files, in the program's dim order, with the reference.""" assert len(paths) == len(expected) for path, tensor in zip(paths, expected): - actual = torch.frombuffer( - bytearray(path.read_bytes()), dtype=tensor.dtype - ).reshape(tensor.shape) + actual = from_bytes(path.read_bytes(), tensor, channels_last) if tensor.dtype == torch.float32: torch.testing.assert_close(actual, tensor, atol=1e-5, rtol=1e-5) else: @@ -154,7 +199,7 @@ def test_host_delegate_matches_reference(tmp_path, name, model, shapes): assert_matches(outputs[step], as_list(emulator(*inputs))) # With ESP-NN scratch planned in the arena, the arena also holds scratch. planned = sum( - reference.plan_activation_arena(delegate.graph)[0] + reference.plan_activation_arena(delegate.graph, external_io=True)[0] for delegate in emulator.delegates.values() ) assert stats["delegates"] == len(emulator.delegates) @@ -199,3 +244,211 @@ def test_version_6_graphs_still_run(tmp_path): assert_matches(outputs[step], as_list(emulator(*inputs))) assert all(d.graph.version == 6 for d in emulator.delegates.values()) assert stats["parameters"] > 0 + + +FALLBACK_CASES = [ + ("transpose-select-stack", op_cases.TransposeSelect(), [(1, 3, 4, 5)]) +] + [(name, model, shapes) for name, model, shapes, _ in fallback_cases.CASES] + + +@pytest.mark.parametrize( + "name, model, shapes", FALLBACK_CASES, ids=[c[0] for c in FALLBACK_CASES] +) +def test_programs_with_cpu_operators(tmp_path, name, model, shapes): + # Delegates sit between portable kernels, which receive and return their + # tensors (some delegates return one tensor as several outputs); the + # kernels' float rounding gets the emulator suite's tolerance. + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program) + assert report["cpu_ops"] + exported = edge.exported_program() + steps = random_steps(shapes, 2) + (outputs,), _ = run_host(tmp_path, edge.to_executorch().buffer, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + expected = [t.to(torch.float32) for t in as_list(emulator(*inputs))] + peak = max(float(t.abs().max()) for t in expected) + for path, tensor in zip(outputs[step], expected): + actual = from_bytes(path.read_bytes(), tensor, False) + torch.testing.assert_close( + actual, tensor, atol=0.01 * max(peak, 1.0), rtol=0.01 + ) + + +class StreamingConv(torch.nn.Module): + """One step of a streaming convolution: the state holds the last frames.""" + + def __init__(self, channels_last: bool): + super().__init__() + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + self.memory_format = ( + torch.channels_last if channels_last else torch.contiguous_format + ) + + def forward(self, x, state): + window = torch.cat([state, x], dim=2) + new_state = window[:, :, 1:].clone(memory_format=self.memory_format) + return self.conv(window).clone(memory_format=self.memory_format), new_state + + +class LateStateRead(StreamingConv): + """Writes its new state before it reads the old one.""" + + def __init__(self, channels_last: bool): + super().__init__(channels_last) + self.conv = torch.nn.Conv2d(4, 8, (2, 1)) + self.frame = torch.nn.Conv2d(4, 4, 1) + + def forward(self, x, state): + new_state = ( + self.frame(x).expand(-1, -1, 2, -1).clone(memory_format=self.memory_format) + ) + return self.conv(state).clone(memory_format=self.memory_format), new_state + + +class EarlyOutput(LateStateRead): + """LateStateRead with its output computed before its new state.""" + + def forward(self, x, state): + output = self.conv(state).clone(memory_format=self.memory_format) + new_state = ( + self.frame(x).expand(-1, -1, 2, -1).clone(memory_format=self.memory_format) + ) + return output, new_state + + +def lower_streaming(channels_last: bool, model_class=StreamingConv, patch=None): + """ + int8 inputs and outputs, not memory planned, optionally channels-last. + patch, if given, edits each delegate's XTS3 graph before serialization. + """ + model = model_class(channels_last).eval() + memory_format = torch.channels_last if channels_last else torch.contiguous_format + shapes = [(1, 4, 1, 1), (1, 4, 2, 1)] + + def sample(): + return tuple( + torch.randn(shape).to(memory_format=memory_format) for shape in shapes + ) + + example = sample() + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(16): + prepared(*sample()) + program = torch.export.export(convert_pt2e(prepared), example, strict=True) + for index in range(2): + quantize_input(program, index) + quantize_output(program, index) + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert not report["cpu_ops"] + if patch is not None: + for _, module in edge.exported_program().graph_module.named_children(): + if isinstance(module, LoweredBackendModule): + graph = xts3.parse_graph(module.processed_bytes) + patch(graph) + module._processed_bytes = xts3.serialize_graph(graph) + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) + ) + return edge.exported_program(), edge.to_executorch(config).buffer, sample + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_device_layout_io_is_used_in_place(tmp_path, channels_last): + exported, program, sample = lower_streaming(channels_last) + emulator = ProgramEmulator(exported) + steps = [] + for _ in range(2): + floats = sample() + steps.append( + [torch.clamp((t * 20).round(), -128, 127).to(torch.int8) for t in floats] + ) + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) + # Only the NCHW state needs transposing, in and out. + assert (stats["staging"] == 0) == channels_last + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_output_sharing_an_input_buffer(tmp_path, channels_last): + # The state output is written into the state input's buffer, as an + # application updating its state in place would bind them. + exported, program, _ = lower_streaming(channels_last, LateStateRead) + emulator = ProgramEmulator(exported) + generator = torch.Generator().manual_seed(3) + steps = [ + [ + torch.randint(-128, 128, shape, generator=generator, dtype=torch.int8) + for shape in [(1, 4, 1, 1), (1, 4, 2, 1)] + ] + for _ in range(4) + ] + if channels_last: + steps = [ + [t.contiguous(memory_format=torch.channels_last) for t in inputs] + for inputs in steps + ] + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io", "--alias", "1:1") + state = steps[0][1] + for step, inputs in enumerate(steps): + expected = as_list(emulator(inputs[0], state)) + assert_matches(outputs[step], expected, channels_last) + state = expected[1] + + +def int8_steps(count: int, channels_last: bool, seed: int = 5): + generator = torch.Generator().manual_seed(seed) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + return [ + [ + torch.randint( + -128, 128, shape, generator=generator, dtype=torch.int8 + ).contiguous(memory_format=memory_format) + for shape in [(1, 4, 1, 1), (1, 4, 2, 1)] + ] + for _ in range(count) + ] + + +def test_misaligned_buffers_are_staged(tmp_path): + exported, program, _ = lower_streaming(channels_last=True) + emulator = ProgramEmulator(exported) + steps = int8_steps(2, channels_last=True) + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io", "--misalign") + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), True) + assert stats["staging"] > 0 + + +def return_state_input(graph): + """Makes the graph return its state input instead of its new state.""" + state_in = graph.inputs[1] + shape = graph.tensors[state_in].shape + (index,) = [ + k for k, t in enumerate(graph.outputs) if graph.tensors[t].shape == shape + ] + graph.tensors[graph.outputs[index]].flags &= ~xts3.TENSOR_FLAG_GRAPH_OUTPUT + graph.tensors[state_in].flags |= xts3.TENSOR_FLAG_GRAPH_OUTPUT + graph.outputs[index] = state_in + + +@pytest.mark.parametrize("channels_last", [False, True]) +@pytest.mark.parametrize("alias", [[], ["--alias", "1:0"]], ids=["", "aliased"]) +def test_input_returned_as_output(tmp_path, channels_last, alias): + # With the alias, the first output is written over the returned input's + # buffer, which must not change the second output. + exported, program, _ = lower_streaming( + channels_last, EarlyOutput, patch=return_state_input + ) + emulator = ProgramEmulator(exported) + steps = int8_steps(1 if alias else 2, channels_last) + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io", *alias) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert torch.equal(expected[1], inputs[1]) + assert_matches(outputs[step], expected, channels_last) diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py index 52022281a70..6b0e3184e3f 100644 --- a/backends/espressif/test/test_xts3_parser_host.py +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -154,6 +154,60 @@ def test_parser_rejects_oversized_upsample_tables(parser): assert parser({"upsample": corrupted}) == {"upsample": "reject"} +def _wide_graph(opcode, width, params, weight=False): + """One operation from a [1, 1, 1, width - 1] input to a [1, 1, 1, width] + output, with a [1, 1, 1, 2] weight and a zero bias when weight is set.""" + activation = xts3.TENSOR_FLAG_QUANTIZED + tensors = [ + xts3.XTS3Tensor(0, (1, 1, 1, width - 1), xts3.XTS3DType.INT8), + xts3.XTS3Tensor(1, (1, 1, 1, width), xts3.XTS3DType.INT8), + ] + tensors[0].flags = activation | xts3.TENSOR_FLAG_GRAPH_INPUT + tensors[1].flags = activation | xts3.TENSOR_FLAG_GRAPH_OUTPUT + for tensor in tensors: + tensor.quant_scale = 1.0 + inputs = [0] + constants = b"" + if weight: + constant = xts3.TENSOR_FLAG_CONSTANT + tensors += [ + xts3.XTS3Tensor(2, (1, 1, 1, 2), xts3.XTS3DType.INT8, 0, 2, constant), + xts3.XTS3Tensor(3, (1,), xts3.XTS3DType.INT32, 16, 4, constant), + ] + inputs += [2, 3] + constants = bytes(20) + graph = xts3.XTS3Graph( + tensors=tensors, + operations=[xts3.XTS3Operation(0, opcode, inputs, [1], params)], + constant_data=constants, + inputs=[0], + outputs=[1], + ) + return xts3.serialize_graph(graph) + + +def test_parser_rejects_outputs_wider_than_esp_nn_supports(parser): + # ESP-NN's pooling and convolution kernels take uint16_t dimensions: a + # (1, 2) window with padding (0, 1) widens 65535 to 65536. + pool = struct.pack(xts3.INT8_POOL_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, -128, 127) + conv = struct.pack( + xts3.INT8_CONV_BASE_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, 1, -128, 127 + ) + struct.pack("<2i", 1 << 30, 0) + blobs = {} + for name, opcode, params, weight in [ + ("pool", xts3.XTS3OpCode.MAX_POOL2D, pool, False), + ("conv", xts3.XTS3OpCode.CONV2D, conv, True), + ]: + blobs[f"{name}_widest"] = _wide_graph(opcode, 65535, params, weight) + blobs[f"{name}_too_wide"] = _wide_graph(opcode, 65536, params, weight) + assert parser(blobs) == { + "pool_widest": "accept", + "pool_too_wide": "reject", + "conv_widest": "accept", + "conv_too_wide": "reject", + } + + def test_parser_rejects_misaligned_int32_constants(parser): graph = _lowered(ops.DetectionHead(), [(1, 3, 16, 16)]) blob = xts3.serialize_graph(graph) From a27966868ebaeb8b6453854cce509bb01002a4a1 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Sun, 15 Nov 2026 17:15:41 -0800 Subject: [PATCH 06/25] Let the ESP-NN delegate own quantized mutable buffers A streaming model keeps a few frames of history in mutable buffers. Lowered to the ESP-NN delegate today, each buffer stays in the program: it is passed into the delegate and its new value out of it on every call, in float, or in int8 when the application exports it as a stateless input and output and carries the state itself. For a 32-filter streaming keyword-spotting model on ESP32-S3 with six state tensors, those copies, their layout conversions and the extra delegate arguments took about a fifth of each call. EspNNPartitioner(delegate_mutable_buffers=True), also exposed by to_espnn_edge(), gives a mutable buffer to the delegate that holds every access to it, through tag_mutated_buffer(), when the buffer is read and written with the same int8 quantization: a float32 buffer read through quantize ops (what PT2E produces for a cat and slice shift register) or an int8 buffer read through dequantize ops. Other buffers stay in the program, as they do by default. The partition's graph is then XTS3 version 8, which adds a table of (state, update) tensor pairs; each state's quantized initial value is stored with the constants. An update that the runtime cannot write into a state's slot (a graph input or output, another state, or an update shared by several states) gets a copy of its own, made after the last operator. The runtime keeps each state in a reserved arena slot and initializes it at load. The update is planned into the state's own slot when no operator reads the state at or after the update's first write, and copied over the state after the last operator otherwise. Every loaded method instance has its own state. reset_delegate_states(), or the reset_states backend option through executorch::runtime::set_option(), restores the initial state of every EspNNBackend delegate before its next execute(); reloading a method resets it alone. ProgramEmulator and the reference interpreter keep the same state across calls and can reset it, and plan_activation_arena() mirrors the in-place update. New tests cover which buffers the delegate takes, the reference against the quantized PyTorch model across steps and a reset, and, on the host build, states updated in place and by copy, a state updated from a graph input, a reset, two instances, and the parser's checks of the state table. The emulator suite gains a stateful program. --- backends/espressif/README.md | 52 ++++ .../include/espnn/espnn_backend.h | 21 ++ .../include/espnn/int8_interpreter.h | 13 + .../espnn_runtime/include/espnn/xts3_graph.h | 27 ++- .../espnn_runtime/src/espnn_backend.cpp | 40 ++++ .../espnn_runtime/src/int8_interpreter.cpp | 62 ++++- .../espnn_runtime/src/xts3_graph.cpp | 88 ++++++- backends/espressif/lowering.py | 14 +- .../espressif/partition/espnn_partitioner.py | 74 +++++- .../espressif/reference/esp_nn_reference.py | 44 +++- .../espressif/reference/program_emulator.py | 19 +- backends/espressif/serialization/xts3.py | 189 ++++++++++++++- .../espressif/test/host/espnn_host_runner.cpp | 31 ++- backends/espressif/test/test_host_runtime.py | 57 +++++ .../espressif/test/test_mutable_buffers.py | 226 ++++++++++++++++++ .../espressif/test/test_xts3_parser_host.py | 38 +++ .../espressif/export_espnn_test_programs.py | 13 +- 17 files changed, 974 insertions(+), 34 deletions(-) create mode 100644 backends/espressif/test/test_mutable_buffers.py diff --git a/backends/espressif/README.md b/backends/espressif/README.md index e9f8013a0a2..832ed470a82 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -143,6 +143,58 @@ The application then binds its buffers once, with `Method::set_input()` and to update a state tensor in place): the delegate detects the overlap and writes that output through its staging buffer after all operators have run. +### Streaming state + +A streaming model keeps a few frames of history between calls in a mutable +buffer. By default the buffer stays in the program, which passes it to the +delegate and back on every call. With `delegate_mutable_buffers=True`, the +delegate that reads and updates a buffer owns it instead: the program loses +that input and output, and the delegate keeps the buffer as int8 in its arena, +written in place when no operator reads the old value after the new one is +produced. + +```python +class Streaming(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("frames", torch.zeros(1, 40, 2, 1)) + self.conv = torch.nn.Conv2d(40, 32, (3, 1)) + + def forward(self, x): # x: (1, 40, 1, 1), one new frame + window = torch.cat([self.frames, x], dim=2) + self.frames.copy_(window[:, :, 1:]) + return self.conv(window) + +# ... prepare_pt2e, calibrate, convert_pt2e as above, then reset the buffer +# that calibration updated before exporting. +edge, report = to_espnn_edge( + torch.export.export(quantized, (torch.zeros(1, 40, 1, 1),)), + delegate_mutable_buffers=True, +) +print(report["delegated_buffers"]) # ['frames'] +``` + +The delegate takes a buffer only when one partition holds every access to it +and the buffer is read and written with the same int8 quantization, as a +`cat` and slice shift register is with `EspNNQuantizer`. Other buffers stay +in the program, with a warning. + +On the device, each loaded method has its own state. It starts from the +buffer's initial value when the method is loaded and persists across +`execute()` calls. `reset_delegate_states()` (from `espnn/espnn_backend.h`) +or the `reset_states` backend option restores the initial state of every +EspNNBackend delegate before its next `execute()`; to reset one method alone, +load it again. After an `execute()` that fails, the state is undefined until +it is reset. The reset may be requested from any task. + +```cpp +executorch::backends::espressif::reset_delegate_states(); +// or, through the generic backend-option API: +executorch::runtime::BackendOptions<1> options; +options.set_option("reset_states", true); +executorch::runtime::set_option("EspNNBackend", options.view()); +``` + ## Running on the ESP32-S3 You need ESP-IDF and an activated ESP-IDF shell. diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index b966fe61c98..a349d896426 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -20,6 +20,10 @@ namespace espressif { /// Backend ID that `EspNNBackend.preprocess()` writes into lowered programs. constexpr const char* kEspNNBackendId = "EspNNBackend"; +/// Backend option (bool) that calls reset_delegate_states() when true: +/// `executorch::runtime::set_option(kEspNNBackendId, options)`. +constexpr char kResetStatesOption[] = "reset_states"; + /** * Counters accumulated by every EspNNBackend delegate since startup or the * last reset_delegate_stats() call. @@ -41,6 +45,10 @@ struct DelegateStats { /// Sum of the buffers that delegates allocated on their first execute() /// for inputs and outputs they cannot use in place. size_t staging_bytes; + /// Delegate-owned state tensors (mutable buffers), and how many of them + /// are updated in place rather than copied after the last operator. + uint32_t states; + uint32_t states_updated_in_place; }; /** @@ -64,6 +72,19 @@ DelegateStats delegate_stats(); /// Zeroes all delegate counters. void reset_delegate_stats(); +/** + * Restores the state that delegates own (the mutable buffers lowered with + * `delegate_mutable_buffers=True`) to its initial value, in every loaded + * EspNNBackend delegate, before that delegate's next execute(). + * + * Each loaded method instance has its own state, kept between executions + * until this is called or the method is destroyed; loading the method again + * also starts from the initial state. Safe to call from any task, including + * while a method executes on another one: that execution finishes with the + * old state and the next one starts from the initial state. + */ +void reset_delegate_states(); + } // namespace espressif } // namespace backends } // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index f1d79513600..2776d915cab 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -204,6 +204,18 @@ class Int8Interpreter { size_t scratch_bytes() const { return scratch_size_; } + /// Number of delegate-owned state tensors (mutable buffers). + size_t state_count() const { + return graph_.num_states; + } + /// Number of states whose new value is written in place over the old one + /// rather than copied after the last operation. + size_t states_updated_in_place() const; + /** + * Restores every state tensor to its initial value. load() does this too. + * Not thread-safe with respect to invoke() on the same interpreter. + */ + void reset_states(); /// Size of the per-operation and per-tensor tables kept in RAM after load(). size_t table_bytes() const; /// Size of the precomputed requantization tables. @@ -262,6 +274,7 @@ class Int8Interpreter { int32_t* params_ = nullptr; size_t params_size_ = 0; int64_t* op_us_ = nullptr; + uint16_t updates_in_place_ = 0; uint32_t inputs_[MAX_GRAPH_IO]{}; uint32_t outputs_[MAX_GRAPH_IO]{}; size_t input_count_ = 0; diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h index a275c626463..bfe194bfff3 100644 --- a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -21,21 +21,26 @@ * (version 6), fixed-size tensor and operation records, a parameter region * and a constant region. Version 7 pads the input/output table to a 4-byte * boundary, so that the records and the parameter region are word-aligned and - * the runtime can use them where they are. parse_graph() bounds-checks every - * region and record before any accessor reads it. + * the runtime can use them where they are. Version 8 adds a state table after + * the input/output table: pairs of a state tensor, which persists across + * invokes, and the tensor holding its next value. parse_graph() bounds-checks + * every region and record before any accessor reads it. */ namespace executorch { namespace backends { namespace espressif { -/// Graph versions accepted by this runtime; 6 adds the input/output table and -/// 7 aligns the tables that follow it. +/// Graph versions accepted by this runtime; 6 adds the input/output table, 7 +/// aligns the tables that follow it and 8 adds the state table. static constexpr uint32_t GRAPH_ESP_NN_INT8_VERSION = 5; static constexpr uint32_t GRAPH_ESP_NN_INT8_IO_VERSION = 6; static constexpr uint32_t GRAPH_ESP_NN_INT8_ALIGNED_VERSION = 7; +static constexpr uint32_t GRAPH_ESP_NN_INT8_STATE_VERSION = 8; static constexpr size_t GRAPH_HEADER_SIZE = 22; static constexpr size_t GRAPH_IO_TABLE_HEADER_SIZE = 4; +static constexpr size_t GRAPH_STATE_TABLE_HEADER_SIZE = 4; +static constexpr size_t MAX_GRAPH_STATES = 16; static constexpr size_t MAX_GRAPH_IO = 16; static constexpr size_t TENSOR_RECORD_SIZE = 36; static constexpr size_t OPERATION_RECORD_SIZE = 44; @@ -47,6 +52,10 @@ static constexpr uint8_t TENSOR_FLAG_CONSTANT = 1U << 0; static constexpr uint8_t TENSOR_FLAG_QUANTIZED = 1U << 1; static constexpr uint8_t TENSOR_FLAG_GRAPH_INPUT = 1U << 2; static constexpr uint8_t TENSOR_FLAG_GRAPH_OUTPUT = 1U << 3; +/// Delegate-owned int8 state (version 8): its initial value is in the constant +/// region at constant_offset, and its state-table pair replaces it after every +/// invoke. +static constexpr uint8_t TENSOR_FLAG_STATE = 1U << 4; /// Serialized parameter block sizes and modes, mirrored in xts3.py. static constexpr size_t INT8_CONV_BASE_PARAMS_SIZE = 20; @@ -152,6 +161,9 @@ struct GraphView { uint16_t num_inputs; uint16_t num_outputs; const uint8_t* io_ids; + uint16_t num_states; + /// num_states (state, update) tensor ID pairs, little-endian u32. + const uint8_t* state_ids; }; /// Decoded convolution parameters; multipliers and shifts are per channel. @@ -211,6 +223,13 @@ uint32_t graph_output_count(const GraphView& graph); uint32_t graph_input_id(const GraphView& graph, uint32_t index); /// Tensor ID of output @p index, or UINT32_MAX if out of range. uint32_t graph_output_id(const GraphView& graph, uint32_t index); +/// State pair @p index: the persistent tensor and the tensor holding its next +/// value. @retval false @p index is out of range. +bool graph_state_pair( + const GraphView& graph, + uint32_t index, + uint32_t& state_id, + uint32_t& update_id); /// Decodes tensor record @p index. @retval false Out of range. bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor); /// Decodes operation record @p index. @retval false Out of range. diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index dc88b74ca77..70838b785e5 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -10,6 +10,7 @@ #include +#include #include #include #include @@ -36,6 +37,8 @@ using executorch::runtime::Backend; using executorch::runtime::BackendExecutionContext; using executorch::runtime::BackendInitContext; using executorch::runtime::BackendInterface; +using executorch::runtime::BackendOption; +using executorch::runtime::BackendOptionContext; using executorch::runtime::CompileSpec; using executorch::runtime::DelegateHandle; using executorch::runtime::Error; @@ -50,6 +53,9 @@ namespace espressif { namespace { DelegateStats g_stats{}; +// Incremented by each reset request; a delegate whose state is older restores +// its initial state before its next execute. +std::atomic g_state_generation{0}; constexpr float kHalfEvenRoundingBias = 12582912.0f; @@ -199,6 +205,7 @@ bool store_output( struct DelegateState { Int8Interpreter interpreter; bool profiled = false; + uint32_t state_generation = 0; // Device-layout copies of the inputs and outputs whose tensors the // interpreter cannot use directly (float, NCHW, misaligned, an input that is // also an output, or an output that overlaps an input), one slot per input @@ -285,11 +292,16 @@ class EspNNBackendInterface final : public BackendInterface { } if (state->interpreter.constants_copied()) processed->Free(); + state->state_generation = + g_state_generation.load(std::memory_order_relaxed); ++g_stats.delegates; g_stats.arena_bytes += state->interpreter.arena_bytes(); g_stats.scratch_bytes += state->interpreter.scratch_bytes(); g_stats.parameter_bytes += state->interpreter.parameter_bytes(); g_stats.table_bytes += state->interpreter.table_bytes(); + g_stats.states += state->interpreter.state_count(); + g_stats.states_updated_in_place += + state->interpreter.states_updated_in_place(); return state; } @@ -309,6 +321,14 @@ class EspNNBackendInterface final : public BackendInterface { return Error::InvalidArgument; } const int64_t started = esp_timer_get_time(); + if (interpreter.state_count() != 0) { + const uint32_t generation = + g_state_generation.load(std::memory_order_relaxed); + if (generation != state->state_generation) { + interpreter.reset_states(); + state->state_generation = generation; + } + } // An aligned int8 tensor already in the device layout is used in place; // any other goes through a staging slot. Int8TensorInfo info{}; @@ -431,6 +451,22 @@ class EspNNBackendInterface final : public BackendInterface { return Error::Ok; } + Error set_option( + BackendOptionContext&, + const Span& backend_options) override { + for (const BackendOption& option : backend_options) { + const bool* value = std::get_if(&option.value); + if (std::strcmp(option.key, kResetStatesOption) != 0 || + value == nullptr) { + std::printf("EspNN delegate: unknown option %s\n", option.key); + return Error::NotSupported; + } + if (*value) + reset_delegate_states(); + } + return Error::Ok; + } + void destroy(DelegateHandle* handle) const override { if (handle != nullptr) static_cast(handle)->~DelegateState(); @@ -459,6 +495,10 @@ DelegateStats delegate_stats() { g_stats = {}; } +void reset_delegate_states() { + g_state_generation.fetch_add(1, std::memory_order_relaxed); +} + } // namespace espressif } // namespace backends } // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 1e8418661a3..aead7cea365 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -314,7 +314,7 @@ bool Int8Interpreter::load( "ERROR: model is XTS3 v%u; the int8 runtime requires v%u to v%u\n", static_cast(graph_.header.version), static_cast(GRAPH_ESP_NN_INT8_VERSION), - static_cast(GRAPH_ESP_NN_INT8_ALIGNED_VERSION)); + static_cast(GRAPH_ESP_NN_INT8_STATE_VERSION)); return false; } const bool misaligned = @@ -381,6 +381,7 @@ bool Int8Interpreter::load( reset(); return false; } + reset_states(); if (options_.copy_constants_to_psram && blob_copy_ == nullptr && heap_caps_get_total_size(MALLOC_CAP_SPIRAM) != 0) { const size_t largest = @@ -466,7 +467,9 @@ bool Int8Interpreter::plan_memory() { : 0); for (uint32_t i = 0; i < tensor_count; ++i) { first[i] = - (tensors_[i].flags & TENSOR_FLAG_GRAPH_INPUT) != 0 ? -1 : INT32_MAX; + (tensors_[i].flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE)) != 0 + ? -1 + : INT32_MAX; last[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); @@ -498,6 +501,24 @@ bool Int8Interpreter::plan_memory() { } } + // A state keeps its slot across invokes, so it is live for the whole graph. + // Its update is written straight into that slot when no operation reads the + // state at or after the update's first write; otherwise the update gets its + // own slot and is copied over the state after the last operation. + updates_in_place_ = 0; + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if (last[state] < first[update]) { + updates_in_place_ |= 1U << k; + planned[update] = true; + } else { + last[update] = op_count; + } + last[state] = op_count; + } + size_t arena_size = 0; for (uint32_t step = 0; step < entries; ++step) { int32_t best = -1; @@ -533,6 +554,13 @@ bool Int8Interpreter::plan_memory() { if (candidate + bytes[best] > arena_size) arena_size = candidate + bytes[best]; } + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if ((updates_in_place_ & (1U << k)) != 0) + offsets[update] = offsets[state]; + } for (uint32_t i = 0; i < tensor_count; ++i) { if ((tensors_[i].flags & unplanned) == 0 && offsets[i] == SIZE_MAX) { std::printf( @@ -1109,6 +1137,25 @@ int32_t Int8Interpreter::output_argmax(size_t index) const { return best; } +size_t Int8Interpreter::states_updated_in_place() const { + size_t count = 0; + for (uint32_t k = 0; k < graph_.num_states; ++k) + count += (updates_in_place_ >> k) & 1U; + return count; +} + +void Int8Interpreter::reset_states() { + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + std::memcpy( + activation(state), + tensor_constant_data(graph_, tensors_[state]), + tensor_data_size(tensors_[state])); + } +} + size_t Int8Interpreter::table_bytes() const { if (!loaded_) return 0; @@ -1917,6 +1964,17 @@ bool Int8Interpreter::invoke() { } op_started = now; } + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if ((updates_in_place_ & (1U << k)) == 0) { + std::memcpy( + activation(state), + activation(update), + tensor_data_size(tensors_[state])); + } + } const int64_t finished = esp_timer_get_time(); if (finished - last_yield > YIELD_INTERVAL_US) vTaskDelay(1); diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index 92626a8a6c0..1e6927b668c 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -554,6 +554,8 @@ bool validate_int8_dataflow(const GraphView& graph) { produced.set(i); ++input_count; } + if ((tensor.flags & TENSOR_FLAG_STATE) != 0) + produced.set(i); if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) ++output_count; } @@ -641,6 +643,38 @@ bool validate_int8_dataflow(const GraphView& graph) { return false; } } + TensorSet paired_states; + TensorSet paired_updates; + for (uint32_t k = 0; k < graph.num_states; ++k) { + uint32_t state_id = 0; + uint32_t update_id = 0; + TensorRecord state{}; + TensorRecord update{}; + graph_state_pair(graph, k, state_id, update_id); + read_tensor(graph, state_id, state); + read_tensor(graph, update_id, update); + const uint8_t not_an_update = TENSOR_FLAG_STATE | TENSOR_FLAG_CONSTANT | + TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT; + if ((state.flags & TENSOR_FLAG_STATE) == 0 || + (update.flags & not_an_update) != 0 || !produced[update_id] || + paired_states[state_id] || paired_updates[update_id] || + !same_shape(state, update) || !same_quantization(state, update)) { + std::printf( + "ERROR: state pair %u is invalid\n", static_cast(k)); + return false; + } + paired_states.set(state_id); + paired_updates.set(update_id); + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_STATE) != 0 && !paired_states[i]) { + std::printf( + "ERROR: state tensor %u has no update\n", static_cast(i)); + return false; + } + } return true; } @@ -655,9 +689,8 @@ int32_t read_le_i32(const uint8_t* p) { } bool is_int8_graph_version(uint32_t version) { - return version == GRAPH_ESP_NN_INT8_VERSION || - version == GRAPH_ESP_NN_INT8_IO_VERSION || - version == GRAPH_ESP_NN_INT8_ALIGNED_VERSION; + return version >= GRAPH_ESP_NN_INT8_VERSION && + version <= GRAPH_ESP_NN_INT8_STATE_VERSION; } namespace { @@ -709,6 +742,18 @@ uint32_t graph_input_id(const GraphView& graph, uint32_t index) { return flagged_tensor(graph, TENSOR_FLAG_GRAPH_INPUT, index); } +bool graph_state_pair( + const GraphView& graph, + uint32_t index, + uint32_t& state_id, + uint32_t& update_id) { + if (graph.state_ids == nullptr || index >= graph.num_states) + return false; + state_id = read_u32(graph.state_ids + 8 * index); + update_id = read_u32(graph.state_ids + 8 * index + 4); + return true; +} + uint32_t graph_output_id(const GraphView& graph, uint32_t index) { if (graph.io_ids != nullptr) { return index < graph.num_outputs @@ -896,6 +941,25 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.tensor_table_offset = GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE + io_bytes; } + if (graph.header.version >= GRAPH_ESP_NN_INT8_STATE_VERSION) { + const size_t at = graph.tensor_table_offset; + if (!range_valid(at, GRAPH_STATE_TABLE_HEADER_SIZE, size)) + return false; + graph.num_states = read_u16(data + at); + const size_t state_bytes = 8U * static_cast(graph.num_states); + if (graph.num_states > MAX_GRAPH_STATES || read_u16(data + at + 2) != 0 || + !range_valid(at + GRAPH_STATE_TABLE_HEADER_SIZE, state_bytes, size)) { + std::printf("ERROR: graph state table is invalid\n"); + return false; + } + graph.state_ids = data + at + GRAPH_STATE_TABLE_HEADER_SIZE; + for (uint32_t i = 0; i < 2U * graph.num_states; ++i) { + if (read_u32(graph.state_ids + 4 * i) >= graph.header.num_tensors) + return false; + } + graph.tensor_table_offset = + at + GRAPH_STATE_TABLE_HEADER_SIZE + state_bytes; + } if (graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION) { const size_t padded = (graph.tensor_table_offset + 3U) & ~size_t{3}; if (!range_valid( @@ -905,7 +969,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; for (size_t i = graph.tensor_table_offset; i < padded; ++i) { if (data[i] != 0) { - std::printf("ERROR: graph IO table padding is not zero\n"); + std::printf("ERROR: graph table padding is not zero\n"); return false; } } @@ -938,7 +1002,8 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.parameter_data = data + graph.parameter_offset; graph.constant_data = data + graph.constant_offset; - const unsigned allowed_flags = 0x0FU; + const unsigned allowed_flags = + graph.header.version >= GRAPH_ESP_NN_INT8_STATE_VERSION ? 0x1FU : 0x0FU; for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; read_tensor(graph, i, tensor); @@ -988,6 +1053,17 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; } const size_t data_size = tensor_data_size(tensor); + const bool is_state = (tensor.flags & TENSOR_FLAG_STATE) != 0; + if (is_state && + (is_constant || + (tensor.flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != 0 || + !range_valid( + tensor.constant_offset, data_size, graph.header.constant_size))) { + std::printf( + "ERROR: state tensor %u is invalid\n", static_cast(i)); + return false; + } const bool misaligned_int32 = is_constant && tensor.dtype == static_cast(DataType::INT32) && (tensor.constant_offset % sizeof(int32_t)) != 0; @@ -997,7 +1073,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { (is_constant && !range_valid( tensor.constant_offset, data_size, graph.header.constant_size)) || - (!is_constant && tensor.constant_offset != 0)) { + (!is_constant && !is_state && tensor.constant_offset != 0)) { std::printf( "ERROR: tensor %u constant data range or shape is invalid\n", static_cast(i)); diff --git a/backends/espressif/lowering.py b/backends/espressif/lowering.py index 9998b993a6d..7e0c30b2015 100644 --- a/backends/espressif/lowering.py +++ b/backends/espressif/lowering.py @@ -51,7 +51,11 @@ def _operator_library_report(edge: EdgeProgramManager) -> dict: def to_espnn_edge( - program, allow_cpu_fallback: bool = True, *, use_operator_library: bool = False + program, + allow_cpu_fallback: bool = True, + *, + use_operator_library: bool = False, + delegate_mutable_buffers: bool = False, ): """ Lower a PT2E-quantized torch.export program for the ESP32 ExecuTorch @@ -63,6 +67,9 @@ def to_espnn_edge( operators and executable Q/DQ. Its report counts esp_nn_ops and lists remaining cpu_ops by method, excluding Q/DQ and structural getitem nodes. + + With delegate_mutable_buffers=True, the delegates own the quantized mutable + buffers (streaming state) they read and update; see EspNNPartitioner. """ if use_operator_library: edge = to_edge_transform_and_lower( @@ -81,7 +88,10 @@ def to_espnn_edge( ) return edge, report - partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback) + partitioner = EspNNPartitioner( + allow_cpu_fallback=allow_cpu_fallback, + delegate_mutable_buffers=delegate_mutable_buffers, + ) edge = to_edge_transform_and_lower( program, transform_passes=[DuplicateDequantNodePass()], diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index c1aa0de730e..80934aef6ad 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -8,6 +8,8 @@ from __future__ import annotations +import logging + from typing import Callable, Optional import torch @@ -39,7 +41,7 @@ Partitioner, PartitionResult, ) -from executorch.exir.backend.utils import tag_constant_data +from executorch.exir.backend.utils import tag_constant_data, tag_mutated_buffer from torch.export import ExportedProgram from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner from torch.fx.passes.operator_support import OperatorSupportBase @@ -276,6 +278,60 @@ def _has_compute(nodes: set) -> bool: return any(not _is_passthrough(node) for node in nodes) +def _qparams(node: torch.fx.Node, op: str) -> Optional[tuple]: + if node.op != "call_function" or not _operator_name(node.target).startswith( + f"quantized_decomposed.{op}." + ): + return None + return tuple(node.args[1:]) + + +def _int8_state_compatible(buffer: torch.fx.Node, writer: torch.fx.Node) -> bool: + """ + Whether the delegate can keep a mutable buffer as int8: a float32 buffer + read only through quantize ops whose new value is dequantized, or an int8 + buffer read only through dequantize ops whose new value is quantized, all + with the same parameters. + """ + reader, producer = { + torch.float32: ("quantize_per_tensor", "dequantize_per_tensor"), + torch.int8: ("dequantize_per_tensor", "quantize_per_tensor"), + }.get(buffer.meta["val"].dtype, (None, None)) + if reader is None or not buffer.users: + return False + reads = {_qparams(user, reader) for user in buffer.users} + return len(reads) == 1 and None not in reads and _qparams(writer, producer) in reads + + +def _tag_quantized_mutable_buffers(exported_program: ExportedProgram) -> list[str]: + """ + Give each mutable buffer to the partition that holds all its accesses, + unless the delegate could not keep it as int8. Returns the delegated + buffers' names. + """ + tag_mutated_buffer(exported_program) + signature = exported_program.graph_signature + nodes = {node.name: node for node in exported_program.graph.nodes} + writers = { + buffer: nodes[output] for output, buffer in signature.buffers_to_mutate.items() + } + delegated = [] + for node in exported_program.graph.find_nodes(op="placeholder"): + buffer = signature.inputs_to_buffers.get(node.name) + if buffer not in writers or "delegation_tag" not in node.meta: + continue + if _int8_state_compatible(node, writers[buffer]): + delegated.append(buffer) + continue + del node.meta["delegation_tag"] + logging.warning( + f"Mutable buffer {buffer} stays in the program: the ESP-NN delegate " + "owns a buffer only when it is read and written with the same " + "int8 quantization." + ) + return delegated + + class EspNNPartitioner(Partitioner): """ Delegates PT2E-quantized regions of a graph to the ESP32 ESP-NN backend. @@ -286,18 +342,28 @@ class EspNNPartitioner(Partitioner): With allow_cpu_fallback=False any operator left outside a delegate is an error, whether it is unsupported or its region could not be delegated; the standalone XTS3 firmware path requires this. + + With delegate_mutable_buffers, a quantized mutable buffer (streaming state) + whose readers and writer all fall in one partition is owned by that + delegate: it leaves the program's inputs and outputs, and the delegate + keeps it as int8 between executions. The buffer must be float32 and read + only through quantize ops, or int8 and read only through dequantize ops, + with the same quantization as its new value. Any other buffer stays in the + program. """ def __init__( self, compile_specs: Optional[list[CompileSpec]] = None, allow_cpu_fallback: bool = True, + delegate_mutable_buffers: bool = False, ): specs = list(compile_specs or []) if not any(spec.key == INT8_FORMAT_SPEC.key for spec in specs): specs.append(INT8_FORMAT_SPEC) self.delegation_spec = DelegationSpec(EspNNBackend.__name__, specs) self.allow_cpu_fallback = allow_cpu_fallback + self.delegate_mutable_buffers = delegate_mutable_buffers self.report: dict = {} def ops_to_not_decompose( @@ -374,7 +440,13 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: if not tags and not self.allow_cpu_fallback: raise RuntimeError("No ESP32 int8 partition could be formed") tag_constant_data(exported_program) + delegated_buffers = ( + _tag_quantized_mutable_buffers(exported_program) + if self.delegate_mutable_buffers + else [] + ) self.report = { + "delegated_buffers": delegated_buffers, "partitions": len(tags), "rejected_partitions": rejected, "delegated_ops": sum(1 for node in delegated if not _is_passthrough(node)), diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py index 2a1874e1ba4..266b7917115 100644 --- a/backends/espressif/reference/esp_nn_reference.py +++ b/backends/espressif/reference/esp_nn_reference.py @@ -31,6 +31,7 @@ SOFTMAX_OUTPUT_ZERO_POINT, TENSOR_FLAG_GRAPH_INPUT, TENSOR_FLAG_GRAPH_OUTPUT, + TENSOR_FLAG_STATE, TRANSPOSE_CONV_PARAMS, TRANSPOSE_CONV_PARAMS_SIZE, UPSAMPLE_HEADER, @@ -229,6 +230,23 @@ def __init__(self, graph: XTS3Graph): self.outputs = list(graph.outputs) or [ t.tensor_id for t in graph.tensors if t.flags & TENSOR_FLAG_GRAPH_OUTPUT ] + self.states: dict[int, np.ndarray] = {} + self.reset_states() + + def reset_states(self) -> None: + """Restores every delegate-owned state tensor to its initial value.""" + for state_id, _ in self.graph.states: + tensor = self.graph.tensors[state_id] + self.states[state_id] = ( + np.frombuffer( + self.graph.constant_data, + dtype=np.int8, + count=int(np.prod(tensor.shape)), + offset=tensor.data_offset, + ) + .copy() + .reshape(device_shape(tensor)) + ) def constant(self, tensor_id: int) -> np.ndarray: tensor = self.graph.tensors[tensor_id] @@ -252,7 +270,8 @@ def value(self, tensor_id: int, values) -> np.ndarray: raise KeyError(f"Tensor {tensor_id} has not been produced") def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: - values: dict[int, np.ndarray] = {} + """Runs one invoke; delegate-owned state carries over to the next.""" + values: dict[int, np.ndarray] = dict(self.states) for tensor_id in self.inputs: tensor = self.graph.tensors[tensor_id] data = np.asarray(inputs[tensor_id], dtype=np.int8) @@ -263,6 +282,8 @@ def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: values[tensor_id] = data for operation in self.graph.operations: values[operation.outputs[0]] = self._run_operation(operation, values) + for state_id, update_id in self.graph.states: + self.states[state_id] = values[update_id].copy() return values def run_outputs(self, inputs: list[np.ndarray]) -> list[np.ndarray]: @@ -593,7 +614,9 @@ def plan_activation_arena( # noqa: C901 for tensor in graph.tensors: if tensor.is_constant or (external_io and tensor.flags & io_flags): continue - first[tensor.tensor_id] = -1 if tensor.flags & TENSOR_FLAG_GRAPH_INPUT else big + first[tensor.tensor_id] = ( + -1 if tensor.flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE) else big + ) last[tensor.tensor_id] = ( op_count if tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT else -1 ) @@ -607,8 +630,21 @@ def plan_activation_arena( # noqa: C901 if tensor_id in first: first[tensor_id] = min(first[tensor_id], index) last[tensor_id] = max(last[tensor_id], index) + # A state keeps its slot for the whole graph. Its update shares that slot + # when no operation reads the state at or after the update's first write. + in_place = {} + for state_id, update_id in graph.states: + if last[state_id] < first[update_id]: + in_place[update_id] = state_id + else: + last[update_id] = op_count + last[state_id] = op_count order = sorted( - (tensor_id for tensor_id in sizes if first[tensor_id] != big), + ( + tensor_id + for tensor_id in sizes + if first[tensor_id] != big and tensor_id not in in_place + ), key=lambda tensor_id: (-sizes[tensor_id], tensor_id), ) offsets: dict[int, int] = {} @@ -631,6 +667,8 @@ def plan_activation_arena( # noqa: C901 moved = True offsets[tensor_id] = candidate arena = max(arena, candidate + sizes[tensor_id]) + for update_id, state_id in in_place.items(): + offsets[update_id] = offsets[state_id] return arena, offsets diff --git a/backends/espressif/reference/program_emulator.py b/backends/espressif/reference/program_emulator.py index 98754ab76b2..8087db24442 100644 --- a/backends/espressif/reference/program_emulator.py +++ b/backends/espressif/reference/program_emulator.py @@ -26,7 +26,13 @@ def delegate_output_dtypes(lowered_module) -> list[torch.dtype]: values = output.args[0] if not isinstance(values, (list, tuple)): values = (values,) - return [torch.float32 if _is_dequantize(value) else torch.int8 for value in values] + # New values of delegate-owned buffers stay in the delegate. + state_updates = set(program.graph_signature.buffers_to_mutate) + return [ + torch.float32 if _is_dequantize(value) else torch.int8 + for value in values + if value.name not in state_updates + ] class DelegateEmulator: @@ -37,6 +43,9 @@ def __init__(self, lowered_module): self.interpreter = reference.ReferenceInterpreter(self.graph) self.output_dtypes = delegate_output_dtypes(lowered_module) + def reset_states(self) -> None: + self.interpreter.reset_states() + def __call__(self, *inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: device_inputs = [] for tensor_id, value in zip(self.interpreter.inputs, inputs): @@ -69,7 +78,8 @@ class ProgramEmulator(torch.fx.Interpreter): """ Host model of an ExecuTorch program lowered with EspNNPartitioner: delegates run on the bit-exact ESP-NN reference, every other operator - runs in PyTorch (the device uses the ExecuTorch portable kernels). + runs in PyTorch (the device uses the ExecuTorch portable kernels). State + that a delegate owns persists from one call to the next, as on the device. """ def __init__(self, exported_program): @@ -94,5 +104,10 @@ def call_function(self, target, args, kwargs): def __call__(self, *inputs): return self.run(*inputs) + def reset_states(self) -> None: + """Restores the state that delegates own, as the runtime's reset does.""" + for delegate in self.delegates.values(): + delegate.reset_states() + __all__ = ["DelegateEmulator", "ProgramEmulator", "delegate_output_dtypes"] diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index e16c545b6ef..2c39e33d67e 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -20,11 +20,16 @@ MAGIC = b"XTS3" # Version 7 is version 6 with the IO table padded to 4 bytes, so that the -# runtime can use the tensor records and parameter blocks in place. +# runtime can use the tensor records and parameter blocks in place. Version 8 +# adds a state table after the IO table, and is written only for graphs that +# own mutable buffers. XTS3_VERSION = 7 -XTS3_VERSIONS = (5, 6, 7) +XTS3_STATE_VERSION = 8 +XTS3_VERSIONS = (5, 6, 7, 8) IO_TABLE_HEADER = " int: TENSOR_FLAG_QUANTIZED = 1 << 1 TENSOR_FLAG_GRAPH_INPUT = 1 << 2 TENSOR_FLAG_GRAPH_OUTPUT = 1 << 3 +# Delegate-owned int8 state (a mutable buffer of the partition). Its initial +# value is in the constant region at data_offset, but it is not a constant: the +# state table pairs it with the tensor that replaces it after every invoke. +TENSOR_FLAG_STATE = 1 << 4 @dataclass @@ -151,6 +161,20 @@ class XTS3Graph: tensor_names: dict[int, str] = field(default_factory=dict) inputs: list[int] = field(default_factory=list) outputs: list[int] = field(default_factory=list) + # (state tensor, update tensor) pairs; version 8 only + states: list[tuple[int, int]] = field(default_factory=list) + + +def _mutated_buffer_placeholders(exported_program) -> set[str]: + signature = getattr(exported_program, "graph_signature", None) + if signature is None: + return set() + mutated = set(signature.buffers_to_mutate.values()) + return { + name + for name, buffer in signature.inputs_to_buffers.items() + if buffer in mutated + } class ExportedTensorResolver: @@ -169,6 +193,7 @@ def __init__(self, exported_program): ) or {} ) + self.mutated_buffers = _mutated_buffer_placeholders(exported_program) def _lookup_mapping(self, mapping, name): if mapping is None: @@ -184,6 +209,8 @@ def _lookup_mapping(self, mapping, name): return None def resolve(self, node): # noqa: C901 + if node is not None and node.name in self.mutated_buffers: + return None if node is not None and node.op == "call_function": target = str(node.target) if "dequantize_per_channel" in target: @@ -1755,7 +1782,7 @@ def _user_input_placeholders(exported_program): return [node for node in placeholders if node.name in user_inputs] -def _mark_int8_graph_io( +def _mark_int8_graph_io( # noqa: C901 exported_program, registry: TensorRegistry ) -> tuple[list[int], list[int]]: graph_module = exported_program.graph_module @@ -1776,6 +1803,8 @@ def _mark_int8_graph_io( raise ValueError("Int8 graph has no quantized runtime input") outputs: list[int] = [] + signature = getattr(exported_program, "graph_signature", None) + state_updates = set(signature.buffers_to_mutate) if signature else set() for node in graph_module.graph.nodes: if node.op != "output": continue @@ -1783,6 +1812,8 @@ def _mark_int8_graph_io( if not isinstance(values, (list, tuple)): values = (values,) for value in values: + if getattr(value, "name", None) in state_updates: + continue source = _quantized_source(value) if source is None or source not in registry.node_to_id: raise ValueError( @@ -1797,6 +1828,111 @@ def _mark_int8_graph_io( return inputs, outputs +def _mark_int8_state( + exported_program, registry: TensorRegistry +) -> list[tuple[int, int]]: + """ + Pair each mutable buffer of the partition with the int8 tensor holding its + new value. The buffer's tensor becomes TENSOR_FLAG_STATE, with its + quantized initial value in the constant region. + """ + signature = exported_program.graph_signature + if not signature.buffers_to_mutate: + return [] + nodes = {node.name: node for node in exported_program.graph_module.graph.nodes} + writers = { + buffer: nodes[output] for output, buffer in signature.buffers_to_mutate.items() + } + states: list[tuple[int, int]] = [] + for node in exported_program.graph_module.graph.nodes: + buffer = signature.inputs_to_buffers.get(node.name) + if node.op != "placeholder" or buffer not in writers: + continue + if node not in registry.node_to_id: + raise ValueError(f"Mutable buffer {buffer} is not read as an int8 tensor") + state = registry.tensors[registry.node_to_id[node]] + source = _quantized_source(writers[buffer]) + if source is None or source not in registry.node_to_id: + raise ValueError( + f"The new value of mutable buffer {buffer} is not an int8 tensor" + ) + update = registry.tensors[registry.node_to_id[source]] + if ( + tuple(update.shape) != tuple(state.shape) + or update.quant_scale != state.quant_scale + or update.quant_zero_point != state.quant_zero_point + ): + raise ValueError( + f"Mutable buffer {buffer} and its new value have different " + "shapes or quantization parameters" + ) + initial = exported_program.state_dict.get( + buffer, exported_program.constants.get(buffer) + ) + if initial is None: + raise ValueError(f"Mutable buffer {buffer} has no initial value") + if initial.dtype == torch.float32: + initial = torch.ops.quantized_decomposed.quantize_per_tensor( + initial, + state.quant_scale, + state.quant_zero_point, + -128, + 127, + torch.int8, + ) + if initial.dtype != torch.int8: + raise ValueError(f"Mutable buffer {buffer} must be float32 or int8") + raw = _int8_device_bytes( + initial.detach().cpu().contiguous(), tuple(state.shape) + ) + registry._align_constant_data() + state.data_offset = len(registry.constant_data) + state.data_size = len(raw) + registry.constant_data.extend(raw) + state.flags |= TENSOR_FLAG_STATE + states.append((state.tensor_id, update.tensor_id)) + return _separate_state_updates(states, registry) + + +def _separate_state_updates( + states: list[tuple[int, int]], registry: TensorRegistry +) -> list[tuple[int, int]]: + """ + Give a copy of its own to each update the runtime cannot write into a + state's slot: a graph input or output, a constant, a state, or an update + shared by several states. The copies run after every other operation. + """ + from .int8_ops import copy_operation, copy_segment + + shared = ( + TENSOR_FLAG_GRAPH_INPUT + | TENSOR_FLAG_GRAPH_OUTPUT + | TENSOR_FLAG_CONSTANT + | TENSOR_FLAG_STATE + ) + used: set[int] = set() + for index, (state_id, update_id) in enumerate(states): + update = registry.tensors[update_id] + if update.flags & shared or update_id in used: + copy = XTS3Tensor( + tensor_id=len(registry.tensors), + shape=tuple(update.shape), + dtype=XTS3DType.INT8, + quant_scale=update.quant_scale, + quant_zero_point=update.quant_zero_point, + ) + registry.tensors.append(copy) + elements = prod(update.shape) + segment = copy_segment(0, [elements], [1], 0, [1], 0, update, copy) + registry.extra_operations.append( + copy_operation([update_id], copy.tensor_id, [segment]) + ) + update_id = copy.tensor_id + states[index] = (state_id, update_id) + used.add(update_id) + return states + + def build_graph(exported_program) -> XTS3Graph: """Lower a PT2E-quantized ExportedProgram (or delegate partition) to an XTS3 int8 graph.""" graph_module = exported_program.graph_module @@ -1819,11 +1955,16 @@ def build_graph(exported_program) -> XTS3Graph: lowered.op_id = len(operations) operations.append(lowered) graph_inputs, graph_outputs = _mark_int8_graph_io(exported_program, registry) + states = _mark_int8_state(exported_program, registry) + for extra in registry.extra_operations: + extra.op_id = len(operations) + operations.append(extra) return XTS3Graph( tensors=registry.tensors, operations=operations, constant_data=bytes(registry.constant_data), - version=XTS3_VERSION, + version=XTS3_STATE_VERSION if states else XTS3_VERSION, + states=states, tensor_names={ tensor_id: node.name for node, tensor_id in registry.node_to_id.items() }, @@ -1870,7 +2011,7 @@ def _serialize_operation(operation: XTS3Operation, params_offset: int) -> bytes: ) -def check_runtime_limits(graph: XTS3Graph) -> None: +def check_runtime_limits(graph: XTS3Graph) -> None: # noqa: C901 """ Raise ValueError for graphs the device runtime would reject at init(): the same limits espnn_runtime/src/xts3_graph.cpp enforces. @@ -1882,6 +2023,13 @@ def check_runtime_limits(graph: XTS3Graph) -> None: f"Partition has {len(graph.inputs)} inputs and {len(graph.outputs)} " f"outputs; the ESP-NN runtime supports at most {MAX_GRAPH_IO} of each" ) + if graph.states and graph.version < XTS3_STATE_VERSION: + raise ValueError("Delegate-owned state needs XTS3 version 8") + if len(graph.states) > MAX_GRAPH_STATES: + raise ValueError( + f"Partition owns {len(graph.states)} mutable buffers; the ESP-NN " + f"runtime supports at most {MAX_GRAPH_STATES}" + ) if len(graph.tensors) > MAX_GRAPH_TENSORS: raise ValueError( f"Partition has {len(graph.tensors)} tensors; the ESP-NN runtime " @@ -1927,7 +2075,7 @@ def check_dataflow(graph: XTS3Graph) -> None: ready.update( tensor.tensor_id for tensor in graph.tensors - if tensor.flags & TENSOR_FLAG_CONSTANT + if tensor.flags & (TENSOR_FLAG_CONSTANT | TENSOR_FLAG_STATE) ) for index, operation in enumerate(graph.operations): for tensor_id in operation.inputs: @@ -1940,6 +2088,9 @@ def check_dataflow(graph: XTS3Graph) -> None: for tensor_id in graph.outputs: if tensor_id not in ready: raise ValueError(f"Graph output {tensor_id} is never written") + for state_id, update_id in graph.states: + if update_id not in ready: + raise ValueError(f"The new value of state {state_id} is never written") def serialize_graph(graph: XTS3Graph) -> bytes: @@ -1963,6 +2114,10 @@ def serialize_graph(graph: XTS3Graph) -> bytes: ) + struct.pack( f"<{len(graph.inputs) + len(graph.outputs)}I", *graph.inputs, *graph.outputs ) + if graph.version >= 8: + io_table += struct.pack(STATE_TABLE_HEADER, len(graph.states), 0) + for state_id, update_id in graph.states: + io_table += struct.pack("<2I", state_id, update_id) if graph.version >= 7: io_table += b"\x00" * ((-(HEADER_SIZE + len(io_table))) % 4) prefix = ( @@ -1993,7 +2148,7 @@ def serialize_graph(graph: XTS3Graph) -> bytes: def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 - """Parse and bounds-check an XTS3 v5, v6 or v7 int8 graph.""" + """Parse and bounds-check an XTS3 v5 to v8 int8 graph.""" if len(data) < HEADER_SIZE: raise ValueError("Graph blob is smaller than the header") magic, version, tensor_count, operation_count, parameter_size, constant_size = ( @@ -2018,10 +2173,25 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 ids = struct.unpack_from(f"<{input_count + output_count}I", data, offset) graph_inputs, graph_outputs = list(ids[:input_count]), list(ids[input_count:]) offset += 4 * (input_count + output_count) + states: list[tuple[int, int]] = [] + if version >= 8: + if offset + STATE_TABLE_HEADER_SIZE > len(data): + raise ValueError("Graph state table exceeds blob") + state_count, reserved = struct.unpack_from(STATE_TABLE_HEADER, data, offset) + if reserved: + raise ValueError("Graph state table reserved field is not zero") + offset += STATE_TABLE_HEADER_SIZE + if offset + 8 * state_count > len(data): + raise ValueError("Graph state table exceeds blob") + pairs = struct.unpack_from(f"<{2 * state_count}I", data, offset) + if any(tensor_id >= tensor_count for tensor_id in pairs): + raise ValueError("Graph state table names an unknown tensor") + states = list(zip(pairs[0::2], pairs[1::2])) + offset += 8 * state_count if version >= 7: padding = (-offset) % 4 if data[offset : offset + padding] != b"\x00" * padding: - raise ValueError("Graph IO table padding is not zero") + raise ValueError("Graph table padding is not zero") offset += padding if offset + tensor_count * TENSOR_SIZE > len(data): @@ -2114,11 +2284,14 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 version=version, inputs=graph_inputs, outputs=graph_outputs, + states=states, ) __all__ = [ "MAGIC", + "TENSOR_FLAG_STATE", + "XTS3_STATE_VERSION", "check_runtime_limits", "XTS3_VERSION", "XTS3_VERSIONS", diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index edc36dffe5f..507af470e6b 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -14,6 +14,7 @@ * * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] * [--alias INPUT:OUTPUT]... [--misalign] + * [--reset-before STEP]... * * Step s of instance k reads input i from IO_DIR/in__.bin and writes * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its @@ -23,8 +24,10 @@ * buffers through set_input and set_output_data_ptr, and --alias makes an * input and an output share one buffer: that input is then read from its file * at step 0 only and afterwards holds the output of the previous step. - * --misalign places every runner buffer one byte past a 16-byte boundary. At - * the end, the runner prints the delegates' memory as a STATS line. + * --misalign places every runner buffer one byte past a 16-byte boundary. + * --reset-before asks EspNNBackend, through its reset_states backend option, + * to restore its delegates' state before that step. At the end, the runner + * prints the delegates' memory as a STATS line. */ #include @@ -37,6 +40,8 @@ #include #include +#include +#include #include #include #include @@ -141,10 +146,13 @@ int main(int argc, char** argv) { bool share_io = false; size_t misalign = 0; std::vector> aliases; + std::vector resets; for (int a = 4; a < argc; ++a) { const std::string arg = argv[a]; if (arg == "--instances" && a + 1 < argc) { instance_count = std::atoi(argv[++a]); + } else if (arg == "--reset-before" && a + 1 < argc) { + resets.push_back(std::atoi(argv[++a])); } else if (arg == "--share-io") { share_io = true; } else if (arg == "--misalign") { @@ -228,6 +236,19 @@ int main(int argc, char** argv) { } for (int step = 0; step < steps; ++step) { + for (int reset : resets) { + if (reset != step) + continue; + executorch::runtime::BackendOptions<1> options; + if (options.set_option( + executorch::backends::espressif::kResetStatesOption, true) != + Error::Ok) + fail("BackendOptions::set_option"); + Error error = executorch::runtime::set_option( + executorch::backends::espressif::kEspNNBackendId, options.view()); + if (error != Error::Ok) + fail("set_option", static_cast(error)); + } for (int k = 0; k < instance_count; ++k) { Instance& instance = instances[k]; for (size_t i = 0; i < instance.inputs.size(); ++i) { @@ -266,13 +287,15 @@ int main(int argc, char** argv) { const auto stats = executorch::backends::espressif::delegate_stats(); std::printf( "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu " - "staging=%zu\n", + "staging=%zu states=%u states_in_place=%u\n", static_cast(stats.delegates), stats.arena_bytes, stats.scratch_bytes, stats.parameter_bytes, stats.table_bytes, - stats.staging_bytes); + stats.staging_bytes, + static_cast(stats.states), + static_cast(stats.states_updated_in_place)); return 0; } diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 3f0dc7a934e..fe05b31310e 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -41,6 +41,12 @@ test_int8_lowering as lowering_cases, test_int8_ops as op_cases, ) +from executorch.backends.espressif.test.test_mutable_buffers import ( + lower_input_as_state, + lower_stateful, + ShiftRegister, + StateReadLate, +) from executorch.exir import ExecutorchBackendConfig from executorch.exir.lowered_backend_module import LoweredBackendModule from executorch.exir.passes import MemoryPlanningPass @@ -452,3 +458,54 @@ def test_input_returned_as_output(tmp_path, channels_last, alias): expected = as_list(emulator(*inputs)) assert torch.equal(expected[1], inputs[1]) assert_matches(outputs[step], expected, channels_last) + + +def stateful_steps(count: int): + return random_steps([(1, 4, 1, 1)], count, seed=7) + + +@pytest.mark.parametrize( + "model_class, in_place", [(ShiftRegister, 1), (StateReadLate, 0)] +) +def test_delegate_owned_state_persists(tmp_path, model_class, in_place): + exported, program, report = lower_stateful(model_class()) + assert report["delegated_buffers"] == ["state"] + assert len(exported.graph_signature.user_inputs) == 1 + assert not exported.graph_signature.buffers_to_mutate + steps = stateful_steps(5) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + assert stats["states_in_place"] == in_place + # The state changes the outputs, so the check above depends on it. + replay = ProgramEmulator(exported) + stateful = [as_list(replay(*inputs))[0] for inputs in steps] + fresh = [as_list(ProgramEmulator(exported)(*inputs))[0] for inputs in steps] + assert not all(torch.equal(a, b) for a, b in zip(stateful, fresh)) + + +def test_state_updated_from_a_graph_input(tmp_path): + exported, program, _ = lower_input_as_state() + steps = stateful_steps(4) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + + +def test_state_reset_and_instances(tmp_path): + exported, program, _ = lower_stateful(ShiftRegister()) + steps = stateful_steps(6) + first, second = run_host( + tmp_path, program, steps, "--instances", 2, "--reset-before", 4 + )[0] + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + if step == 4: + emulator.reset_states() + expected = as_list(emulator(*inputs)) + assert_matches(first[step], expected) + assert_matches(second[step], expected) diff --git a/backends/espressif/test/test_mutable_buffers.py b/backends/espressif/test/test_mutable_buffers.py new file mode 100644 index 00000000000..7f00188b7e3 --- /dev/null +++ b/backends/espressif/test/test_mutable_buffers.py @@ -0,0 +1,226 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Lowering of mutable buffers (streaming state) with delegate_mutable_buffers: +which buffers the ESP-NN delegate owns, and the stateful host reference. +test_host_runtime.py runs the same models on the delegate. + + pytest backends/espressif/test/test_mutable_buffers.py +""" + +from __future__ import annotations + +import copy + +import pytest + +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif import to_espnn_edge +from executorch.backends.espressif.partition import espnn_partitioner as partitioner +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer +from executorch.backends.espressif.reference import ProgramEmulator +from executorch.backends.espressif.serialization import xts3 +from executorch.exir.lowered_backend_module import LoweredBackendModule +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +class ShiftRegister(torch.nn.Module): + """A streaming convolution over the last frames, kept in a buffer.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 2, 1)) + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + + def forward(self, x): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:]) + return self.conv(window) + + +class StateReadLate(ShiftRegister): + """Computes its new state before an operator reads the old one.""" + + def forward(self, x): + new_state = torch.cat([x, x], dim=2) + y = self.conv(torch.cat([self.state, x], dim=2)) + self.state.copy_(new_state) + return y + + +class InputAsState(torch.nn.Module): + """ + Explicit int8 Q/DQ (no PT2E): the new state is the quantized input itself, + a graph input of the delegate, which the delegate copies into the state. + """ + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + @staticmethod + def qdq(value, scale): + args = (scale, 0, -128, 127, torch.int8) + quantized = torch.ops.quantized_decomposed.quantize_per_tensor(value, *args) + return torch.ops.quantized_decomposed.dequantize_per_tensor(quantized, *args) + + def forward(self, x): + frame = self.qdq(x, 0.05) + y = self.qdq(self.qdq(self.state, 0.05) + frame, 0.1) + self.state.copy_(frame) + return y + + +def lower_input_as_state(): + program = torch.export.export( + InputAsState().eval(), (torch.zeros(1, 4, 1, 1),), strict=True + ) + edge, report = to_espnn_edge( + program, allow_cpu_fallback=False, delegate_mutable_buffers=True + ) + return edge.exported_program(), edge.to_executorch().buffer, report + + +class ScaledState(ShiftRegister): + """Its new state has other quantization parameters than the state.""" + + def forward(self, x): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:] * 0.5) + return self.conv(window) + + +def quantize_stateful(model): + shape = (1, 4, 1, 1) + generator = torch.Generator().manual_seed(5) + prepared = prepare_pt2e( + torch.export.export(model.eval(), (torch.zeros(shape),), strict=True).module(), + EspNNQuantizer(), + ) + for _ in range(16): + prepared(torch.randn(shape, generator=generator)) + converted = convert_pt2e(prepared) + # Calibration ran the model, so start again from the initial state. + converted.state.zero_() + return converted + + +def lower_stateful(model, delegate_mutable_buffers: bool = True, quantized=None): + converted = copy.deepcopy(quantized) if quantized else quantize_stateful(model) + program = torch.export.export(converted, (torch.zeros(1, 4, 1, 1),), strict=True) + edge, report = to_espnn_edge( + program, + allow_cpu_fallback=False, + delegate_mutable_buffers=delegate_mutable_buffers, + ) + return edge.exported_program(), edge.to_executorch().buffer, report + + +def delegate_graphs(exported) -> list[xts3.XTS3Graph]: + return [ + xts3.parse_graph(module.processed_bytes) + for _, module in exported.graph_module.named_children() + if isinstance(module, LoweredBackendModule) + ] + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def test_delegate_owns_the_state(): + exported, _, report = lower_stateful(ShiftRegister()) + assert report["delegated_buffers"] == ["state"] + assert len(exported.graph_signature.user_inputs) == 1 + assert not exported.graph_signature.buffers_to_mutate + (graph,) = delegate_graphs(exported) + assert graph.version == xts3.XTS3_STATE_VERSION + ((state, update),) = graph.states + assert graph.tensors[state].flags & xts3.TENSOR_FLAG_STATE + assert graph.inputs == [graph.inputs[0]] and state not in graph.inputs + assert xts3.parse_graph(xts3.serialize_graph(graph)).states == graph.states + + +def test_reference_tracks_the_quantized_model(): + model = quantize_stateful(ShiftRegister()) + exported, _, _ = lower_stateful(None, quantized=model) + emulator = ProgramEmulator(exported) + generator = torch.Generator().manual_seed(9) + for step in range(5): + x = torch.randn(1, 4, 1, 1, generator=generator) + if step == 3: + model.state.zero_() + emulator.reset_states() + torch.testing.assert_close(emulator(x), model(x), atol=0.05, rtol=0) + + +def test_mutable_buffers_stay_in_the_program_by_default(): + exported, _, report = lower_stateful(ShiftRegister(), False) + assert report["delegated_buffers"] == [] + assert list(exported.graph_signature.buffers_to_mutate.values()) == ["state"] + (graph,) = delegate_graphs(exported) + assert graph.version == xts3.XTS3_VERSION and not graph.states + + +def test_state_with_other_quantization_stays_in_the_program(): + exported, _, report = lower_stateful(ScaledState()) + assert report["delegated_buffers"] == [] + assert list(exported.graph_signature.buffers_to_mutate.values()) == ["state"] + + +@pytest.mark.parametrize( + "dtype, reader, writer, writer_scale, owned", + [ + (torch.float32, "quantize", "dequantize", 0.1, True), + (torch.int8, "dequantize", "quantize", 0.1, True), + (torch.int8, "dequantize", "quantize", 0.2, False), + (torch.int8, "quantize", "dequantize", 0.1, False), + ], +) +def test_int8_state_compatibility(dtype, reader, writer, writer_scale, owned): + ops = torch.ops.quantized_decomposed + graph = torch.fx.Graph() + buffer = graph.placeholder("buffer") + buffer.meta["val"] = torch.zeros(2, dtype=dtype) + read = graph.call_function( + getattr(ops, f"{reader}_per_tensor").default, + (buffer, 0.1, 0, -128, 127, torch.int8), + ) + new_value = graph.call_function( + getattr(ops, f"{writer}_per_tensor").default, + (read, writer_scale, 0, -128, 127, torch.int8), + ) + assert partitioner._int8_state_compatible(buffer, new_value) == owned + + +def test_state_updated_from_a_graph_input(): + exported, _, report = lower_input_as_state() + assert report["delegated_buffers"] == ["state"] + (graph,) = delegate_graphs(exported) + ((state, update),) = graph.states + # The runtime cannot write a graph input into the state's slot, so the + # update is a copy of it, made after every other operation. + assert update not in graph.inputs + copy = graph.operations[-1] + assert copy.opcode == xts3.XTS3OpCode.COPY and copy.outputs == [update] + assert copy.inputs == graph.inputs + emulator = ProgramEmulator(exported) + frames = [torch.full((1, 4, 1, 1), 0.1 * k) for k in (1, 2, 3)] + previous = torch.zeros(1, 4, 1, 1) + for frame in frames: + expected = InputAsState.qdq( + InputAsState.qdq(previous, 0.05) + InputAsState.qdq(frame, 0.05), 0.1 + ) + torch.testing.assert_close(emulator(frame), expected) + previous = frame diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py index 6b0e3184e3f..f5f036ae5f2 100644 --- a/backends/espressif/test/test_xts3_parser_host.py +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -88,6 +88,8 @@ def _layout(graph: xts3.XTS3Graph): io_table = xts3.IO_TABLE_HEADER_SIZE + 4 * ( len(graph.inputs) + len(graph.outputs) ) + if graph.version >= 8: + io_table += xts3.STATE_TABLE_HEADER_SIZE + 8 * len(graph.states) tensors = xts3.HEADER_SIZE + io_table if graph.version >= 7: tensors += -tensors % 4 @@ -224,3 +226,39 @@ def test_parser_rejects_misaligned_int32_constants(parser): "aligned": "accept", "misaligned": "reject", } + + +def test_parser_validates_state_tables(parser): + from executorch.backends.espressif.test import test_mutable_buffers as stateful + + exported, _, _ = stateful.lower_stateful(stateful.ShiftRegister()) + (graph,) = stateful.delegate_graphs(exported) + blob = xts3.serialize_graph(graph) + assert graph.version == xts3.XTS3_STATE_VERSION + ((state, update),) = graph.states + table = xts3.HEADER_SIZE + xts3.IO_TABLE_HEADER_SIZE + table += 4 * (len(graph.inputs) + len(graph.outputs)) + tensors, _ = _layout(graph) + flags = tensors + xts3.TENSOR_SIZE * state + 9 + blobs = { + "valid": blob, + "unpaired": _patch(blob, table, " Date: Sun, 15 Nov 2026 13:23:39 -0800 Subject: [PATCH 07/25] Document and show the streaming export recipe for the ESP-NN delegate The pieces a streaming keyword-spotting model on ESP32-S3 needs (int8, channels-last or trivially laid out inputs and outputs left to the application's buffers, mutable buffers owned by the delegate, and a quantization that matches the model's reference) were each documented on their own. export_streaming_model.py puts them together on a small model whose program has one int8 input, one int8 output, no CPU operators and no planned memory, and the README adds the quantization choices that matter for such models and the device-side pattern: bind once, execute per frame, reset the delegate's state on a detection. The host test runs the example's program for several frames without any staging copy. --- backends/espressif/README.md | 44 +++++++ backends/espressif/test/test_host_runtime.py | 27 ++++ examples/espressif/export_streaming_model.py | 127 +++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 examples/espressif/export_streaming_model.py diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 832ed470a82..e64d8a9f6a6 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -29,6 +29,7 @@ backends/espressif/ test/ pytest suite (lowering, operators, CPU fallback, parser) examples/espressif/ quantize_and_lower.py minimal quantize -> lower -> .pte example + export_streaming_model.py streaming model with delegate-owned state export_espnn_test_programs.py the test models as BundleIO programs for the runner camera_stream/ live camera classification on the ExecuTorch runtime ``` @@ -195,6 +196,49 @@ options.set_option("reset_states", true); executorch::runtime::set_option("EspNNBackend", options.view()); ``` +### Streaming keyword spotting + +[`examples/espressif/export_streaming_model.py`](../../examples/espressif/export_streaming_model.py) +puts the pieces above together for a streaming keyword-spotting model: one +frame of features in, one score out, the frame history in delegate-owned +buffers, int8 input and output in the application's memory. The resulting +program has one input and one output, no CPU operators, and needs no +planned memory. + +Quantization choices that matter for such models: + +- `EspNNQuantizer` observes activations with histograms. For small streaming + models, min/max ranges (`minmax_quantizer()` in the example) often track a + float model, or a TensorFlow Lite reference, more closely. +- To reproduce an existing int8 model exactly, give each activation the + reference model's scale and zero point: after `prepare_pt2e`, replace its + observers with `FixedQParamsObserver(scale, zero_point, ...)`. +- Weight scales are `max|w| / 127` per output channel; check that a layer with + very small weights is not clamped by the weight observer's `eps`. +- Calibrate on consecutive frames, so the histories see realistic values, and + zero the buffers before exporting, since calibration updated them. + +On the device, the application binds its buffers once and calls `execute()` +for each frame: + +```cpp +#include +#include + +// After Program::load() and load_method() with a method allocator sized by +// method_meta(), and no planned buffers (the program has none): +method.set_input(EValue(Tensor(&features_impl)), 0); // int8[1, 40, 1, 1] +method.set_output_data_ptr(&score, sizeof(score), 0); // int8[1, 1] +for (;;) { + compute_features(features); // writes the bound buffer in place + method.execute(); + if (score > threshold) { + on_detection(); + executorch::backends::espressif::reset_delegate_states(); + } +} +``` + ## Running on the ESP32-S3 You need ESP-IDF and an activated ESP-IDF shell. diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index fe05b31310e..5f4cbd83d9c 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -21,6 +21,7 @@ import os import subprocess +import sys from pathlib import Path import pytest @@ -509,3 +510,29 @@ def test_state_reset_and_instances(tmp_path): expected = as_list(emulator(*inputs)) assert_matches(first[step], expected) assert_matches(second[step], expected) + + +def test_streaming_example_runs_without_copies(tmp_path): + sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "examples/espressif")) + import export_streaming_model as example + + edge = example.lower(calibration_frames=32) + program = edge.to_executorch(example.executorch_config()).buffer + generator = torch.Generator().manual_seed(1) + steps = [ + [ + torch.randint( + -128, + 128, + (1, example.FEATURES, 1, 1), + generator=generator, + dtype=torch.int8, + ) + ] + for _ in range(6) + ] + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 2 and stats["staging"] == 0 diff --git a/examples/espressif/export_streaming_model.py b/examples/espressif/export_streaming_model.py new file mode 100644 index 00000000000..36a24903579 --- /dev/null +++ b/examples/espressif/export_streaming_model.py @@ -0,0 +1,127 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Export a streaming keyword-spotting model for the ESP-NN delegate, with the +streaming state owned by the delegate and the program's one input and one +output left to the application's buffers. + +Each call takes one frame of 40 features and returns one int8 score; the +convolutions see the last frames, kept in two buffers. Replace the random +weights and calibration frames with a trained model and real features. + + python examples/espressif/export_streaming_model.py --output kws.pte +""" + +from __future__ import annotations + +import argparse +from dataclasses import replace + +import torch +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass +from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output +from torchao.quantization.pt2e.observer import MinMaxObserver +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +FEATURES = 40 + + +class StreamingKws(torch.nn.Module): + """Two dilated temporal convolutions over a frame history, then a score.""" + + def __init__(self): + super().__init__() + self.register_buffer("history1", torch.zeros(1, FEATURES, 2, 1)) + self.register_buffer("history2", torch.zeros(1, 32, 4, 1)) + self.conv1 = torch.nn.Conv2d(FEATURES, 32, (3, 1)) + self.conv2 = torch.nn.Conv2d(32, 32, (3, 1), dilation=(2, 1)) + self.score = torch.nn.Linear(32, 1) + + def forward(self, frame): + window1 = torch.cat([self.history1, frame], dim=2) + self.history1.copy_(window1[:, :, 1:]) + hidden = torch.relu(self.conv1(window1)) + window2 = torch.cat([self.history2, hidden], dim=2) + self.history2.copy_(window2[:, :, 1:]) + hidden = torch.relu(self.conv2(window2)) + return torch.sigmoid(self.score(hidden.flatten(1))) + + +def minmax_quantizer() -> EspNNQuantizer: + """EspNNQuantizer with min/max activation ranges instead of histograms.""" + quantizer = EspNNQuantizer() + config = quantizer.global_config + minmax = MinMaxObserver.with_args(eps=2**-12) + return quantizer.set_global( + replace( + config, + input_activation=replace( + config.input_activation, observer_or_fake_quant_ctr=minmax + ), + output_activation=replace( + config.output_activation, observer_or_fake_quant_ctr=minmax + ), + ) + ) + + +def lower(calibration_frames: int = 200, seed: int = 0): + """Quantizes and lowers the model; returns the EdgeProgramManager.""" + torch.manual_seed(seed) + model = StreamingKws().eval() + example = (torch.zeros(1, FEATURES, 1, 1),) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), minmax_quantizer() + ) + # Calibrate on consecutive frames, so the histories see realistic values. + for _ in range(calibration_frames): + prepared(torch.randn(example[0].shape)) + quantized = convert_pt2e(prepared) + # Calibration updated the histories; export from the initial state. + quantized.history1.zero_() + quantized.history2.zero_() + + program = torch.export.export(quantized, example, strict=True) + # The application passes int8 features and reads an int8 score. + quantize_input(program, 0) + quantize_output(program, 0) + edge, report = to_espnn_edge( + program, allow_cpu_fallback=False, delegate_mutable_buffers=True + ) + print( + f"{report['partitions']} partition(s), {report['delegated_ops']} delegated " + f"operators, delegate-owned buffers: {report['delegated_buffers']}" + ) + return edge + + +def executorch_config() -> ExecutorchBackendConfig: + """Inputs and outputs live in the application's buffers.""" + return ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", default="streaming_kws_espnn.pte") + args = parser.parse_args() + with torch.no_grad(): + program = lower().to_executorch(executorch_config()).buffer + with open(args.output, "wb") as f: + f.write(program) + print(f"wrote {args.output} ({len(program)} bytes)") + + +if __name__ == "__main__": + main() From b14df9a4759f1b4f4c50018d910ff8b81568ab59 Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Mon, 16 Nov 2026 16:26:37 -0800 Subject: [PATCH 08/25] Fix ESP-NN arena reuse for boundary conversions Record per-boundary conversion requirements at export so float and NCHW I/O can reuse the activation arena while compatible int8 I/O remains directly bound. Preserve independent ExecuTorch input/output planning and arena-backed I/O for older exports. Keep conversion metadata independent per partition. Host regression tests cover arena sizing, mixed boundaries, all four planning settings, and legacy exports. Validation: 268 host tests passed, 11 skipped. Independent review and six additional native host probes passed. Authored with Codex (OpenAI). --- backends/espressif/README.md | 20 +-- backends/espressif/espnn_preprocess.py | 46 +++++++ .../include/espnn/int8_interpreter.h | 9 +- .../espnn_runtime/src/espnn_backend.cpp | 38 +++-- .../espnn_runtime/src/int8_interpreter.cpp | 29 +++- .../espressif/partition/espnn_partitioner.py | 6 +- .../espressif/test/host/espnn_host_runner.cpp | 40 +++--- backends/espressif/test/test_host_runtime.py | 130 +++++++++++++++++- 8 files changed, 267 insertions(+), 51 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index e64d8a9f6a6..beb8bdbf077 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -103,14 +103,18 @@ Calibrate on a few hundred real samples; random data gives useless scales. ### Inputs and outputs -The delegate keeps rank-4 tensors in NHWC order. It reads an int8 input and -writes an int8 output directly in the ExecuTorch tensor's memory when that -memory is 16-byte aligned and already in this layout: the tensor is not rank -4, has one channel or one pixel, or has a channels-last dim order. Anything -else (float, NCHW with several channels and pixels, or a misaligned buffer) -goes through a delegate-owned staging buffer and is quantized or transposed -on the way. The staging buffer is allocated by the first `execute()` that -needs it. +The delegate keeps rank-4 tensors in NHWC order. It uses an int8 input or +output directly when its buffer is 16-byte aligned and already in this +layout: the tensor is not rank 4, has one channel or one pixel, or has a +channels-last dim order. Each input and output is handled independently, +whether its buffer is memory-planned by ExecuTorch or supplied by the caller. + +Float tensors and NCHW tensors with several channels and pixels need dtype +or layout conversion. The exporter records which I/O slots need conversion, +so their int8 buffers can share the delegate's activation arena with +intermediate tensors. Older exports without this metadata use the arena for +all I/O. Misalignment or overlapping I/O may require a separate staging +buffer, allocated on the first `execute()` that needs it. For the smallest and fastest program, give the delegate int8, channels-last inputs and outputs and leave them out of ExecuTorch's memory plan, so that diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py index 1b7f7f32376..162ca4583cb 100644 --- a/backends/espressif/espnn_preprocess.py +++ b/backends/espressif/espnn_preprocess.py @@ -11,6 +11,8 @@ import logging from typing import List +import torch + from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult from executorch.exir.backend.compile_spec_schema import CompileSpec @@ -20,6 +22,41 @@ BACKEND_ID = "EspNNBackend" INT8_FORMAT_SPEC = CompileSpec("espnn_format", b"int8") +_IO_CONVERSION_SPEC_KEY = "espnn_io_conversion_slots" + + +def _io_conversion_slots(program, graph) -> int: + signature = program.graph_signature + nodes = {node.name: node for node in program.graph.nodes} + io_nodes = [ + node + for node in program.graph.nodes + if node.op == "placeholder" and node.name in signature.user_inputs + ] + [nodes[name] for name in signature.user_outputs] + io_ids = graph.inputs + graph.outputs + if len(io_nodes) != len(io_ids): + raise ValueError("ESP-NN boundary tensors do not match the serialized I/O") + slots = 0 + seen = set() + for slot, (node, tensor_id) in enumerate(zip(io_nodes, io_ids)): + if tensor_id in seen: + continue + seen.add(tensor_id) + tensor = node.meta["val"] + # Returned inputs need a protected copy even when their layout matches. + returned_input = slot < len(graph.inputs) and tensor_id in graph.outputs + if ( + returned_input + or tensor.dtype != torch.int8 + or ( + tensor.dim() == 4 + and tensor.shape[1] != 1 + and tensor.shape[2] * tensor.shape[3] != 1 + and not tensor.is_contiguous(memory_format=torch.channels_last) + ) + ): + slots |= 1 << slot + return slots class EspNNBackend(BackendDetails): @@ -48,6 +85,15 @@ def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResu ) graph = build_graph(edge_program) blob = serialize_graph(graph) + compile_specs[:] = [ + spec for spec in compile_specs if spec.key != _IO_CONVERSION_SPEC_KEY + ] + compile_specs.append( + CompileSpec( + _IO_CONVERSION_SPEC_KEY, + _io_conversion_slots(edge_program, graph).to_bytes(4, "little"), + ) + ) logger.info( "EspNN delegate: %d tensors, %d operations, %d constant bytes, %d byte blob", len(graph.tensors), diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 2776d915cab..5be390b5b27 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -49,11 +49,12 @@ struct Int8RuntimeOptions { /// operation. Saves memory, but the scratch then lives wherever the arena /// does (PSRAM for a large model). bool scratch_in_arena = false; - /// Leave graph inputs and outputs out of the activation arena: the caller - /// binds a device-layout (NHWC) buffer to each of them with bind_input() - /// and bind_output() before invoke(), and the operations read and write - /// those buffers directly. + /// Bind graph I/O to caller-provided device-layout (NHWC) buffers, outside + /// this interpreter's arena, except the slots selected by conversion_slots. bool external_io = false; + /// With external_io, include these I/O slots in the arena for conversion. + /// Bits select slots in input-then-output order, determined at export time. + uint32_t conversion_slots = 0; }; /** diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 70838b785e5..d4d6593c4c0 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -204,12 +204,10 @@ bool store_output( struct DelegateState { Int8Interpreter interpreter; + uint32_t conversion_slots = 0; bool profiled = false; uint32_t state_generation = 0; - // Device-layout copies of the inputs and outputs whose tensors the - // interpreter cannot use directly (float, NCHW, misaligned, an input that is - // also an output, or an output that overlaps an input), one slot per input - // then per output. Allocated, zeroed, by the first execute() that needs one. + // Alignment and aliasing fallbacks for I/O that is not already in the arena. int8_t* staging = nullptr; ~DelegateState() { @@ -223,6 +221,8 @@ struct DelegateState { size_t offset = 0; size_t total = 0; for (size_t k = 0; k < slots; ++k) { + if ((conversion_slots & (1U << k)) != 0) + continue; Int8TensorInfo info{}; if (k < inputs ? !interpreter.input_info(k, info) : !interpreter.output_info(k - inputs, info)) @@ -263,14 +263,25 @@ class EspNNBackendInterface final : public BackendInterface { Result init( BackendInitContext& context, FreeableBuffer* processed, - ArrayRef) const override { + ArrayRef compile_specs) const override { + Int8RuntimeOptions options{}; + for (const auto& spec : compile_specs) { + if (std::strcmp(spec.key, "espnn_io_conversion_slots") == 0) { + if (spec.value.buffer == nullptr || spec.value.nbytes != 4) + return Error::DelegateInvalidCompatibility; + options.external_io = true; + options.conversion_slots = + read_le_u32(static_cast(spec.value.buffer)); + } + } void* memory = context.get_runtime_allocator()->allocate( sizeof(DelegateState), alignof(DelegateState)); if (memory == nullptr) return Error::MemoryAllocationFailed; DelegateState* state = new (memory) DelegateState(); - Int8RuntimeOptions options{}; - options.external_io = true; + // Older exports use the arena for every I/O slot. + state->conversion_slots = + options.external_io ? options.conversion_slots : UINT32_MAX; #if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM options.copy_constants_to_psram = true; #endif @@ -329,8 +340,8 @@ class EspNNBackendInterface final : public BackendInterface { state->state_generation = generation; } } - // An aligned int8 tensor already in the device layout is used in place; - // any other goes through a staging slot. + // Conversion buffers share the activation arena; compatible I/O is bound + // directly, independently of ExecuTorch's input/output memory planning. Int8TensorInfo info{}; for (size_t i = 0; i < inputs; ++i) { const Layout layout = @@ -338,7 +349,10 @@ class EspNNBackendInterface final : public BackendInterface { ? tensor_layout(args[i]->toTensor(), info) : Layout::kInvalid; bool ok = layout != Layout::kInvalid; - if (ok) { + if (ok && (state->conversion_slots & (1U << i)) != 0) { + ok = load_input( + info, layout, args[i]->toTensor(), interpreter.input_data(i)); + } else if (ok) { const Tensor& tensor = args[i]->toTensor(); // An input that is also returned as an output is staged, so that // outputs written over the caller's input buffers cannot change it. @@ -374,6 +388,10 @@ class EspNNBackendInterface final : public BackendInterface { ? tensor_layout(args[inputs + o]->toTensor(), info) : Layout::kInvalid; bool ok = layout != Layout::kInvalid; + if (ok && (state->conversion_slots & (1U << (inputs + o))) != 0) { + staged_outputs |= 1U << o; + continue; + } // A graph tensor returned more than once is written where its first // output is bound, and a graph input returned as an output stays where // the input is bound; both are copied to their outputs afterwards. diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index aead7cea365..0f6107040a7 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -371,6 +371,12 @@ bool Int8Interpreter::load( reset(); return false; } + if ((static_cast(options_.conversion_slots) >> + (input_count_ + output_count_)) != 0) { + std::printf("ERROR: conversion mask selects a nonexistent I/O slot\n"); + reset(); + return false; + } for (uint32_t i = 0; i < input_count_; ++i) inputs_[i] = graph_input_id(graph_, i); for (uint32_t i = 0; i < output_count_; ++i) @@ -461,10 +467,21 @@ bool Int8Interpreter::plan_memory() { std::printf("ERROR: unable to allocate memory-planner tables\n"); return false; } - // With external_io, graph inputs and outputs live in the caller's buffers. - const uint8_t unplanned = TENSOR_FLAG_CONSTANT | - (options_.external_io ? TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT - : 0); + const auto unplanned = [&](uint32_t id) { + if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) != 0) + return true; + if (!options_.external_io || + (tensors_[id].flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) == 0) + return false; + for (size_t k = 0; k < input_count_ + output_count_; ++k) { + const uint32_t io = + k < input_count_ ? inputs_[k] : outputs_[k - input_count_]; + if (io == id && (options_.conversion_slots & (1U << k)) != 0) + return false; + } + return true; + }; for (uint32_t i = 0; i < tensor_count; ++i) { first[i] = (tensors_[i].flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE)) != 0 @@ -473,7 +490,7 @@ bool Int8Interpreter::plan_memory() { last[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); - planned[i] = (tensors_[i].flags & unplanned) != 0; + planned[i] = unplanned(i); offsets[i] = SIZE_MAX; } for (uint32_t e = tensor_count; e < entries; ++e) { @@ -562,7 +579,7 @@ bool Int8Interpreter::plan_memory() { offsets[update] = offsets[state]; } for (uint32_t i = 0; i < tensor_count; ++i) { - if ((tensors_[i].flags & unplanned) == 0 && offsets[i] == SIZE_MAX) { + if (!unplanned(i) && offsets[i] == SIZE_MAX) { std::printf( "ERROR: tensor %u is never produced or consumed\n", static_cast(i)); diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index 80934aef6ad..d4ff36c2d5c 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -413,7 +413,11 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: tag = f"espnn_{len(tags)}" for node in nodes: node.meta["delegation_tag"] = tag - tags[tag] = self.delegation_spec + # Preprocessing adds conversion metadata specific to this partition. + tags[tag] = DelegationSpec( + self.delegation_spec.backend_id, + list(self.delegation_spec.compile_specs), + ) delegated = { node for node in graph_module.graph.nodes diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index 507af470e6b..2b450f7995e 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -19,11 +19,11 @@ * Step s of instance k reads input i from IO_DIR/in__.bin and writes * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its * dim order. With --instances, the method is loaded N times and each step runs - * on every instance in turn. With --share-io (the program's inputs and - * outputs must not be memory planned), the tensors are bound once to runner - * buffers through set_input and set_output_data_ptr, and --alias makes an - * input and an output share one buffer: that input is then read from its file - * at step 0 only and afterwards holds the output of the previous step. + * on every instance in turn. Unplanned outputs are bound to runner buffers. + * With --share-io, inputs must be unplanned and are bound once through + * set_input. --alias makes an unplanned input and output share one buffer: + * that input is read from its file at step 0 only and afterwards holds the + * output of the previous step. * --misalign places every runner buffer one byte past a 16-byte boundary. * --reset-before asks EspNNBackend, through its reset_states backend option, * to restore its delegates' state before that step. At the end, the runner @@ -201,28 +201,36 @@ int main(int argc, char** argv) { instance.method = std::make_unique(std::move(method.get())); for (size_t i = 0; i < meta->num_inputs(); ++i) { - instance.inputs.push_back(std::make_unique( - meta->input_tensor_meta(i).get(), misalign)); + instance.inputs.push_back( + std::make_unique( + meta->input_tensor_meta(i).get(), misalign)); } for (size_t o = 0; o < meta->num_outputs(); ++o) { - instance.outputs.push_back(std::make_unique( - meta->output_tensor_meta(o).get(), misalign)); + instance.outputs.push_back( + std::make_unique( + meta->output_tensor_meta(o).get(), misalign)); } for (const auto& [input, output] : aliases) { if (input >= instance.inputs.size() || output >= instance.outputs.size() || + meta->input_tensor_meta(input).get().is_memory_planned() || + meta->output_tensor_meta(output).get().is_memory_planned() || instance.inputs[input]->nbytes != instance.outputs[output]->nbytes) fail("invalid --alias"); } - if (!share_io) - continue; - for (size_t i = 0; i < instance.inputs.size(); ++i) { - Error error = instance.method->set_input( - EValue(executorch::aten::Tensor(&instance.inputs[i]->impl)), i); - if (error != Error::Ok) - fail("set_input", static_cast(error)); + if (share_io) { + for (size_t i = 0; i < instance.inputs.size(); ++i) { + if (meta->input_tensor_meta(i).get().is_memory_planned()) + fail("--share-io requires unplanned inputs"); + Error error = instance.method->set_input( + EValue(executorch::aten::Tensor(&instance.inputs[i]->impl)), i); + if (error != Error::Ok) + fail("set_input", static_cast(error)); + } } for (size_t o = 0; o < instance.outputs.size(); ++o) { + if (meta->output_tensor_meta(o).get().is_memory_planned()) + continue; void* buffer = instance.outputs[o]->data; for (const auto& [input, output] : aliases) { if (output == o) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 5f4cbd83d9c..5e02e0db7c7 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -43,6 +43,7 @@ test_int8_ops as op_cases, ) from executorch.backends.espressif.test.test_mutable_buffers import ( + InputAsState, lower_input_as_state, lower_stateful, ShiftRegister, @@ -83,6 +84,26 @@ def forward(self, x): return torch.cat([conv(x) for conv in self.convs], dim=2) +class AddChain(torch.nn.Module): + def __init__(self, count): + super().__init__() + self.count = count + + def forward(self, x): + x = InputAsState.qdq(x, 0.1) + for _ in range(self.count): + x = InputAsState.qdq(x + x, 0.1) + return x + + +class CpuBetweenAdds(torch.nn.Module): + def forward(self, x): + x = InputAsState.qdq(x, 0.1) + x = InputAsState.qdq(x + x, 0.1) + x = InputAsState.qdq(torch.cumsum(x, dim=1), 0.1) + return InputAsState.qdq(x + x, 0.1) + + # Fully delegated programs, checked bit-exactly; transpose-select-stack leaves a # cat on the CPU and runs with the CPU-fallback programs below. single-conv has # no activation besides its input and output. @@ -106,7 +127,13 @@ def disable_grad(): yield -def lower(model, shapes, planned_io: bool = True, xts3_version: int | None = None): +def lower( + model, + shapes, + alloc_graph_input: bool = True, + alloc_graph_output: bool = True, + xts3_version: int | None = None, +): _, program = op_cases.quantize(model, shapes) edge, report = to_espnn_edge(program, allow_cpu_fallback=False) assert not report["cpu_ops"] @@ -118,7 +145,7 @@ def lower(model, shapes, planned_io: bool = True, xts3_version: int | None = Non module._processed_bytes = xts3.serialize_graph(graph) config = ExecutorchBackendConfig( memory_planning_pass=MemoryPlanningPass( - alloc_graph_input=planned_io, alloc_graph_output=planned_io + alloc_graph_input=alloc_graph_input, alloc_graph_output=alloc_graph_output ) ) return edge.exported_program(), edge.to_executorch(config).buffer @@ -206,7 +233,7 @@ def test_host_delegate_matches_reference(tmp_path, name, model, shapes): assert_matches(outputs[step], as_list(emulator(*inputs))) # With ESP-NN scratch planned in the arena, the arena also holds scratch. planned = sum( - reference.plan_activation_arena(delegate.graph, external_io=True)[0] + reference.plan_activation_arena(delegate.graph)[0] for delegate in emulator.delegates.values() ) assert stats["delegates"] == len(emulator.delegates) @@ -218,6 +245,90 @@ def test_host_delegate_matches_reference(tmp_path, name, model, shapes): assert stats["arena"] >= planned +@pytest.mark.parametrize("alloc_graph_input", [False, True]) +@pytest.mark.parametrize("alloc_graph_output", [False, True]) +@pytest.mark.parametrize( + "input_int8, output_int8, channels_last, legacy, adds, arena_slots", + [ + pytest.param(False, False, False, False, 2, 2, id="float-nchw"), + pytest.param(False, False, True, False, 2, 2, id="float-nhwc"), + pytest.param(True, True, False, False, 2, 2, id="int8-nchw"), + pytest.param(True, True, True, False, 2, 1, id="int8-nhwc"), + pytest.param(False, True, True, False, 1, 1, id="float-input-only"), + pytest.param(True, False, True, False, 1, 1, id="float-output-only"), + pytest.param(False, False, False, True, 2, 2, id="legacy-float"), + pytest.param(True, True, True, True, 2, 2, id="legacy-int8"), + ], +) +def test_conversion_buffers_share_the_arena( + tmp_path, + input_int8, + output_int8, + channels_last, + legacy, + adds, + arena_slots, + alloc_graph_input, + alloc_graph_output, +): + shape = (1, 16, 64, 64) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + sample = torch.zeros(shape).contiguous(memory_format=memory_format) + program = torch.export.export(AddChain(adds), (sample,), strict=True) + if input_int8: + quantize_input(program, 0) + if output_int8: + quantize_output(program, 0) + edge, _ = to_espnn_edge(program, allow_cpu_fallback=False) + if legacy: + for module in edge.exported_program().graph_module.children(): + if isinstance(module, LoweredBackendModule): + module.compile_specs[:] = [ + spec + for spec in module.compile_specs + if spec.key != "espnn_io_conversion_slots" + ] + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=alloc_graph_input, alloc_graph_output=alloc_graph_output + ) + ) + buffer = edge.to_executorch(config).buffer + _, loaded = run_host(tmp_path, buffer, []) + assert loaded["arena"] == arena_slots * sample.numel() + assert loaded["staging"] == 0 + steps = random_steps([shape], 2) + steps = [ + [ + ((x * 20).round().to(torch.int8) if input_int8 else x).contiguous( + memory_format=memory_format + ) + ] + for (x,) in steps + ] + (outputs,), stats = run_host(tmp_path, buffer, steps) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) + assert stats["arena"] == loaded["arena"] + assert stats["staging"] == 0 + + +def test_partitions_keep_independent_conversion_plans(tmp_path): + program = torch.export.export(CpuBetweenAdds(), (torch.zeros(1, 4),)) + quantize_input(program, 0) + edge, _ = to_espnn_edge(program) + steps = [[torch.full((1, 4), value, dtype=torch.int8)] for value in (1, 3)] + (outputs,), stats = run_host(tmp_path, edge.to_executorch().buffer, steps) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["delegates"] == 2 + # First partition: only its output needs conversion. Second: both I/O. + assert stats["arena"] == 16 + 32 + assert stats["staging"] == 0 + + def test_instances_run_independently(tmp_path): model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] exported, program = lower(model, shapes) @@ -233,7 +344,9 @@ def test_instances_run_independently(tmp_path): def test_unplanned_io_shared_with_the_caller(tmp_path): model, shapes = op_cases.TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)] - exported, program = lower(model, shapes, planned_io=False) + exported, program = lower( + model, shapes, alloc_graph_input=False, alloc_graph_output=False + ) steps = random_steps(shapes, 2) (outputs,), _ = run_host(tmp_path, program, steps, "--share-io") emulator = ProgramEmulator(exported) @@ -377,8 +490,13 @@ def test_device_layout_io_is_used_in_place(tmp_path, channels_last): (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) - # Only the NCHW state needs transposing, in and out. - assert (stats["staging"] == 0) == channels_last + # NCHW I/O uses the arena; channels-last I/O is bound directly. + assert stats["staging"] == 0 + if channels_last: + assert stats["arena"] == sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ) @pytest.mark.parametrize("channels_last", [False, True]) From 7ddab8c73c957595e5a6c248922df00de6ad4b54 Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Mon, 16 Nov 2026 16:57:04 -0800 Subject: [PATCH 09/25] Clarify ESP-NN state reset documentation Remove the whole-method timing guarantee for concurrent resets. Each delegate independently applies a reset before its next execution. Authored with Codex (OpenAI). --- .../espressif/espnn_runtime/include/espnn/espnn_backend.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index a349d896426..61642f0a448 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -79,9 +79,7 @@ void reset_delegate_stats(); * * Each loaded method instance has its own state, kept between executions * until this is called or the method is destroyed; loading the method again - * also starts from the initial state. Safe to call from any task, including - * while a method executes on another one: that execution finishes with the - * old state and the next one starts from the initial state. + * also starts from the initial state. Safe to call from any task. */ void reset_delegate_states(); From f13e923be7e8e381ab3105b94c36769a2c57b01f Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 18:15:18 -0800 Subject: [PATCH 10/25] Apply clang-format to the ESP-NN host runner --- backends/espressif/test/host/espnn_host_runner.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index 2b450f7995e..0fca33b19fa 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -201,14 +201,12 @@ int main(int argc, char** argv) { instance.method = std::make_unique(std::move(method.get())); for (size_t i = 0; i < meta->num_inputs(); ++i) { - instance.inputs.push_back( - std::make_unique( - meta->input_tensor_meta(i).get(), misalign)); + instance.inputs.push_back(std::make_unique( + meta->input_tensor_meta(i).get(), misalign)); } for (size_t o = 0; o < meta->num_outputs(); ++o) { - instance.outputs.push_back( - std::make_unique( - meta->output_tensor_meta(o).get(), misalign)); + instance.outputs.push_back(std::make_unique( + meta->output_tensor_meta(o).get(), misalign)); } for (const auto& [input, output] : aliases) { if (input >= instance.inputs.size() || From a3e993124798fbcf75b0547f8631f35a190744ea Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 18:34:40 -0800 Subject: [PATCH 11/25] Always copy ESP-NN delegate inputs The ESP32-S3 esp-nn kernels may load whole vectors past the end of a tensor they read; the general convolution reads up to 16 bytes past each row of its input. An input bound in place could be the caller's memory ending at the edge of mapped RAM. Graph inputs now always get slots in the delegate's activation arena and are copied into them, quantized or transposed if needed, and the arena gets 16 readable bytes after its last slot. The exporter records every input as a conversion slot, but the runtime plans inputs into the arena regardless of the recorded slots, so exports that left compatible inputs out of them still run. Int8Interpreter::bind_input() is removed so that nothing can bind one in place. Outputs keep their conversion slots and direct binding. Since inputs are copied before any operator runs, an output may be bound over an input's memory without staging, so the overlap check and the input staging slots go away. --- backends/espressif/README.md | 33 +++++----- backends/espressif/espnn_preprocess.py | 6 +- .../include/espnn/int8_interpreter.h | 25 ++++--- .../espnn_runtime/src/espnn_backend.cpp | 66 ++++--------------- .../espnn_runtime/src/int8_interpreter.cpp | 45 ++++++------- .../espressif/reference/esp_nn_reference.py | 7 +- .../test/host/espnn_host_interpreter.cpp | 25 +++---- backends/espressif/test/test_host_runtime.py | 53 +++++++++++++-- 8 files changed, 129 insertions(+), 131 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index beb8bdbf077..99cdd4e4dc2 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -103,22 +103,25 @@ Calibrate on a few hundred real samples; random data gives useless scales. ### Inputs and outputs -The delegate keeps rank-4 tensors in NHWC order. It uses an int8 input or -output directly when its buffer is 16-byte aligned and already in this -layout: the tensor is not rank 4, has one channel or one pixel, or has a -channels-last dim order. Each input and output is handled independently, -whether its buffer is memory-planned by ExecuTorch or supplied by the caller. - -Float tensors and NCHW tensors with several channels and pixels need dtype -or layout conversion. The exporter records which I/O slots need conversion, -so their int8 buffers can share the delegate's activation arena with -intermediate tensors. Older exports without this metadata use the arena for -all I/O. Misalignment or overlapping I/O may require a separate staging -buffer, allocated on the first `execute()` that needs it. +The delegate keeps rank-4 tensors in NHWC order. Inputs are always copied +into slots in the delegate's activation arena, quantized or transposed if +needed, because ESP-NN kernels may load whole vectors past the end of a +tensor they read. The delegate writes an int8 output directly when its +buffer is 16-byte aligned and already in this layout: the tensor is not +rank 4, has one channel or one pixel, or has a channels-last dim order. Each +output is handled independently, whether its buffer is memory-planned by +ExecuTorch or supplied by the caller. + +Float outputs and NCHW outputs with several channels and pixels need dtype +or layout conversion. The exporter records which outputs need conversion, +so their int8 buffers can share the arena with intermediate tensors. Older +exports without this metadata use the arena for all outputs. A misaligned +output goes through a separate staging buffer, allocated on the first +`execute()` that needs it. For the smallest and fastest program, give the delegate int8, channels-last inputs and outputs and leave them out of ExecuTorch's memory plan, so that -the application's own buffers are used: +the application's own buffers are used without conversion: ```python from executorch.exir import ExecutorchBackendConfig @@ -145,8 +148,8 @@ pte = edge.to_executorch(config).buffer The application then binds its buffers once, with `Method::set_input()` and `Method::set_output_data_ptr()`, and each step is a call to `Method::execute()`. An output may share memory with an input (for example, -to update a state tensor in place): the delegate detects the overlap and -writes that output through its staging buffer after all operators have run. +to update a state tensor in place), since the inputs are copied before any +operator runs. ### Streaming state diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py index 162ca4583cb..552d05451a8 100644 --- a/backends/espressif/espnn_preprocess.py +++ b/backends/espressif/espnn_preprocess.py @@ -43,10 +43,10 @@ def _io_conversion_slots(program, graph) -> int: continue seen.add(tensor_id) tensor = node.meta["val"] - # Returned inputs need a protected copy even when their layout matches. - returned_input = slot < len(graph.inputs) and tensor_id in graph.outputs + # Kernels may load whole vectors past the end of a tensor they read, so + # inputs are always copied into the arena. if ( - returned_input + slot < len(graph.inputs) or tensor.dtype != torch.int8 or ( tensor.dim() == 4 diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 5be390b5b27..1cab97a131e 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -49,11 +49,13 @@ struct Int8RuntimeOptions { /// operation. Saves memory, but the scratch then lives wherever the arena /// does (PSRAM for a large model). bool scratch_in_arena = false; - /// Bind graph I/O to caller-provided device-layout (NHWC) buffers, outside - /// this interpreter's arena, except the slots selected by conversion_slots. + /// Bind graph outputs to caller-provided device-layout (NHWC) buffers, + /// outside this interpreter's arena, except the slots selected by + /// conversion_slots. Graph inputs always have arena slots. bool external_io = false; - /// With external_io, include these I/O slots in the arena for conversion. - /// Bits select slots in input-then-output order, determined at export time. + /// With external_io, include these output slots in the arena for + /// conversion. Bits select slots in input-then-output order, determined at + /// export time. uint32_t conversion_slots = 0; }; @@ -131,17 +133,12 @@ class Int8Interpreter { bool output_info(size_t index, Int8TensorInfo& info) const; /** * With Int8RuntimeOptions::external_io, makes @p data, a device-layout - * buffer of input_info().bytes, the storage of input @p index until it is - * bound again. invoke() only reads it. + * buffer of output_info().bytes, the storage of output @p index until it is + * bound again. An output that is also a graph input cannot be bound: + * output_data() returns the input's slot. * - * @retval false external_io is off or @p index is out of range. - */ - bool bind_input(size_t index, const int8_t* data); - /** - * Same for output @p index. invoke() writes the buffer while it may still - * read the inputs, so it must not overlap a bound input. An output that is - * also a graph input cannot be bound: output_data() returns the input's - * buffer. + * @retval false external_io is off, @p index is out of range, or the output + * is a graph input. */ bool bind_output(size_t index, int8_t* data); /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index d4d6593c4c0..e1db1da3ae7 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -114,12 +114,6 @@ bool arena_aligned(const void* data) { return reinterpret_cast(data) % 16 == 0; } -bool overlaps(const void* a, size_t a_bytes, const void* b, size_t b_bytes) { - const uintptr_t a_begin = reinterpret_cast(a); - const uintptr_t b_begin = reinterpret_cast(b); - return a_begin < b_begin + b_bytes && b_begin < a_begin + a_bytes; -} - bool load_input( const Int8TensorInfo& info, Layout layout, @@ -207,7 +201,7 @@ struct DelegateState { uint32_t conversion_slots = 0; bool profiled = false; uint32_t state_generation = 0; - // Alignment and aliasing fallbacks for I/O that is not already in the arena. + // Copies of misaligned outputs that are not in the arena. int8_t* staging = nullptr; ~DelegateState() { @@ -220,12 +214,11 @@ struct DelegateState { const size_t slots = inputs + interpreter.output_count(); size_t offset = 0; size_t total = 0; - for (size_t k = 0; k < slots; ++k) { + for (size_t k = inputs; k < slots; ++k) { if ((conversion_slots & (1U << k)) != 0) continue; Int8TensorInfo info{}; - if (k < inputs ? !interpreter.input_info(k, info) - : !interpreter.output_info(k - inputs, info)) + if (!interpreter.output_info(k - inputs, info)) return nullptr; if (k == slot) offset = total; @@ -340,40 +333,20 @@ class EspNNBackendInterface final : public BackendInterface { state->state_generation = generation; } } - // Conversion buffers share the activation arena; compatible I/O is bound - // directly, independently of ExecuTorch's input/output memory planning. + // Inputs are copied into their arena slots, so outputs may be written over + // the caller's input buffers. Output conversion buffers share the arena; + // compatible outputs are bound directly, independently of ExecuTorch's + // output memory planning. Int8TensorInfo info{}; for (size_t i = 0; i < inputs; ++i) { const Layout layout = args[i]->isTensor() && interpreter.input_info(i, info) ? tensor_layout(args[i]->toTensor(), info) : Layout::kInvalid; - bool ok = layout != Layout::kInvalid; - if (ok && (state->conversion_slots & (1U << i)) != 0) { - ok = load_input( - info, layout, args[i]->toTensor(), interpreter.input_data(i)); - } else if (ok) { - const Tensor& tensor = args[i]->toTensor(); - // An input that is also returned as an output is staged, so that - // outputs written over the caller's input buffers cannot change it. - bool returned = false; - for (size_t o = 0; o < outputs; ++o) { - Int8TensorInfo output{}; - interpreter.output_info(o, output); - returned = returned || output.tensor_id == info.tensor_id; - } - if (tensor.scalar_type() == ScalarType::Char && - layout == Layout::kDevice && - arena_aligned(tensor.const_data_ptr()) && !returned) { - ok = interpreter.bind_input(i, tensor.const_data_ptr()); - } else { - int8_t* slot = state->staging_slot(i); - if (slot == nullptr) - return Error::MemoryAllocationFailed; - ok = load_input(info, layout, tensor, slot) && - interpreter.bind_input(i, slot); - } - } + const bool ok = + layout != Layout::kInvalid && + load_input( + info, layout, args[i]->toTensor(), interpreter.input_data(i)); if (!ok) { std::printf( "EspNN delegate: input %u has an unsupported dtype, shape or dim order\n", @@ -410,20 +383,9 @@ class EspNNBackendInterface final : public BackendInterface { staged_outputs |= 1U << o; } else if (ok) { Tensor tensor = args[inputs + o]->toTensor(); - // The operations may still read an input after writing an output, so - // an output that shares memory with an input is staged. - bool direct = tensor.scalar_type() == ScalarType::Char && - layout == Layout::kDevice && arena_aligned(tensor.const_data_ptr()); - for (size_t i = 0; direct && i < inputs; ++i) { - Int8TensorInfo input{}; - interpreter.input_info(i, input); - direct = !overlaps( - tensor.const_data_ptr(), - info.bytes, - interpreter.input_data(i), - input.bytes); - } - if (direct) { + if (tensor.scalar_type() == ScalarType::Char && + layout == Layout::kDevice && + arena_aligned(tensor.const_data_ptr())) { ok = interpreter.bind_output(o, tensor.mutable_data_ptr()); } else { int8_t* slot = state->staging_slot(inputs + o); diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 0f6107040a7..2151a6d18db 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -467,17 +467,20 @@ bool Int8Interpreter::plan_memory() { std::printf("ERROR: unable to allocate memory-planner tables\n"); return false; } + // Kernels may load whole vectors past the end of a tensor they read, which + // stays within the arena but not within a caller's buffer, so graph inputs + // are always planned. const auto unplanned = [&](uint32_t id) { if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) != 0) return true; if (!options_.external_io || (tensors_[id].flags & - (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) == 0) + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != + TENSOR_FLAG_GRAPH_OUTPUT) return false; - for (size_t k = 0; k < input_count_ + output_count_; ++k) { - const uint32_t io = - k < input_count_ ? inputs_[k] : outputs_[k - input_count_]; - if (io == id && (options_.conversion_slots & (1U << k)) != 0) + for (size_t o = 0; o < output_count_; ++o) { + if (outputs_[o] == id && + (options_.conversion_slots & (1U << (input_count_ + o))) != 0) return false; } return true; @@ -593,23 +596,28 @@ bool Int8Interpreter::plan_memory() { // Loading then needs at most the tensor offsets alongside the arena. plan_buffers.release(); - // With external_io, a graph whose every activation is an input or output - // needs no arena. + // With external_io, a graph whose every activation is an output needs no + // arena. arena_size_ = arena_size; if (arena_size_ == 0) return true; - arena_ = static_cast(allocate_preferring_internal( - arena_size_, - options_.prefer_internal_arena, - options_.internal_reserve_bytes, - &arena_internal_)); + // Readable bytes after the last slot, for kernels that load a whole vector + // past the end of a tensor. + const size_t allocated = arena_size_ + ARENA_ALIGNMENT; + arena_ = allocated < arena_size_ + ? nullptr + : static_cast(allocate_preferring_internal( + allocated, + options_.prefer_internal_arena, + options_.internal_reserve_bytes, + &arena_internal_)); if (arena_ == nullptr) { std::printf( "ERROR: unable to allocate %u-byte activation arena\n", static_cast(arena_size_)); return false; } - std::memset(arena_, 0, arena_size_); + std::memset(arena_, 0, allocated); for (uint32_t i = 0; i < tensor_count; ++i) { if (offsets[i] != SIZE_MAX) tensor_data_[i] = reinterpret_cast(arena_ + offsets[i]); @@ -1002,13 +1010,6 @@ int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { return tensor_data_[tensor_id]; } -bool Int8Interpreter::bind_input(size_t index, const int8_t* data) { - if (!loaded_ || !options_.external_io || index >= input_count_) - return false; - tensor_data_[inputs_[index]] = const_cast(data); - return true; -} - bool Int8Interpreter::bind_output(size_t index, int8_t* data) { if (!loaded_ || !options_.external_io || index >= output_count_ || (tensors_[outputs_[index]].flags & TENSOR_FLAG_GRAPH_INPUT) != 0) @@ -1936,10 +1937,6 @@ bool Int8Interpreter::invoke() { if (!loaded_) return false; if (options_.external_io) { - for (size_t i = 0; i < input_count_; ++i) { - if (tensor_data_[inputs_[i]] == nullptr) - return false; - } for (size_t o = 0; o < output_count_; ++o) { if (tensor_data_[outputs_[o]] == nullptr) return false; diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py index 266b7917115..0588a6e8aad 100644 --- a/backends/espressif/reference/esp_nn_reference.py +++ b/backends/espressif/reference/esp_nn_reference.py @@ -603,7 +603,8 @@ def plan_activation_arena( # noqa: C901 ) -> tuple[int, dict[int, int]]: """ Mirror of Int8Interpreter::plan_memory in the firmware. With external_io - (as EspNNBackend loads graphs), graph inputs and outputs are not planned. + (as EspNNBackend loads graphs), graph outputs that are not also graph + inputs are not planned. """ op_count = len(graph.operations) big = 1 << 30 @@ -612,7 +613,9 @@ def plan_activation_arena( # noqa: C901 sizes: dict[int, int] = {} io_flags = TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT for tensor in graph.tensors: - if tensor.is_constant or (external_io and tensor.flags & io_flags): + if tensor.is_constant or ( + external_io and (tensor.flags & io_flags) == TENSOR_FLAG_GRAPH_OUTPUT + ): continue first[tensor.tensor_id] = ( -1 if tensor.flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE) else big diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp index e3914d01a08..7a10f4ad830 100644 --- a/backends/espressif/test/host/espnn_host_interpreter.cpp +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -16,8 +16,8 @@ * * The graph is placed OFFSET bytes past a 16-byte boundary, so that a test can * load it from addresses with any alignment. Inputs and outputs are raw - * device-layout (NHWC) int8 files. With --external-io, inputs and outputs are - * bound to runner buffers, the outputs prefilled with 0x5a. + * device-layout (NHWC) int8 files. With --external-io, outputs are bound to + * runner buffers prefilled with 0x5a. */ #include @@ -85,21 +85,14 @@ int main(int argc, char** argv) { if (inputs.size() != interpreter.input_count() || outputs.size() != interpreter.output_count()) return fail("wrong number of inputs or outputs"); - std::vector> bound(inputs.size() + outputs.size()); - for (size_t k = 0; options.external_io && k < bound.size(); ++k) { + std::vector> bound(outputs.size()); + for (size_t o = 0; options.external_io && o < bound.size(); ++o) { Int8TensorInfo info{}; - if (k < inputs.size()) - interpreter.input_info(k, info); - else - interpreter.output_info(k - inputs.size(), info); - bound[k].resize(info.bytes / 8 + 2); - int8_t* buffer = reinterpret_cast(bound[k].data()); - if (k < inputs.size()) { - interpreter.bind_input(k, buffer); - } else { - std::memset(buffer, 0x5a, info.bytes); - interpreter.bind_output(k - inputs.size(), buffer); - } + interpreter.output_info(o, info); + bound[o].resize(info.bytes / 8 + 2); + int8_t* buffer = reinterpret_cast(bound[o].data()); + std::memset(buffer, 0x5a, info.bytes); + interpreter.bind_output(o, buffer); } for (size_t i = 0; i < inputs.size(); ++i) { Int8TensorInfo info{}; diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 5e02e0db7c7..329dfa51845 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -50,6 +50,7 @@ StateReadLate, ) from executorch.exir import ExecutorchBackendConfig +from executorch.exir.backend.compile_spec_schema import CompileSpec from executorch.exir.lowered_backend_module import LoweredBackendModule from executorch.exir.passes import MemoryPlanningPass from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output @@ -253,9 +254,9 @@ def test_host_delegate_matches_reference(tmp_path, name, model, shapes): pytest.param(False, False, False, False, 2, 2, id="float-nchw"), pytest.param(False, False, True, False, 2, 2, id="float-nhwc"), pytest.param(True, True, False, False, 2, 2, id="int8-nchw"), - pytest.param(True, True, True, False, 2, 1, id="int8-nhwc"), + pytest.param(True, True, True, False, 2, 2, id="int8-nhwc"), pytest.param(False, True, True, False, 1, 1, id="float-input-only"), - pytest.param(True, False, True, False, 1, 1, id="float-output-only"), + pytest.param(True, False, True, False, 1, 2, id="float-output-only"), pytest.param(False, False, False, True, 2, 2, id="legacy-float"), pytest.param(True, True, True, True, 2, 2, id="legacy-int8"), ], @@ -324,8 +325,49 @@ def test_partitions_keep_independent_conversion_plans(tmp_path): for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs))) assert stats["delegates"] == 2 - # First partition: only its output needs conversion. Second: both I/O. - assert stats["arena"] == 16 + 32 + # Inputs are always copied; both partitions' outputs need conversion. + assert stats["arena"] == 32 + 32 + assert stats["staging"] == 0 + + +def test_inputs_are_copied_for_exports_that_bound_them(tmp_path): + # Exports that recorded no conversion slots for compatible inputs. + sample = torch.zeros(1, 16, 8, 8).contiguous(memory_format=torch.channels_last) + program = torch.export.export(AddChain(2), (sample,), strict=True) + quantize_input(program, 0) + quantize_output(program, 0) + edge, _ = to_espnn_edge(program, allow_cpu_fallback=False) + for module in edge.exported_program().graph_module.children(): + if isinstance(module, LoweredBackendModule): + module.compile_specs[:] = [ + ( + CompileSpec(spec.key, bytes(4)) + if spec.key == "espnn_io_conversion_slots" + else spec + ) + for spec in module.compile_specs + ] + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) + ) + generator = torch.Generator().manual_seed(2) + steps = [ + [ + torch.randint( + -128, 128, sample.shape, generator=generator, dtype=torch.int8 + ).contiguous(memory_format=torch.channels_last) + ] + for _ in range(2) + ] + (outputs,), stats = run_host( + tmp_path, edge.to_executorch(config).buffer, steps, "--share-io" + ) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), True) + assert stats["arena"] == 2 * sample.numel() assert stats["staging"] == 0 @@ -490,7 +532,8 @@ def test_device_layout_io_is_used_in_place(tmp_path, channels_last): (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) - # NCHW I/O uses the arena; channels-last I/O is bound directly. + # Inputs and NCHW outputs use the arena; channels-last outputs are bound + # directly. assert stats["staging"] == 0 if channels_last: assert stats["arena"] == sum( From 7e0e4143115cc8f085b4971885d23f441d8465f0 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 18:35:58 -0800 Subject: [PATCH 12/25] Fall back to portable kernels where esp-nn S3 leaves outputs unwritten Two esp-nn ESP32-S3 kernels return without writing all of their output, so the delegate produced whatever the output buffer held before. The max pool kernel handles only a multiple of four channels and writes nothing otherwise, and the pointwise convolution passes its number of output positions as a uint16_t width, leaving every position past 65535 unwritten. Both cases now use the ANSI kernels. invoke() also clears an output bound to memory outside the arena once it holds the invocation lock, so that a kernel leaving part of its output unwritten cannot let the caller's previous contents show through. The emulator and host suites gain a max pool over six channels. --- .../espnn_runtime/src/int8_interpreter.cpp | 26 ++++++++++++++++--- backends/espressif/test/test_int8_ops.py | 12 +++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 2151a6d18db..465bb52f78c 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -209,6 +209,15 @@ int8_t quantize_value(float value, float inverse_scale, int32_t zero_point) { static_cast(std::nearbyintf(scaled)) + zero_point); } +// esp-nn's ESP32-S3 pointwise convolution passes the number of output +// positions as a uint16_t width and leaves the rest unwritten. +bool pointwise_beyond_uint16( + const data_dims_t& filter, + const data_dims_t& output) { + return filter.width == 1 && filter.height == 1 && + static_cast(output.width) * output.height > UINT16_MAX; +} + } // namespace // Fields that no operation uses together share storage: the input list @@ -1199,7 +1208,7 @@ bool Int8Interpreter::run_operation(uint32_t index) { run_generic_conv(s, input, output); return true; } - if (reference) { + if (reference || pointwise_beyond_uint16(s.filter_dims, s.output_dims)) { esp_nn_conv_s8_ansi( &s.input_dims, input, @@ -1353,7 +1362,9 @@ bool Int8Interpreter::run_operation(uint32_t index) { sw = s.pool[3]; const uint16_t ph = s.pool[4], pw = s.pool[5]; if (s.opcode == OpCode::MAX_POOL2D) { - if (reference) { + // The ESP32-S3 kernel takes a multiple of four channels and writes + // nothing otherwise. + if (reference || channels % 4 != 0) { esp_nn_max_pool_s8_ansi( input, in_w, @@ -1647,7 +1658,7 @@ void Int8Interpreter::run_dilated_by_phases( data_dims_t tile_dims{tile_w, tile_h, in_c, 1}; data_dims_t result_dims{cols, rows, out_c, 1}; if (s.opcode == OpCode::CONV2D) { - if (reference) { + if (reference || pointwise_beyond_uint16(s.filter_dims, result_dims)) { esp_nn_conv_s8_ansi( &tile_dims, tile, @@ -1943,6 +1954,15 @@ bool Int8Interpreter::invoke() { } } InvokeLock lock; + // A kernel that leaves part of its output unwritten must not leave a + // caller's previous contents in a bound output. + const uintptr_t arena = reinterpret_cast(arena_); + for (size_t o = 0; options_.external_io && o < output_count_; ++o) { + int8_t* data = tensor_data_[outputs_[o]]; + const uintptr_t address = reinterpret_cast(data); + if (address < arena || address - arena >= arena_size_) + std::memset(data, 0, tensor_data_size(tensors_[outputs_[o]])); + } if (scratch_ != nullptr) { esp_nn_set_conv_scratch_buf(scratch_); esp_nn_set_depthwise_conv_scratch_buf(scratch_); diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py index ee33d41fd23..e70ef48cf90 100644 --- a/backends/espressif/test/test_int8_ops.py +++ b/backends/espressif/test/test_int8_ops.py @@ -210,6 +210,17 @@ def forward(self, image, features): return mask, logits +class OddChannelPool(nn.Module): + """Max pooling over a channel count that is not a multiple of four.""" + + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 6, 3, padding=1) + + def forward(self, x): + return F.max_pool2d(F.relu(self.conv(x)), 2) + + class TransposeSelect(nn.Module): def __init__(self): super().__init__() @@ -263,6 +274,7 @@ def forward(self, x): ("broadcast-pad-split-cat", BroadcastPadSplit(), [(1, 4, 6, 6)], 2), ("two-inputs-two-outputs", TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)], 2), ("transpose-select-stack", TransposeSelect(), [(1, 3, 4, 5)], 2), + ("max-pool-six-channels", OddChannelPool(), [(1, 3, 8, 8)], 2), ] From 6a807c5856ce2578db5c85dbf59cb6fe8b15d79e Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 19:17:18 -0800 Subject: [PATCH 13/25] Keep ESP-NN delegate outputs that operators read in the arena An output that a later operator reads was still bound to the caller's buffer, so the ESP32-S3 kernels' vector loads past the end of a tensor could reach past that buffer, and an application that bound another output over it would change what the operator read. With external I/O, Int8Interpreter now gives every output that an operation reads an arena slot, whatever the export recorded, and reports those slots through conversion_slots(), so EspNNBackend copies such outputs out after the last operator like any other output in the arena. bind_output() refuses any output that has an arena slot. The host runner gains --share-output, and a host test binds two outputs to one buffer while an operator still reads the first. --- backends/espressif/README.md | 11 +++--- .../include/espnn/int8_interpreter.h | 16 ++++++--- .../espnn_runtime/src/espnn_backend.cpp | 7 ++-- .../espnn_runtime/src/int8_interpreter.cpp | 24 ++++++++++--- .../espressif/reference/esp_nn_reference.py | 11 ++++-- .../espressif/test/host/espnn_host_runner.cpp | 26 ++++++++++++-- backends/espressif/test/test_host_runtime.py | 36 ++++++++++++++++++- 7 files changed, 108 insertions(+), 23 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 99cdd4e4dc2..beb3c3bf0b7 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -106,11 +106,12 @@ Calibrate on a few hundred real samples; random data gives useless scales. The delegate keeps rank-4 tensors in NHWC order. Inputs are always copied into slots in the delegate's activation arena, quantized or transposed if needed, because ESP-NN kernels may load whole vectors past the end of a -tensor they read. The delegate writes an int8 output directly when its -buffer is 16-byte aligned and already in this layout: the tensor is not -rank 4, has one channel or one pixel, or has a channels-last dim order. Each -output is handled independently, whether its buffer is memory-planned by -ExecuTorch or supplied by the caller. +tensor they read. For the same reason, an output that an operator reads keeps +its arena slot and is copied out at the end. The delegate writes any other +int8 output directly when its buffer is 16-byte aligned and already in this +layout: the tensor is not rank 4, has one channel or one pixel, or has a +channels-last dim order. Each output is handled independently, whether its +buffer is memory-planned by ExecuTorch or supplied by the caller. Float outputs and NCHW outputs with several channels and pixels need dtype or layout conversion. The exporter records which outputs need conversion, diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 1cab97a131e..74239ccb8b7 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -51,7 +51,8 @@ struct Int8RuntimeOptions { bool scratch_in_arena = false; /// Bind graph outputs to caller-provided device-layout (NHWC) buffers, /// outside this interpreter's arena, except the slots selected by - /// conversion_slots. Graph inputs always have arena slots. + /// conversion_slots. Graph inputs and outputs that an operation reads + /// always have arena slots. bool external_io = false; /// With external_io, include these output slots in the arena for /// conversion. Bits select slots in input-then-output order, determined at @@ -134,11 +135,12 @@ class Int8Interpreter { /** * With Int8RuntimeOptions::external_io, makes @p data, a device-layout * buffer of output_info().bytes, the storage of output @p index until it is - * bound again. An output that is also a graph input cannot be bound: - * output_data() returns the input's slot. + * bound again. An output with an arena slot cannot be bound: output_data() + * returns the slot. * * @retval false external_io is off, @p index is out of range, or the output - * is a graph input. + * has an arena slot (it is a graph input, an operation reads it, or + * conversion_slots selects it). */ bool bind_output(size_t index, int8_t* data); /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. @@ -183,6 +185,11 @@ class Int8Interpreter { const GraphView& graph() const { return graph_; } + /// With external_io, the I/O slots given arena slots: conversion_slots plus + /// every output that an operation reads, in the same bit order. + uint32_t conversion_slots() const { + return options_.conversion_slots; + } /// Duration of the last invoke() in microseconds. int64_t last_invoke_us() const { return last_invoke_us_; @@ -237,6 +244,7 @@ class Int8Interpreter { void rebase_constants(const uint8_t* old_base, size_t size); bool tensor_info(uint32_t tensor_id, Int8TensorInfo& info) const; int8_t* activation(uint32_t tensor_id) const; + bool in_arena(const int8_t* data) const; const int8_t* operand(uint32_t tensor_id) const; void run_generic_conv(const OpState& s, const int8_t* input, int8_t* output) const; diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index e1db1da3ae7..5c17c29329e 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -272,9 +272,6 @@ class EspNNBackendInterface final : public BackendInterface { if (memory == nullptr) return Error::MemoryAllocationFailed; DelegateState* state = new (memory) DelegateState(); - // Older exports use the arena for every I/O slot. - state->conversion_slots = - options.external_io ? options.conversion_slots : UINT32_MAX; #if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM options.copy_constants_to_psram = true; #endif @@ -294,6 +291,10 @@ class EspNNBackendInterface final : public BackendInterface { state->~DelegateState(); return Error::DelegateInvalidCompatibility; } + // Older exports use the arena for every I/O slot. + state->conversion_slots = options.external_io + ? state->interpreter.conversion_slots() + : UINT32_MAX; if (state->interpreter.constants_copied()) processed->Free(); state->state_generation = diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 465bb52f78c..0481746a459 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -390,6 +390,18 @@ bool Int8Interpreter::load( inputs_[i] = graph_input_id(graph_, i); for (uint32_t i = 0; i < output_count_; ++i) outputs_[i] = graph_output_id(graph_, i); + // Like graph inputs, outputs that an operation reads keep their arena slots + // (see plan_memory()). + for (uint32_t op = 0; options_.external_io && op < op_count; ++op) { + OperationRecord record{}; + read_operation(graph_, op, record); + for (uint8_t j = 0; j < record.num_inputs; ++j) { + for (uint32_t o = 0; o < output_count_; ++o) { + if (outputs_[o] == record.inputs[j]) + options_.conversion_slots |= 1U << (input_count_ + o); + } + } + } // Operations first: the planner places each operation's ESP-NN scratch. if (!prepare_operations() || !plan_memory()) { @@ -1015,13 +1027,19 @@ void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { } } +bool Int8Interpreter::in_arena(const int8_t* data) const { + const uintptr_t address = reinterpret_cast(data); + const uintptr_t arena = reinterpret_cast(arena_); + return address >= arena && address - arena < arena_size_; +} + int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { return tensor_data_[tensor_id]; } bool Int8Interpreter::bind_output(size_t index, int8_t* data) { if (!loaded_ || !options_.external_io || index >= output_count_ || - (tensors_[outputs_[index]].flags & TENSOR_FLAG_GRAPH_INPUT) != 0) + in_arena(tensor_data_[outputs_[index]])) return false; tensor_data_[outputs_[index]] = data; return true; @@ -1956,11 +1974,9 @@ bool Int8Interpreter::invoke() { InvokeLock lock; // A kernel that leaves part of its output unwritten must not leave a // caller's previous contents in a bound output. - const uintptr_t arena = reinterpret_cast(arena_); for (size_t o = 0; options_.external_io && o < output_count_; ++o) { int8_t* data = tensor_data_[outputs_[o]]; - const uintptr_t address = reinterpret_cast(data); - if (address < arena || address - arena >= arena_size_) + if (!in_arena(data)) std::memset(data, 0, tensor_data_size(tensors_[outputs_[o]])); } if (scratch_ != nullptr) { diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py index 0588a6e8aad..cf35bbbaba6 100644 --- a/backends/espressif/reference/esp_nn_reference.py +++ b/backends/espressif/reference/esp_nn_reference.py @@ -603,8 +603,8 @@ def plan_activation_arena( # noqa: C901 ) -> tuple[int, dict[int, int]]: """ Mirror of Int8Interpreter::plan_memory in the firmware. With external_io - (as EspNNBackend loads graphs), graph outputs that are not also graph - inputs are not planned. + (as EspNNBackend loads graphs), graph outputs that are neither graph inputs + nor read by an operation are not planned. """ op_count = len(graph.operations) big = 1 << 30 @@ -612,9 +612,14 @@ def plan_activation_arena( # noqa: C901 last: dict[int, int] = {} sizes: dict[int, int] = {} io_flags = TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT + read = { + tensor_id for operation in graph.operations for tensor_id in operation.inputs + } for tensor in graph.tensors: if tensor.is_constant or ( - external_io and (tensor.flags & io_flags) == TENSOR_FLAG_GRAPH_OUTPUT + external_io + and (tensor.flags & io_flags) == TENSOR_FLAG_GRAPH_OUTPUT + and tensor.tensor_id not in read ): continue first[tensor.tensor_id] = ( diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index 0fca33b19fa..8e5e302b448 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -13,8 +13,9 @@ * backends/espressif/test/test_host_runtime.py. * * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] - * [--alias INPUT:OUTPUT]... [--misalign] - * [--reset-before STEP]... + * [--alias INPUT:OUTPUT]... + * [--share-output FIRST:SECOND]... + * [--misalign] [--reset-before STEP]... * * Step s of instance k reads input i from IO_DIR/in__.bin and writes * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its @@ -23,7 +24,8 @@ * With --share-io, inputs must be unplanned and are bound once through * set_input. --alias makes an unplanned input and output share one buffer: * that input is read from its file at step 0 only and afterwards holds the - * output of the previous step. + * output of the previous step. --share-output writes unplanned output SECOND + * into output FIRST's buffer. * --misalign places every runner buffer one byte past a 16-byte boundary. * --reset-before asks EspNNBackend, through its reset_states backend option, * to restore its delegates' state before that step. At the end, the runner @@ -146,6 +148,7 @@ int main(int argc, char** argv) { bool share_io = false; size_t misalign = 0; std::vector> aliases; + std::vector> shared_outputs; std::vector resets; for (int a = 4; a < argc; ++a) { const std::string arg = argv[a]; @@ -162,6 +165,11 @@ int main(int argc, char** argv) { if (std::sscanf(argv[++a], "%u:%u", &input, &output) != 2) fail("--alias takes INPUT:OUTPUT"); aliases.emplace_back(input, output); + } else if (arg == "--share-output" && a + 1 < argc) { + unsigned first = 0, second = 0; + if (std::sscanf(argv[++a], "%u:%u", &first, &second) != 2) + fail("--share-output takes FIRST:SECOND"); + shared_outputs.emplace_back(first, second); } else { fail(("unknown argument " + arg).c_str()); } @@ -216,6 +224,14 @@ int main(int argc, char** argv) { instance.inputs[input]->nbytes != instance.outputs[output]->nbytes) fail("invalid --alias"); } + for (const auto& [first, second] : shared_outputs) { + if (first >= instance.outputs.size() || + second >= instance.outputs.size() || + meta->output_tensor_meta(first).get().is_memory_planned() || + meta->output_tensor_meta(second).get().is_memory_planned() || + instance.outputs[first]->nbytes != instance.outputs[second]->nbytes) + fail("invalid --share-output"); + } if (share_io) { for (size_t i = 0; i < instance.inputs.size(); ++i) { if (meta->input_tensor_meta(i).get().is_memory_planned()) @@ -234,6 +250,10 @@ int main(int argc, char** argv) { if (output == o) buffer = instance.inputs[input]->data; } + for (const auto& [first, second] : shared_outputs) { + if (second == o) + buffer = instance.outputs[first]->data; + } Error error = instance.method->set_output_data_ptr( buffer, instance.outputs[o]->nbytes, o); if (error != Error::Ok) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 329dfa51845..e45a65960a8 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -479,6 +479,18 @@ def forward(self, x, state): return output, new_state +class ChainedOutputs(StreamingConv): + """Returns a tensor and two functions of it, computed one after the other.""" + + def forward(self, x, state): + a = self.conv(torch.cat([state, x], dim=2)).clone( + memory_format=self.memory_format + ) + b = torch.relu(a).clone(memory_format=self.memory_format) + c = torch.sigmoid(a).clone(memory_format=self.memory_format) + return a, b, c + + def lower_streaming(channels_last: bool, model_class=StreamingConv, patch=None): """ int8 inputs and outputs, not memory planned, optionally channels-last. @@ -500,8 +512,9 @@ def sample(): for _ in range(16): prepared(*sample()) program = torch.export.export(convert_pt2e(prepared), example, strict=True) - for index in range(2): + for index in range(len(program.graph_signature.user_inputs)): quantize_input(program, index) + for index in range(len(program.graph_signature.user_outputs)): quantize_output(program, index) edge, report = to_espnn_edge(program, allow_cpu_fallback=False) assert not report["cpu_ops"] @@ -593,6 +606,27 @@ def test_misaligned_buffers_are_staged(tmp_path): assert stats["staging"] > 0 +@pytest.mark.parametrize("channels_last", [False, True]) +def test_outputs_read_by_operators_keep_their_slots(tmp_path, channels_last): + # Output 1 is written over output 0's buffer before an operator reads + # output 0 for output 2, so output 0 must stay in its arena slot. + exported, program, _ = lower_streaming(channels_last, ChainedOutputs) + emulator = ProgramEmulator(exported) + steps = int8_steps(2, channels_last) + (outputs,), stats = run_host( + tmp_path, program, steps, "--share-io", "--share-output", "0:1" + ) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert_matches(outputs[step][2:], expected[2:], channels_last) + assert stats["staging"] == 0 + if channels_last: + assert stats["arena"] == sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ) + + def return_state_input(graph): """Makes the graph return its state input instead of its new state.""" state_in = graph.inputs[1] From 2776db9c77430bd82bc4551bc43b6759c74074e8 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:35:42 -0800 Subject: [PATCH 14/25] Build the ESP-NN host tests against the device's esp-nn The host build fetched esp-nn v1.5.0 while idf_component.yml pins v1.4.1 for the device, so the host tests checked kernels that no device build runs. Fetch v1.4.1 by commit, matching the pin. --- backends/espressif/espnn_runtime/host/espnn_host.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends/espressif/espnn_runtime/host/espnn_host.cmake b/backends/espressif/espnn_runtime/host/espnn_host.cmake index c4bda908409..686a612889a 100644 --- a/backends/espressif/espnn_runtime/host/espnn_host.cmake +++ b/backends/espressif/espnn_runtime/host/espnn_host.cmake @@ -12,8 +12,9 @@ include(FetchContent) +# The version that ../../idf_component.yml pins for the device build. set(ESPNN_GIT_TAG - "ace42ac7721119cd543fc40ffe719e601fa8f20f" # v1.5.0 + "d9be5422a79f4115e4275452dcaba3fe363a697d" # v1.4.1 CACHE STRING "esp-nn commit fetched when ESPNN_LOCAL_PATH is not set" ) set(ESPNN_LOCAL_PATH From 807df86a4ead3f5e1b7d2879d3a6b3bbd1ceb546 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:35:53 -0800 Subject: [PATCH 15/25] Read the cycle counter for the ESP-NN delegate's cooperative yield Without profiling, invoke() read the timer every sixteen operators to decide whether to yield, so the longest stretch without a yield depended on how slow those operators were. It now reads the cycle counter, a few cycles, after every operator, and the timer only once 2^20 cycles (4 ms at 240 MHz) have passed. A yield still comes within a second plus one operator. On an ESP32-S3 the streaming keyword model runs at 440 us per step this way, against 466 us with a timer read per operator. The host build gets an esp_cpu.h stand-in. --- backends/espressif/espnn_runtime/Kconfig | 2 +- .../espnn_runtime/host/include/esp_cpu.h | 24 +++++++++++++++++++ .../include/espnn/int8_interpreter.h | 2 +- .../espnn_runtime/src/int8_interpreter.cpp | 18 +++++++++----- 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 backends/espressif/espnn_runtime/host/include/esp_cpu.h diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index 6cff21ee7f0..e09ef8fe8d9 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -34,6 +34,6 @@ menu "ExecuTorch ESP-NN backend" help Times every operator, which adds a timer read per operator to each execute. Without it, the delegate reads the timer only - every few operators, for its cooperative yield. + every few milliseconds, for its cooperative yield. endmenu diff --git a/backends/espressif/espnn_runtime/host/include/esp_cpu.h b/backends/espressif/espnn_runtime/host/include/esp_cpu.h new file mode 100644 index 00000000000..9f3e889760b --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_cpu.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for esp_cpu_get_cycle_count(): a 240 MHz count of a monotonic +// clock, wrapping like the 32-bit cycle counter. + +#pragma once + +#include +#include + +inline uint32_t esp_cpu_get_cycle_count() { + const uint64_t ns = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + return static_cast(ns * 240 / 1000); +} diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 74239ccb8b7..d1a9c495af3 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -42,7 +42,7 @@ struct Int8RuntimeOptions { /// Internal RAM left free when deciding whether the arena fits there. size_t internal_reserve_bytes = 48 * 1024; /// Time every operation for op_time_us() and print_profile(). Otherwise - /// invoke() reads the timer only a few times per call. + /// invoke() reads the timer only every few milliseconds. bool profile_operations = false; /// Plan each operation's ESP-NN scratch in the activation arena, live only /// while the operation runs, instead of one buffer sized for the largest diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 0481746a459..47fa64cda90 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -31,10 +32,10 @@ namespace { constexpr size_t ARENA_ALIGNMENT = 16; constexpr int64_t YIELD_INTERVAL_US = 1000000; -// Without profiling, invoke() checks the yield deadline only every this many -// operations and when it returns: one timer read costs as much as a small -// operation. -constexpr uint32_t YIELD_CHECK_OPERATIONS = 16; +// Without profiling, invoke() reads the cycle counter, which takes a few +// cycles, after every operation, and the timer, which costs as much as a small +// operation, only once this many cycles (4 ms at 240 MHz) have passed. +constexpr uint32_t YIELD_CHECK_CYCLES = 1U << 20; // Created by a static initializer rather than on first use, so that invoke() // needs no thread-safe static guard (and no C++ runtime support for one). @@ -1987,6 +1988,7 @@ bool Int8Interpreter::invoke() { const int64_t invoke_started = esp_timer_get_time(); int64_t op_started = invoke_started; int64_t last_yield = invoke_started; + uint32_t last_check = esp_cpu_get_cycle_count(); for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { const OpState& state = op_states_[i]; if (options_.scratch_in_arena && state.scratch_bytes != 0) { @@ -2002,8 +2004,12 @@ bool Int8Interpreter::invoke() { opcode_name(op_states_[i].opcode)); return false; } - if (op_us_ == nullptr && (i + 1) % YIELD_CHECK_OPERATIONS != 0) - continue; + if (op_us_ == nullptr) { + const uint32_t cycles = esp_cpu_get_cycle_count(); + if (cycles - last_check < YIELD_CHECK_CYCLES) + continue; + last_check = cycles; + } int64_t now = esp_timer_get_time(); if (op_us_ != nullptr) op_us_[i] = now - op_started; From ccd20e6b86ccbccda02bae4cd4165201f0148c9c Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:36:30 -0800 Subject: [PATCH 16/25] Cover PSRAM and ESP-NN scratch in the ESP-NN host tests The host build had no PSRAM and esp-nn's portable kernels ask for no scratch, so the sanitized host runs never exercised the PSRAM constants copy or either scratch placement, which only the device used. With ESPNN_HOST_PSRAM the heap stand-in serves PSRAM allocations, and it now allocates exact sizes so AddressSanitizer sees any overrun. ESPNN_DELEGATE_CONSTANTS_TO_PSRAM becomes a host CMake option. With ESPNN_HOST_SCRATCH, a test-only esp_nn.h wrapper makes the convolution, depthwise and mean kernels request scratch and overwrite all of it before they run, so a scratch buffer that is too small fails under ASan and one placed over live activations fails the bit-exact checks. The interpreter runner frees its copy of the graph once the interpreter has copied it, as EspNNBackend does. The CI script runs a plain build and three sanitized ones: default options, optimized kernels, and every option on. --- .ci/scripts/test_espressif_host.sh | 33 ++-- backends/espressif/README.md | 13 +- .../espnn_runtime/host/espnn_host.cmake | 17 +- .../host/include/esp_heap_caps.h | 18 +- .../espnn_runtime/host/include/esp_nn.h | 167 ++++++++++++++++++ .../test/host/espnn_host_interpreter.cpp | 25 ++- .../espressif/test/test_host_interpreter.py | 21 ++- backends/espressif/test/test_host_runtime.py | 23 ++- 8 files changed, 281 insertions(+), 36 deletions(-) create mode 100644 backends/espressif/espnn_runtime/host/include/esp_nn.h diff --git a/.ci/scripts/test_espressif_host.sh b/.ci/scripts/test_espressif_host.sh index 0b66db45043..ea19fd6093a 100755 --- a/.ci/scripts/test_espressif_host.sh +++ b/.ci/scripts/test_espressif_host.sh @@ -5,10 +5,12 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -# Builds the ESP-NN delegate for the host, plain and with ASan/UBSan, and checks -# fully delegated programs against the bit-exact Python reference on both. The -# sanitized build also turns on the delegate's non-default options. Needs no -# ESP-IDF toolchain or emulator. +# Builds the ESP-NN delegate for the host and checks fully delegated programs +# against the bit-exact Python reference: plain, and with ASan/UBSan over +# esp-nn's portable kernels, over its optimized C kernels, and with the +# delegate's non-default options. The sanitized builds make the kernels use +# scratch, in a separate buffer or in the arena. Needs no ESP-IDF toolchain or +# emulator. set -euo pipefail @@ -16,20 +18,23 @@ et_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) build_dir="${ESPRESSIF_HOST_BUILD_DIR:-${et_root}/cmake-out-espnn-host}" cd "${et_root}" -for variant in plain sanitized; do - sanitize=OFF - if [[ "${variant}" == sanitized ]]; then - sanitize=ON - fi - options=(-DESPNN_HOST_SANITIZE="${sanitize}" - -DESPNN_DELEGATE_SCRATCH_IN_ARENA="${sanitize}" - -DESPNN_DELEGATE_PROFILE="${sanitize}") +run_variant() { + local variant=$1 + shift cmake -S backends/espressif/test/host -B "${build_dir}/${variant}" -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DPYTHON_EXECUTABLE="$(command -v python)" "${options[@]}" + -DPYTHON_EXECUTABLE="$(command -v python)" "$@" cmake --build "${build_dir}/${variant}" ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \ ESPNN_HOST_INTERPRETER="${build_dir}/${variant}/espnn_host_interpreter" \ python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py \ backends/espressif/test/test_host_interpreter.py -done +} + +run_variant plain +scratch=(-DESPNN_HOST_SANITIZE=ON -DESPNN_HOST_SCRATCH=ON) +run_variant sanitized "${scratch[@]}" +run_variant sanitized-optimized "${scratch[@]}" -DESPNN_HOST_OPTIMIZED=ON +run_variant sanitized-options "${scratch[@]}" -DESPNN_HOST_OPTIMIZED=ON \ + -DESPNN_HOST_PSRAM=ON -DESPNN_DELEGATE_CONSTANTS_TO_PSRAM=ON \ + -DESPNN_DELEGATE_PROFILE=ON -DESPNN_DELEGATE_SCRATCH_IN_ARENA=ON diff --git a/backends/espressif/README.md b/backends/espressif/README.md index beb3c3bf0b7..b097462d2a4 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -372,9 +372,16 @@ ESPNN_HOST_RUNNER=cmake-out-espnn-host/espnn_host_runner \ ``` `-DESPNN_HOST_SANITIZE=ON` builds with AddressSanitizer and -UndefinedBehaviorSanitizer; `.ci/scripts/test_espressif_host.sh` runs both -builds. The host build checks the delegate's logic, not ESP-NN's S3 assembly -kernels, which only the emulator and the board run. +UndefinedBehaviorSanitizer, `-DESPNN_HOST_OPTIMIZED=ON` uses ESP-NN's optimized +C kernels instead of the reference ones, `-DESPNN_HOST_PSRAM=ON` makes the +heap stand-in serve PSRAM allocations, and `-DESPNN_HOST_SCRATCH=ON` makes +the convolution and mean kernels request scratch and overwrite all of it, as +a check on where the delegate puts it. `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM`, +`ESPNN_DELEGATE_PROFILE` and `ESPNN_DELEGATE_SCRATCH_IN_ARENA` are CMake +options here, off by default. `.ci/scripts/test_espressif_host.sh` +runs a plain build and three sanitized ones that cover these options. The host +build checks the delegate's logic, not ESP-NN's S3 assembly kernels, which only +the emulator and the board run. The code follows the ExecuTorch coding style and passes the repository's `lintrunner` checks: diff --git a/backends/espressif/espnn_runtime/host/espnn_host.cmake b/backends/espressif/espnn_runtime/host/espnn_host.cmake index 686a612889a..4c93fa8dbb1 100644 --- a/backends/espressif/espnn_runtime/host/espnn_host.cmake +++ b/backends/espressif/espnn_runtime/host/espnn_host.cmake @@ -22,6 +22,13 @@ set(ESPNN_LOCAL_PATH CACHE PATH "Existing esp-nn checkout to build instead of fetching one" ) option(ESPNN_HOST_OPTIMIZED "Use esp-nn's generic optimized C kernels" OFF) +option(ESPNN_HOST_PSRAM "Serve PSRAM allocations from the host heap" OFF) +option(ESPNN_HOST_SCRATCH "Make the kernels request and fill ESP-NN scratch" + OFF +) +option(ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + "Copy weights into PSRAM; needs ESPNN_HOST_PSRAM" OFF +) option(ESPNN_DELEGATE_PROFILE "Time every delegate operator" OFF) option(ESPNN_DELEGATE_SCRATCH_IN_ARENA "Plan ESP-NN scratch in the activation arena" OFF @@ -76,7 +83,15 @@ target_include_directories( target_compile_definitions( espnn_backend PRIVATE C10_USING_CUSTOM_GENERATED_MACROS ) -foreach(option ESPNN_DELEGATE_PROFILE ESPNN_DELEGATE_SCRATCH_IN_ARENA) +if(ESPNN_HOST_PSRAM) + target_compile_definitions(espnn_backend PUBLIC ESPNN_HOST_PSRAM=1) +endif() +if(ESPNN_HOST_SCRATCH) + target_compile_definitions(espnn_backend PRIVATE ESPNN_HOST_SCRATCH=1) +endif() +foreach(option ESPNN_DELEGATE_CONSTANTS_TO_PSRAM ESPNN_DELEGATE_PROFILE + ESPNN_DELEGATE_SCRATCH_IN_ARENA +) if(${option}) target_compile_definitions(espnn_backend PRIVATE CONFIG_${option}=1) endif() diff --git a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h index 1298d9d2c9c..4aaf572daa8 100644 --- a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h +++ b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h @@ -9,7 +9,8 @@ */ // Host stand-in for the ESP-IDF heap API the ESP-NN delegate uses: one heap, -// all of it "internal", no PSRAM. +// all of it "internal". With ESPNN_HOST_PSRAM, PSRAM allocations come from the +// same heap; otherwise there is no PSRAM. #pragma once @@ -21,12 +22,19 @@ #define MALLOC_CAP_SPIRAM (1U << 10) #define MALLOC_CAP_INTERNAL (1U << 11) +#ifdef ESPNN_HOST_PSRAM +constexpr bool kHostPsram = true; +#else +constexpr bool kHostPsram = false; +#endif + inline void* heap_caps_aligned_alloc(size_t alignment, size_t size, uint32_t caps) { - if ((caps & MALLOC_CAP_SPIRAM) != 0 || size == 0) + if (((caps & MALLOC_CAP_SPIRAM) != 0 && !kHostPsram) || size == 0) return nullptr; - return std::aligned_alloc( - alignment, (size + alignment - 1) & ~(alignment - 1)); + // Exactly size bytes, so that AddressSanitizer sees any overrun. + void* memory = nullptr; + return posix_memalign(&memory, alignment, size) == 0 ? memory : nullptr; } inline void* heap_caps_calloc(size_t count, size_t size, uint32_t) { @@ -38,7 +46,7 @@ inline void heap_caps_free(void* memory) { } inline size_t heap_caps_get_largest_free_block(uint32_t caps) { - return (caps & MALLOC_CAP_SPIRAM) != 0 ? 0 : SIZE_MAX / 2; + return (caps & MALLOC_CAP_SPIRAM) != 0 && !kHostPsram ? 0 : SIZE_MAX / 2; } inline size_t heap_caps_get_total_size(uint32_t caps) { diff --git a/backends/espressif/espnn_runtime/host/include/esp_nn.h b/backends/espressif/espnn_runtime/host/include/esp_nn.h new file mode 100644 index 00000000000..a44e19a0de8 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_nn.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// esp-nn's header. esp-nn's portable kernels need no scratch, unlike its +// ESP32-S3 kernels. With ESPNN_HOST_SCRATCH, the convolution and mean kernels +// request some and fill all of it before they run, so that a scratch buffer +// that is too small fails under AddressSanitizer, and one that overlaps live +// activations fails the bit-exact output checks. + +#pragma once + +#include_next + +#ifdef ESPNN_HOST_SCRATCH + +#include +#include +#include + +namespace espnn_host { + +inline void* g_conv_scratch = nullptr; +inline void* g_depthwise_scratch = nullptr; +inline void* g_mean_scratch = nullptr; + +// Each term is capped, so that no graph shape can overflow the sizes. +inline int capped(int64_t value) { + return static_cast(std::min(value, 4096)); +} + +inline int scratch_size(const data_dims_t* input, const data_dims_t* output) { + return 4 * capped(input->channels) + + capped(int64_t{output->width} * output->height) + 16; +} + +inline int conv_scratch_size( + const data_dims_t* input_dims, + const data_dims_t*, + const data_dims_t* output_dims, + const conv_params_t*) { + return scratch_size(input_dims, output_dims); +} + +inline int depthwise_conv_scratch_size( + const data_dims_t* input_dims, + const data_dims_t*, + const data_dims_t* output_dims, + const dw_conv_params_t*) { + return scratch_size(input_dims, output_dims); +} + +inline int32_t mean_scratch_size(int32_t, int32_t, int32_t channels) { + return 4 * capped(channels) + 16; +} + +inline void set_conv_scratch(const void* buffer) { + g_conv_scratch = const_cast(buffer); +} + +inline void set_depthwise_conv_scratch(const void* buffer) { + g_depthwise_scratch = const_cast(buffer); +} + +inline void set_mean_scratch(void* buffer) { + g_mean_scratch = buffer; +} + +inline void conv_s8( + const data_dims_t* input_dims, + const int8_t* input_data, + const data_dims_t* filter_dims, + const int8_t* filter_data, + const int32_t* bias, + const data_dims_t* output_dims, + int8_t* out_data, + const conv_params_t* conv_params, + const quant_data_t* quant_data) { + std::memset(g_conv_scratch, 0x5a, scratch_size(input_dims, output_dims)); + esp_nn_conv_s8( + input_dims, + input_data, + filter_dims, + filter_data, + bias, + output_dims, + out_data, + conv_params, + quant_data); +} + +inline void depthwise_conv_s8( + const data_dims_t* input_dims, + const int8_t* input_data, + const data_dims_t* filter_dims, + const int8_t* filter_data, + const int32_t* bias, + const data_dims_t* output_dims, + int8_t* out_data, + const dw_conv_params_t* conv_params, + const quant_data_t* quant_data) { + std::memset(g_depthwise_scratch, 0x5a, scratch_size(input_dims, output_dims)); + esp_nn_depthwise_conv_s8( + input_dims, + input_data, + filter_dims, + filter_data, + bias, + output_dims, + out_data, + conv_params, + quant_data); +} + +inline void mean_nhwc_s8( + const int8_t* input, + int8_t* output, + int32_t height, + int32_t width, + int32_t channels, + int32_t input_zero_point, + int32_t output_zero_point, + int32_t multiplier, + int32_t shift) { + std::memset(g_mean_scratch, 0x5a, mean_scratch_size(height, width, channels)); + esp_nn_mean_nhwc_s8( + input, + output, + height, + width, + channels, + input_zero_point, + output_zero_point, + multiplier, + shift); +} + +} // namespace espnn_host + +#undef esp_nn_get_conv_scratch_size +#undef esp_nn_set_conv_scratch_buf +#undef esp_nn_conv_s8 +#undef esp_nn_get_depthwise_conv_scratch_size +#undef esp_nn_set_depthwise_conv_scratch_buf +#undef esp_nn_depthwise_conv_s8 +#undef esp_nn_get_mean_scratch_size +#undef esp_nn_set_mean_scratch_buf +#undef esp_nn_mean_nhwc_s8 +#define esp_nn_get_conv_scratch_size espnn_host::conv_scratch_size +#define esp_nn_set_conv_scratch_buf espnn_host::set_conv_scratch +#define esp_nn_conv_s8 espnn_host::conv_s8 +#define esp_nn_get_depthwise_conv_scratch_size \ + espnn_host::depthwise_conv_scratch_size +#define esp_nn_set_depthwise_conv_scratch_buf \ + espnn_host::set_depthwise_conv_scratch +#define esp_nn_depthwise_conv_s8 espnn_host::depthwise_conv_s8 +#define esp_nn_get_mean_scratch_size espnn_host::mean_scratch_size +#define esp_nn_set_mean_scratch_buf espnn_host::set_mean_scratch +#define esp_nn_mean_nhwc_s8 espnn_host::mean_nhwc_s8 + +#endif diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp index 7a10f4ad830..a6dcf336d26 100644 --- a/backends/espressif/test/host/espnn_host_interpreter.cpp +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -12,12 +12,15 @@ * Runs one XTS3 graph on Int8Interpreter directly, for * backends/espressif/test/test_host_interpreter.py: * - * espnn_host_interpreter GRAPH OFFSET [--external-io] INPUT... -- OUTPUT... + * espnn_host_interpreter GRAPH OFFSET [--constants-to-psram] [--external-io] + * INPUT... -- OUTPUT... * * The graph is placed OFFSET bytes past a 16-byte boundary, so that a test can * load it from addresses with any alignment. Inputs and outputs are raw - * device-layout (NHWC) int8 files. With --external-io, outputs are bound to - * runner buffers prefilled with 0x5a. + * device-layout (NHWC) int8 files. --constants-to-psram sets + * Int8RuntimeOptions::copy_constants_to_psram. When the interpreter copies the + * graph, the runner frees its own copy, as EspNNBackend does. With + * --external-io, outputs are bound to runner buffers prefilled with 0x5a. */ #include @@ -67,6 +70,8 @@ int main(int argc, char** argv) { for (int a = 3; a < argc; ++a) { if (std::strcmp(argv[a], "--") == 0) past_separator = true; + else if (std::strcmp(argv[a], "--constants-to-psram") == 0) + options.copy_constants_to_psram = true; else if (std::strcmp(argv[a], "--external-io") == 0) options.external_io = true; else @@ -82,6 +87,10 @@ int main(int argc, char** argv) { Int8Interpreter interpreter; if (!interpreter.load(base + offset, graph.size(), options)) return fail("load failed"); + if (interpreter.constants_copied()) { + storage.clear(); + storage.shrink_to_fit(); + } if (inputs.size() != interpreter.input_count() || outputs.size() != interpreter.output_count()) return fail("wrong number of inputs or outputs"); @@ -114,9 +123,15 @@ int main(int argc, char** argv) { return fail(std::string("cannot write ") + outputs[o]); std::fclose(file); } +#ifdef ESPNN_HOST_PSRAM + const int psram = 1; +#else + const int psram = 0; +#endif std::printf( - "STATS parameters=%zu blob_copied=%d\n", + "STATS parameters=%zu blob_copied=%d psram=%d\n", interpreter.parameter_bytes(), - interpreter.constants_copied() ? 1 : 0); + interpreter.constants_copied() ? 1 : 0, + psram); return 0; } diff --git a/backends/espressif/test/test_host_interpreter.py b/backends/espressif/test/test_host_interpreter.py index 9b84e00c1b2..c03937e0453 100644 --- a/backends/espressif/test/test_host_interpreter.py +++ b/backends/espressif/test/test_host_interpreter.py @@ -81,8 +81,8 @@ def run( return outputs, dict(field.split("=") for field in stats.split()[1:]) -@pytest.mark.parametrize("version", [6, 7]) -def test_graph_at_any_alignment(tmp_path, version): +def se_block(version: int): + """An XTS3 graph of the given version, random inputs and its outputs.""" _, program = op_cases.quantize(lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]) graph = xts3.build_graph(program) graph.version = version @@ -96,6 +96,12 @@ def test_graph_at_any_alignment(tmp_path, version): expected = [ value.astype(np.int8).reshape(-1) for value in interpreter.run_outputs(inputs) ] + return graph, inputs, expected + + +@pytest.mark.parametrize("version", [6, 7]) +def test_graph_at_any_alignment(tmp_path, version): + graph, inputs, expected = se_block(version) for offset in (0, 1, 2, 3, 4, 8): outputs, stats = run(tmp_path, graph, offset, inputs) for actual, wanted in zip(outputs, expected): @@ -106,6 +112,17 @@ def test_graph_at_any_alignment(tmp_path, version): assert (stats["parameters"] == "0") == (version == 7) +@pytest.mark.parametrize("version", [6, 7]) +def test_constants_copied_to_psram(tmp_path, version): + # In a build with a PSRAM stand-in the graph is copied there and the + # runner frees the original, so every table pointer must follow the copy. + graph, inputs, expected = se_block(version) + outputs, stats = run(tmp_path, graph, 0, inputs, "--constants-to-psram") + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + assert stats["blob_copied"] == stats["psram"] + + def test_partial_copy_does_not_leak_into_a_bound_output(tmp_path): # One COPY segment writes 4 of the output's 16 elements; the rest of a # caller's output buffer must not keep its previous contents. diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index e45a65960a8..655bf4a7dd3 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -216,6 +216,11 @@ def assert_matches( assert torch.equal(actual, tensor) +def arena_holds_plan(stats, planned: int) -> bool: + """Whether the arena is the planned size, plus any ESP-NN scratch in it.""" + return stats["arena"] == planned if stats["scratch"] else stats["arena"] >= planned + + def random_steps(shapes, count: int, seed: int = 11): generator = torch.Generator().manual_seed(seed) return [ @@ -549,9 +554,12 @@ def test_device_layout_io_is_used_in_place(tmp_path, channels_last): # directly. assert stats["staging"] == 0 if channels_last: - assert stats["arena"] == sum( - reference.plan_activation_arena(delegate.graph, external_io=True)[0] - for delegate in emulator.delegates.values() + assert arena_holds_plan( + stats, + sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ), ) @@ -621,9 +629,12 @@ def test_outputs_read_by_operators_keep_their_slots(tmp_path, channels_last): assert_matches(outputs[step][2:], expected[2:], channels_last) assert stats["staging"] == 0 if channels_last: - assert stats["arena"] == sum( - reference.plan_activation_arena(delegate.graph, external_io=True)[0] - for delegate in emulator.delegates.values() + assert arena_holds_plan( + stats, + sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ), ) From c4006aa75cd01f2b15431587b4535cf3731409b9 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:38:11 -0800 Subject: [PATCH 17/25] Give the ESP-NN delegate's arena the first claim on internal RAM Int8Interpreter::load() allocated the requantization tables, the dilated convolution tile buffer and the separate ESP-NN scratch while it prepared the operations, before planning memory, so they took internal RAM first and could push the activation arena, which every operator reads and writes, out to PSRAM. prepare_operations() now only works out those sizes, plan_memory() allocates the arena, and the new allocate_operation_buffers() then allocates the tables and buffers and points the operations at them. The host heap stand-in can limit internal RAM through ESPNN_HOST_INTERNAL_BYTES, and a host test gives it room for the arena and the reserve only. --- .../host/include/esp_heap_caps.h | 39 ++++++- .../include/espnn/int8_interpreter.h | 1 + .../espnn_runtime/src/int8_interpreter.cpp | 110 +++++++++--------- .../test/host/espnn_host_interpreter.cpp | 5 +- .../espressif/test/test_host_interpreter.py | 20 +++- 5 files changed, 116 insertions(+), 59 deletions(-) diff --git a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h index 4aaf572daa8..c58ff752fee 100644 --- a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h +++ b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h @@ -10,13 +10,15 @@ // Host stand-in for the ESP-IDF heap API the ESP-NN delegate uses: one heap, // all of it "internal". With ESPNN_HOST_PSRAM, PSRAM allocations come from the -// same heap; otherwise there is no PSRAM. +// same heap; otherwise there is no PSRAM. ESPNN_HOST_INTERNAL_BYTES in the +// environment limits the internal aligned allocations to that many bytes. #pragma once #include #include #include +#include #define MALLOC_CAP_8BIT (1U << 2) #define MALLOC_CAP_SPIRAM (1U << 10) @@ -28,13 +30,35 @@ constexpr bool kHostPsram = true; constexpr bool kHostPsram = false; #endif +namespace espnn_host { + +inline std::unordered_map g_internal_blocks; +inline size_t g_internal_used = 0; + +inline size_t internal_free() { + static const char* limit = std::getenv("ESPNN_HOST_INTERNAL_BYTES"); + const size_t capacity = + limit != nullptr ? std::strtoull(limit, nullptr, 10) : SIZE_MAX / 2; + return capacity > g_internal_used ? capacity - g_internal_used : 0; +} + +} // namespace espnn_host + inline void* heap_caps_aligned_alloc(size_t alignment, size_t size, uint32_t caps) { - if (((caps & MALLOC_CAP_SPIRAM) != 0 && !kHostPsram) || size == 0) + const bool psram = (caps & MALLOC_CAP_SPIRAM) != 0; + if ((psram && !kHostPsram) || size == 0 || + (!psram && size > espnn_host::internal_free())) return nullptr; // Exactly size bytes, so that AddressSanitizer sees any overrun. void* memory = nullptr; - return posix_memalign(&memory, alignment, size) == 0 ? memory : nullptr; + if (posix_memalign(&memory, alignment, size) != 0) + return nullptr; + if (!psram) { + espnn_host::g_internal_blocks[memory] = size; + espnn_host::g_internal_used += size; + } + return memory; } inline void* heap_caps_calloc(size_t count, size_t size, uint32_t) { @@ -42,11 +66,18 @@ inline void* heap_caps_calloc(size_t count, size_t size, uint32_t) { } inline void heap_caps_free(void* memory) { + const auto block = espnn_host::g_internal_blocks.find(memory); + if (block != espnn_host::g_internal_blocks.end()) { + espnn_host::g_internal_used -= block->second; + espnn_host::g_internal_blocks.erase(block); + } std::free(memory); } inline size_t heap_caps_get_largest_free_block(uint32_t caps) { - return (caps & MALLOC_CAP_SPIRAM) != 0 && !kHostPsram ? 0 : SIZE_MAX / 2; + if ((caps & MALLOC_CAP_SPIRAM) != 0) + return kHostPsram ? SIZE_MAX / 2 : 0; + return espnn_host::internal_free(); } inline size_t heap_caps_get_total_size(uint32_t caps) { diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index d1a9c495af3..4750cf80b29 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -241,6 +241,7 @@ class Int8Interpreter { bool plan_memory(); bool prepare_operations(); + bool allocate_operation_buffers(); void rebase_constants(const uint8_t* old_base, size_t size); bool tensor_info(uint32_t tensor_id, Int8TensorInfo& info) const; int8_t* activation(uint32_t tensor_id) const; diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 47fa64cda90..4bb3fc6f19a 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -404,8 +404,10 @@ bool Int8Interpreter::load( } } - // Operations first: the planner places each operation's ESP-NN scratch. - if (!prepare_operations() || !plan_memory()) { + // The planner places each operation's ESP-NN scratch, and the arena gets + // the first claim on internal RAM. + if (!prepare_operations() || !plan_memory() || + !allocate_operation_buffers()) { reset(); return false; } @@ -672,16 +674,7 @@ bool Int8Interpreter::prepare_operations() { } } params_size_ = param_words * sizeof(int32_t); - if (params_size_ != 0) { - params_ = static_cast(allocate_preferring_internal( - params_size_, true, options_.internal_reserve_bytes, nullptr)); - if (params_ == nullptr) { - std::printf("ERROR: unable to allocate requantization tables\n"); - return false; - } - } - int32_t* cursor = params_; size_t scratch_size = 0; auto require_scratch = [&scratch_size](OpState& state, int32_t needed) { if (needed <= 0) @@ -692,31 +685,6 @@ bool Int8Interpreter::prepare_operations() { if (bytes > scratch_size) scratch_size = bytes; }; - auto bind_requantization = [this, &cursor]( - OpState& state, const OperationRecord& op) { - const uint8_t* multipliers = nullptr; - const uint8_t* shifts = nullptr; - uint32_t channels = 0; - if (!requantization_arrays( - graph_, tensors_, op, multipliers, shifts, channels)) - return false; - const int32_t* multipliers_in_place = - int32s_in_place(tables_in_place_, multipliers); - const int32_t* shifts_in_place = int32s_in_place(tables_in_place_, shifts); - if (multipliers_in_place != nullptr && shifts_in_place != nullptr) { - state.multipliers = const_cast(multipliers_in_place); - state.shifts = const_cast(shifts_in_place); - return true; - } - state.multipliers = cursor; - state.shifts = cursor + channels; - for (uint32_t c = 0; c < channels; ++c) { - state.multipliers[c] = read_le_i32(multipliers + c * 4); - state.shifts[c] = read_le_i32(shifts + c * 4); - } - cursor += 2U * channels; - return true; - }; for (uint32_t i = 0; i < op_count; ++i) { OperationRecord op{}; read_operation(graph_, i, op); @@ -746,10 +714,6 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, weight)); state.bias = reinterpret_cast( tensor_constant_data(graph_, bias)); - if (!bind_requantization(state, op)) - return false; - state.quant.mult = state.multipliers; - state.quant.shift = state.shifts; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; state.output_dims = { output.shape[3], output.shape[2], output.shape[1], 1}; @@ -890,8 +854,6 @@ bool Int8Interpreter::prepare_operations() { state.dilation_w = read_le_i32(p + 36); state.value0 = read_le_i32(p + 44); state.value1 = read_le_i32(p + 48); - if (!bind_requantization(state, op)) - return false; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; state.output_dims = { output.shape[3], output.shape[2], output.shape[1], 1}; @@ -931,14 +893,6 @@ bool Int8Interpreter::prepare_operations() { state.rows = static_cast(state.elements) / state.cols; break; case OpCode::SOFTMAX: - if (const int32_t* table = int32s_in_place(tables_in_place_, p)) { - state.multipliers = const_cast(table); - } else { - state.multipliers = cursor; - for (int k = 0; k < 256; ++k) - state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); - cursor += 256; - } state.cols = input.shape[1]; state.rows = static_cast(state.elements) / state.cols; break; @@ -970,8 +924,6 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, bias)); state.value0 = read_le_i32(p); state.value1 = read_le_i32(p + 4); - if (!bind_requantization(state, op)) - return false; break; } default: @@ -979,6 +931,59 @@ bool Int8Interpreter::prepare_operations() { } } + scratch_size_ = options_.scratch_in_arena ? 0 : scratch_size; + return true; +} + +bool Int8Interpreter::allocate_operation_buffers() { + if (params_size_ != 0) { + params_ = static_cast(allocate_preferring_internal( + params_size_, true, options_.internal_reserve_bytes, nullptr)); + if (params_ == nullptr) { + std::printf("ERROR: unable to allocate requantization tables\n"); + return false; + } + } + int32_t* cursor = params_; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph_, i, op); + OpState& state = op_states_[i]; + const uint8_t* multipliers = nullptr; + const uint8_t* shifts = nullptr; + uint32_t channels = 0; + if (requantization_arrays( + graph_, tensors_, op, multipliers, shifts, channels)) { + const int32_t* multipliers_in_place = + int32s_in_place(tables_in_place_, multipliers); + const int32_t* shifts_in_place = + int32s_in_place(tables_in_place_, shifts); + if (multipliers_in_place != nullptr && shifts_in_place != nullptr) { + state.multipliers = const_cast(multipliers_in_place); + state.shifts = const_cast(shifts_in_place); + } else { + state.multipliers = cursor; + state.shifts = cursor + channels; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(multipliers + c * 4); + state.shifts[c] = read_le_i32(shifts + c * 4); + } + cursor += 2U * channels; + } + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; + } else if (state.opcode == OpCode::SOFTMAX) { + const uint8_t* p = graph_.parameter_data + op.parameter_offset; + if (const int32_t* table = int32s_in_place(tables_in_place_, p)) { + state.multipliers = const_cast(table); + } else { + state.multipliers = cursor; + for (int k = 0; k < 256; ++k) + state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); + cursor += 256; + } + } + } if (phase_bytes_ != 0) { phase_buffer_ = static_cast(allocate_preferring_internal( phase_bytes_, true, options_.internal_reserve_bytes, nullptr)); @@ -989,7 +994,6 @@ bool Int8Interpreter::prepare_operations() { return false; } } - scratch_size_ = options_.scratch_in_arena ? 0 : scratch_size; if (scratch_size_ != 0) { scratch_ = allocate_preferring_internal( scratch_size_, true, options_.internal_reserve_bytes, nullptr); diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp index a6dcf336d26..0e563741cea 100644 --- a/backends/espressif/test/host/espnn_host_interpreter.cpp +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -129,7 +129,10 @@ int main(int argc, char** argv) { const int psram = 0; #endif std::printf( - "STATS parameters=%zu blob_copied=%d psram=%d\n", + "STATS arena=%zu arena_internal=%d parameters=%zu blob_copied=%d " + "psram=%d\n", + interpreter.arena_bytes(), + interpreter.arena_in_internal_ram() ? 1 : 0, interpreter.parameter_bytes(), interpreter.constants_copied() ? 1 : 0, psram); diff --git a/backends/espressif/test/test_host_interpreter.py b/backends/espressif/test/test_host_interpreter.py index c03937e0453..d8dce393916 100644 --- a/backends/espressif/test/test_host_interpreter.py +++ b/backends/espressif/test/test_host_interpreter.py @@ -49,7 +49,7 @@ def disable_grad(): def run( - tmp_path, graph: xts3.XTS3Graph, offset: int, inputs, *options + tmp_path, graph: xts3.XTS3Graph, offset: int, inputs, *options, env=None ) -> tuple[list, dict]: blob = tmp_path / "graph.xts3" blob.write_bytes(xts3.serialize_graph(graph)) @@ -72,6 +72,7 @@ def run( ], capture_output=True, text=True, + env=None if env is None else {**os.environ, **env}, ) assert result.returncode == 0, result.stdout + result.stderr (stats,) = [line for line in result.stdout.splitlines() if line.startswith("STATS")] @@ -123,6 +124,23 @@ def test_constants_copied_to_psram(tmp_path, version): assert stats["blob_copied"] == stats["psram"] +def test_arena_gets_internal_ram_first(tmp_path): + # Internal RAM holds the arena, its 16 readable bytes past the end and the + # default 48 KiB reserve, but not the version 6 graph's requantization + # tables too. In a build with a PSRAM stand-in, allocating the tables + # first would push the arena out to PSRAM. + graph, inputs, expected = se_block(6) + _, stats = run(tmp_path, graph, 0, inputs) + limit = int(stats["arena"]) + 16 + 48 * 1024 + 1 + outputs, stats = run( + tmp_path, graph, 0, inputs, env={"ESPNN_HOST_INTERNAL_BYTES": str(limit)} + ) + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + assert int(stats["parameters"]) > 0 + assert stats["arena_internal"] == "1" + + def test_partial_copy_does_not_leak_into_a_bound_output(tmp_path): # One COPY segment writes 4 of the output's 16 elements; the rest of a # caller's output buffer must not keep its previous contents. From d6b2edc7f0b53fcabdab43d4400f64ddf3a59a82 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:38:49 -0800 Subject: [PATCH 18/25] Allocate the ESP-NN delegate's staging buffer at init An output outside the arena whose ExecuTorch buffer is not 16-byte aligned is written to a staging buffer and copied out, and that buffer was allocated by the first execute() that needed one, in internal RAM regardless of the interpreter's reserve. A program could then fail, or starve the rest of the application, in the middle of running. EspNNBackend::init() now allocates the staging buffer, sized for the outputs outside the arena that could need it. Only misaligned outputs use it, so it goes to PSRAM when there is any and leaves internal RAM to the arena. Nothing is allocated during execute() again, as the README says. A new counter, DelegateStats::staged_outputs, counts the outputs written through it, and the host tests check that staging happens without growing the buffer. --- backends/espressif/README.md | 18 +-- .../include/espnn/espnn_backend.h | 6 +- .../espnn_runtime/src/espnn_backend.cpp | 115 ++++++++++-------- .../espressif/test/host/espnn_host_runner.cpp | 3 +- backends/espressif/test/test_host_runtime.py | 21 ++-- 5 files changed, 95 insertions(+), 68 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index b097462d2a4..2d625c312ad 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -116,9 +116,10 @@ buffer is memory-planned by ExecuTorch or supplied by the caller. Float outputs and NCHW outputs with several channels and pixels need dtype or layout conversion. The exporter records which outputs need conversion, so their int8 buffers can share the arena with intermediate tensors. Older -exports without this metadata use the arena for all outputs. A misaligned -output goes through a separate staging buffer, allocated on the first -`execute()` that needs it. +exports without this metadata use the arena for all outputs. An output +outside the arena whose buffer turns out misaligned is written to a staging +buffer and copied out. The delegate allocates that buffer at `init()`, sized +for every such output, in PSRAM when there is any. For the smallest and fastest program, give the delegate int8, channels-last inputs and outputs and leave them out of ExecuTorch's memory plan, so that @@ -400,11 +401,12 @@ lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" norm fall back to the CPU. - Dual-core execution only helps large convolutions, and it is off by default. - The delegate handle comes from ExecuTorch's runtime allocator, but each - delegate allocates its activation arena, scratch buffers and optional PSRAM - weight copy with ESP-IDF's `heap_caps` allocator at `init()`, so it can - choose between internal RAM and PSRAM. After that, only the first - `execute()` that needs a staging buffer (see "Inputs and outputs") - allocates. + delegate allocates its activation arena, scratch and staging buffers and + optional PSRAM weight copy with ESP-IDF's `heap_caps` allocator at + `init()`, so it can choose between internal RAM and PSRAM. Nothing is + allocated during `execute()`, except that with `ESPNN_DELEGATE_DUAL_CORE` + ESP-NN's worker task grows its own scratch buffer the first time a large + convolution needs it. ## License diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index 61642f0a448..16842dd6942 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -42,9 +42,11 @@ struct DelegateStats { size_t scratch_bytes; size_t parameter_bytes; size_t table_bytes; - /// Sum of the buffers that delegates allocated on their first execute() - /// for inputs and outputs they cannot use in place. + /// Sum of the buffers that delegates allocated at init() for outputs + /// whose memory turns out not to be 16-byte aligned, and the number of + /// outputs that execute() wrote through them. size_t staging_bytes; + uint32_t staged_outputs; /// Delegate-owned state tensors (mutable buffers), and how many of them /// are updated in place rather than copied after the last operator. uint32_t states; diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 5c17c29329e..60b5b7053be 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -201,7 +201,8 @@ struct DelegateState { uint32_t conversion_slots = 0; bool profiled = false; uint32_t state_generation = 0; - // Copies of misaligned outputs that are not in the arena. + // Copies of outputs outside the arena whose memory is misaligned, in output + // order. int8_t* staging = nullptr; ~DelegateState() { @@ -209,41 +210,68 @@ struct DelegateState { heap_caps_free(staging); } - int8_t* staging_slot(size_t slot) { + // A graph tensor returned more than once is written where its first + // output is bound, and a graph input returned as an output stays where + // the input is bound; both are copied to their outputs afterwards. + bool returned_again(size_t o) const { + Int8TensorInfo info{}; + Int8TensorInfo other{}; + if (!interpreter.output_info(o, info)) + return false; + for (size_t earlier = 0; earlier < o; ++earlier) { + if (interpreter.output_info(earlier, other) && + other.tensor_id == info.tensor_id) + return true; + } + for (size_t i = 0; i < interpreter.input_count(); ++i) { + if (interpreter.input_info(i, other) && other.tensor_id == info.tensor_id) + return true; + } + return false; + } + + // Offset of output end's staging copy, the staging buffer's size for + // end = output_count(), or SIZE_MAX if that overflows. + size_t staging_offset(size_t end) const { const size_t inputs = interpreter.input_count(); - const size_t slots = inputs + interpreter.output_count(); - size_t offset = 0; size_t total = 0; - for (size_t k = inputs; k < slots; ++k) { - if ((conversion_slots & (1U << k)) != 0) - continue; + for (size_t o = 0; o < end; ++o) { Int8TensorInfo info{}; - if (!interpreter.output_info(k - inputs, info)) - return nullptr; - if (k == slot) - offset = total; - const size_t bytes = (info.bytes + 15) & ~static_cast(15); - if (info.bytes > SIZE_MAX - 15 || bytes > SIZE_MAX - total) - return nullptr; - total += bytes; + if ((conversion_slots & (1U << (inputs + o))) != 0 || returned_again(o) || + !interpreter.output_info(o, info)) + continue; + if (info.bytes > SIZE_MAX - 15 - total) + return SIZE_MAX; + total += (info.bytes + 15) & ~static_cast(15); } - if (staging == nullptr) { - staging = static_cast(heap_caps_aligned_alloc( - 16, total, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); - if (staging == nullptr) { - staging = static_cast( - heap_caps_aligned_alloc(16, total, MALLOC_CAP_8BIT)); - } - if (staging == nullptr) { - std::printf( - "EspNN delegate: cannot allocate %u staging bytes\n", - static_cast(total)); - return nullptr; + return total; + } + + // Only misaligned outputs use the staging buffer, so it goes to PSRAM when + // there is any and leaves internal RAM to the arena. + bool allocate_staging() { + const size_t total = staging_offset(interpreter.output_count()); + if (total == 0) + return true; + void* memory = nullptr; + if (total != SIZE_MAX) { + memory = heap_caps_aligned_alloc( + 16, total, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (memory == nullptr) { + memory = heap_caps_aligned_alloc( + 16, total, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); } - std::memset(staging, 0, total); - g_stats.staging_bytes += total; } - return staging + offset; + if (memory == nullptr) { + std::printf( + "EspNN delegate: cannot allocate %u staging bytes\n", + static_cast(total)); + return false; + } + staging = static_cast(memory); + std::memset(staging, 0, total); + g_stats.staging_bytes += total; + return true; } }; @@ -295,6 +323,10 @@ class EspNNBackendInterface final : public BackendInterface { state->conversion_slots = options.external_io ? state->interpreter.conversion_slots() : UINT32_MAX; + if (!state->allocate_staging()) { + state->~DelegateState(); + return Error::MemoryAllocationFailed; + } if (state->interpreter.constants_copied()) processed->Free(); state->state_generation = @@ -366,21 +398,7 @@ class EspNNBackendInterface final : public BackendInterface { staged_outputs |= 1U << o; continue; } - // A graph tensor returned more than once is written where its first - // output is bound, and a graph input returned as an output stays where - // the input is bound; both are copied to their outputs afterwards. - bool repeated = false; - for (size_t earlier = 0; ok && earlier < o; ++earlier) { - Int8TensorInfo first{}; - interpreter.output_info(earlier, first); - repeated = repeated || first.tensor_id == info.tensor_id; - } - for (size_t i = 0; ok && i < inputs; ++i) { - Int8TensorInfo input{}; - interpreter.input_info(i, input); - repeated = repeated || input.tensor_id == info.tensor_id; - } - if (ok && repeated) { + if (ok && state->returned_again(o)) { staged_outputs |= 1U << o; } else if (ok) { Tensor tensor = args[inputs + o]->toTensor(); @@ -389,11 +407,10 @@ class EspNNBackendInterface final : public BackendInterface { arena_aligned(tensor.const_data_ptr())) { ok = interpreter.bind_output(o, tensor.mutable_data_ptr()); } else { - int8_t* slot = state->staging_slot(inputs + o); - if (slot == nullptr) - return Error::MemoryAllocationFailed; - ok = interpreter.bind_output(o, slot); + ok = interpreter.bind_output( + o, state->staging + state->staging_offset(o)); staged_outputs |= 1U << o; + ++g_stats.staged_outputs; } } if (!ok) { diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index 8e5e302b448..bcd0d4a23a2 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -313,13 +313,14 @@ int main(int argc, char** argv) { const auto stats = executorch::backends::espressif::delegate_stats(); std::printf( "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu " - "staging=%zu states=%u states_in_place=%u\n", + "staging=%zu staged=%u states=%u states_in_place=%u\n", static_cast(stats.delegates), stats.arena_bytes, stats.scratch_bytes, stats.parameter_bytes, stats.table_bytes, stats.staging_bytes, + static_cast(stats.staged_outputs), static_cast(stats.states), static_cast(stats.states_updated_in_place)); diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 655bf4a7dd3..f4ea826a04b 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -302,7 +302,9 @@ def test_conversion_buffers_share_the_arena( buffer = edge.to_executorch(config).buffer _, loaded = run_host(tmp_path, buffer, []) assert loaded["arena"] == arena_slots * sample.numel() - assert loaded["staging"] == 0 + # The staging buffer, allocated at init, covers outputs outside the arena. + bound = output_int8 and channels_last and not legacy + assert loaded["staging"] == ((sample.numel() + 15) // 16 * 16 if bound else 0) steps = random_steps([shape], 2) steps = [ [ @@ -317,7 +319,7 @@ def test_conversion_buffers_share_the_arena( for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) assert stats["arena"] == loaded["arena"] - assert stats["staging"] == 0 + assert stats["staged"] == 0 def test_partitions_keep_independent_conversion_plans(tmp_path): @@ -332,7 +334,7 @@ def test_partitions_keep_independent_conversion_plans(tmp_path): assert stats["delegates"] == 2 # Inputs are always copied; both partitions' outputs need conversion. assert stats["arena"] == 32 + 32 - assert stats["staging"] == 0 + assert stats["staged"] == 0 def test_inputs_are_copied_for_exports_that_bound_them(tmp_path): @@ -373,7 +375,7 @@ def test_inputs_are_copied_for_exports_that_bound_them(tmp_path): for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs)), True) assert stats["arena"] == 2 * sample.numel() - assert stats["staging"] == 0 + assert stats["staged"] == 0 def test_instances_run_independently(tmp_path): @@ -552,7 +554,7 @@ def test_device_layout_io_is_used_in_place(tmp_path, channels_last): assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) # Inputs and NCHW outputs use the arena; channels-last outputs are bound # directly. - assert stats["staging"] == 0 + assert stats["staged"] == 0 if channels_last: assert arena_holds_plan( stats, @@ -608,10 +610,13 @@ def test_misaligned_buffers_are_staged(tmp_path): exported, program, _ = lower_streaming(channels_last=True) emulator = ProgramEmulator(exported) steps = int8_steps(2, channels_last=True) + _, loaded = run_host(tmp_path, program, [], "--share-io", "--misalign") (outputs,), stats = run_host(tmp_path, program, steps, "--share-io", "--misalign") for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs)), True) - assert stats["staging"] > 0 + # Staged through the buffer allocated at init; execute() allocates nothing. + assert stats["staged"] > 0 + assert stats["staging"] == loaded["staging"] > 0 @pytest.mark.parametrize("channels_last", [False, True]) @@ -627,7 +632,7 @@ def test_outputs_read_by_operators_keep_their_slots(tmp_path, channels_last): for step, inputs in enumerate(steps): expected = as_list(emulator(*inputs)) assert_matches(outputs[step][2:], expected[2:], channels_last) - assert stats["staging"] == 0 + assert stats["staged"] == 0 if channels_last: assert arena_holds_plan( stats, @@ -741,4 +746,4 @@ def test_streaming_example_runs_without_copies(tmp_path): emulator = ProgramEmulator(edge.exported_program()) for step, inputs in enumerate(steps): assert_matches(outputs[step], as_list(emulator(*inputs))) - assert stats["states"] == 2 and stats["staging"] == 0 + assert stats["states"] == 2 and stats["staged"] == 0 From 2439c2c15c86eede1038c83367648a08846374b3 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:39:14 -0800 Subject: [PATCH 19/25] Reject ESP-NN delegate inputs and outputs without data An unplanned input or output that the application never gave memory reached EspNNBackend::execute() with a null data pointer. The delegate read or wrote through it, or for an output written in place, failed inside the interpreter with Error::Internal and no hint of the cause. execute() now rejects such a tensor before running, printing "input N has no data" or "output N has no data" and returning Error::InvalidArgument. The host runner can leave an unplanned output unbound to test this. --- .../espnn_runtime/src/espnn_backend.cpp | 13 +++++++++ .../espressif/test/host/espnn_host_runner.cpp | 9 ++++++- backends/espressif/test/test_host_runtime.py | 27 ++++++++++++++----- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 60b5b7053be..19ff3adc145 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -108,6 +108,14 @@ Layout tensor_layout(const Tensor& tensor, const Int8TensorInfo& info) { return Layout::kInvalid; } +Error missing_data(const char* kind, size_t index) { + std::printf( + "EspNN delegate: %s %u has no data\n", + kind, + static_cast(index)); + return Error::InvalidArgument; +} + // The kernels may rely on the activation arena's 16-byte alignment, so only // buffers aligned like it are used in place. bool arena_aligned(const void* data) { @@ -376,6 +384,9 @@ class EspNNBackendInterface final : public BackendInterface { args[i]->isTensor() && interpreter.input_info(i, info) ? tensor_layout(args[i]->toTensor(), info) : Layout::kInvalid; + if (layout != Layout::kInvalid && + args[i]->toTensor().const_data_ptr() == nullptr) + return missing_data("input", i); const bool ok = layout != Layout::kInvalid && load_input( @@ -394,6 +405,8 @@ class EspNNBackendInterface final : public BackendInterface { ? tensor_layout(args[inputs + o]->toTensor(), info) : Layout::kInvalid; bool ok = layout != Layout::kInvalid; + if (ok && args[inputs + o]->toTensor().const_data_ptr() == nullptr) + return missing_data("output", o); if (ok && (state->conversion_slots & (1U << (inputs + o))) != 0) { staged_outputs |= 1U << o; continue; diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index bcd0d4a23a2..d5fc1001c68 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -16,6 +16,7 @@ * [--alias INPUT:OUTPUT]... * [--share-output FIRST:SECOND]... * [--misalign] [--reset-before STEP]... + * [--unbound-output OUTPUT] * * Step s of instance k reads input i from IO_DIR/in__.bin and writes * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its @@ -27,11 +28,13 @@ * output of the previous step. --share-output writes unplanned output SECOND * into output FIRST's buffer. * --misalign places every runner buffer one byte past a 16-byte boundary. + * --unbound-output never gives that unplanned output memory. * --reset-before asks EspNNBackend, through its reset_states backend option, * to restore its delegates' state before that step. At the end, the runner * prints the delegates' memory as a STATS line. */ +#include #include #include #include @@ -147,6 +150,7 @@ int main(int argc, char** argv) { int instance_count = 1; bool share_io = false; size_t misalign = 0; + size_t unbound_output = SIZE_MAX; std::vector> aliases; std::vector> shared_outputs; std::vector resets; @@ -160,6 +164,8 @@ int main(int argc, char** argv) { share_io = true; } else if (arg == "--misalign") { misalign = 1; + } else if (arg == "--unbound-output" && a + 1 < argc) { + unbound_output = std::strtoul(argv[++a], nullptr, 10); } else if (arg == "--alias" && a + 1 < argc) { unsigned input = 0, output = 0; if (std::sscanf(argv[++a], "%u:%u", &input, &output) != 2) @@ -243,7 +249,8 @@ int main(int argc, char** argv) { } } for (size_t o = 0; o < instance.outputs.size(); ++o) { - if (meta->output_tensor_meta(o).get().is_memory_planned()) + if (meta->output_tensor_meta(o).get().is_memory_planned() || + o == unbound_output) continue; void* buffer = instance.outputs[o]->data; for (const auto& [input, output] : aliases) { diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index f4ea826a04b..df7c40d033b 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -170,21 +170,25 @@ def from_bytes(data: bytes, like: torch.Tensor, channels_last: bool) -> torch.Te return tensor.permute(0, 3, 1, 2) if channels_last else tensor -def run_host(tmp_path: Path, program: bytes, steps, *args): - """ - Runs one list of input tensors per step. Returns the output files per - instance and step, and the delegates' memory statistics. - """ +def run_runner(tmp_path: Path, program: bytes, steps, *args): (tmp_path / "program.pte").write_bytes(program) for step, inputs in enumerate(steps): for i, tensor in enumerate(inputs): (tmp_path / f"in_{step}_{i}.bin").write_bytes(to_bytes(tensor)) - result = subprocess.run( + return subprocess.run( [RUNNER, str(tmp_path / "program.pte"), str(tmp_path), str(len(steps))] + [str(arg) for arg in args], capture_output=True, text=True, ) + + +def run_host(tmp_path: Path, program: bytes, steps, *args): + """ + Runs one list of input tensors per step. Returns the output files per + instance and step, and the delegates' memory statistics. + """ + result = run_runner(tmp_path, program, steps, *args) assert result.returncode == 0, result.stdout + result.stderr (stats_line,) = [ line for line in result.stdout.splitlines() if line.startswith("STATS ") @@ -403,6 +407,17 @@ def test_unplanned_io_shared_with_the_caller(tmp_path): assert_matches(outputs[step], as_list(emulator(*inputs))) +def test_output_without_memory_is_rejected(tmp_path): + model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + _, program = lower(model, shapes, alloc_graph_output=False) + result = run_runner( + tmp_path, program, random_steps(shapes, 1), "--unbound-output", 0 + ) + assert result.returncode != 0 + assert "output 0 has no data" in result.stdout + assert "execute (18)" in result.stderr # Error::InvalidArgument + + def test_version_6_graphs_still_run(tmp_path): model, shapes = lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] exported, program = lower(model, shapes, xts3_version=6) From 7d44dd0ffd321f7c05ac4164f98eb818405f933b Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:39:35 -0800 Subject: [PATCH 20/25] Require 16-byte aligned buffers in Int8Interpreter::bind_output The ESP32-S3 kernels may rely on the arena's 16-byte alignment for their outputs, and EspNNBackend checked that before binding an ExecuTorch tensor, but Int8Interpreter::bind_output() itself took any pointer, so another caller of the interpreter could bind a buffer that a vector store would write around. bind_output() now refuses a null or misaligned buffer and documents the requirement, and EspNNBackend relies on it, staging any output it refuses. The interpreter runner checks that a misaligned buffer is refused before it binds the aligned one. --- .../include/espnn/int8_interpreter.h | 11 ++++++----- .../espressif/espnn_runtime/src/espnn_backend.cpp | 15 ++++----------- .../espnn_runtime/src/int8_interpreter.cpp | 2 ++ .../test/host/espnn_host_interpreter.cpp | 10 +++++++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 4750cf80b29..6415c15f3a4 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -135,12 +135,13 @@ class Int8Interpreter { /** * With Int8RuntimeOptions::external_io, makes @p data, a device-layout * buffer of output_info().bytes, the storage of output @p index until it is - * bound again. An output with an arena slot cannot be bound: output_data() - * returns the slot. + * bound again. The buffer must be 16-byte aligned like the arena, since the + * kernels may rely on that. An output with an arena slot cannot be bound: + * output_data() returns the slot. * - * @retval false external_io is off, @p index is out of range, or the output - * has an arena slot (it is a graph input, an operation reads it, or - * conversion_slots selects it). + * @retval false external_io is off, @p index is out of range, @p data is + * null or misaligned, or the output has an arena slot (it is a graph + * input, an operation reads it, or conversion_slots selects it). */ bool bind_output(size_t index, int8_t* data); /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 19ff3adc145..b27b999f271 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -116,12 +116,6 @@ Error missing_data(const char* kind, size_t index) { return Error::InvalidArgument; } -// The kernels may rely on the activation arena's 16-byte alignment, so only -// buffers aligned like it are used in place. -bool arena_aligned(const void* data) { - return reinterpret_cast(data) % 16 == 0; -} - bool load_input( const Int8TensorInfo& info, Layout layout, @@ -414,12 +408,11 @@ class EspNNBackendInterface final : public BackendInterface { if (ok && state->returned_again(o)) { staged_outputs |= 1U << o; } else if (ok) { + // bind_output() refuses a misaligned buffer. Tensor tensor = args[inputs + o]->toTensor(); - if (tensor.scalar_type() == ScalarType::Char && - layout == Layout::kDevice && - arena_aligned(tensor.const_data_ptr())) { - ok = interpreter.bind_output(o, tensor.mutable_data_ptr()); - } else { + if (tensor.scalar_type() != ScalarType::Char || + layout != Layout::kDevice || + !interpreter.bind_output(o, tensor.mutable_data_ptr())) { ok = interpreter.bind_output( o, state->staging + state->staging_offset(o)); staged_outputs |= 1U << o; diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 4bb3fc6f19a..50b4bec5296 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -1044,6 +1044,8 @@ int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { bool Int8Interpreter::bind_output(size_t index, int8_t* data) { if (!loaded_ || !options_.external_io || index >= output_count_ || + data == nullptr || + reinterpret_cast(data) % ARENA_ALIGNMENT != 0 || in_arena(tensor_data_[outputs_[index]])) return false; tensor_data_[outputs_[index]] = data; diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp index 0e563741cea..d0a2851b7bd 100644 --- a/backends/espressif/test/host/espnn_host_interpreter.cpp +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -20,7 +20,8 @@ * device-layout (NHWC) int8 files. --constants-to-psram sets * Int8RuntimeOptions::copy_constants_to_psram. When the interpreter copies the * graph, the runner frees its own copy, as EspNNBackend does. With - * --external-io, outputs are bound to runner buffers prefilled with 0x5a. + * --external-io, outputs are bound to 16-byte aligned runner buffers prefilled + * with 0x5a, after checking that a misaligned buffer is refused. */ #include @@ -98,10 +99,13 @@ int main(int argc, char** argv) { for (size_t o = 0; options.external_io && o < bound.size(); ++o) { Int8TensorInfo info{}; interpreter.output_info(o, info); - bound[o].resize(info.bytes / 8 + 2); + bound[o].resize(info.bytes / 8 + 3); int8_t* buffer = reinterpret_cast(bound[o].data()); + buffer += (16 - reinterpret_cast(buffer) % 16) % 16; std::memset(buffer, 0x5a, info.bytes); - interpreter.bind_output(o, buffer); + if (interpreter.bind_output(o, buffer + 1) || + !interpreter.bind_output(o, buffer)) + return fail("bound a misaligned output or refused an aligned one"); } for (size_t i = 0; i < inputs.size(); ++i) { Int8TensorInfo info{}; From a337dfd6d1ded9bf99fae79a700f3b9f3ddb0019 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:39:52 -0800 Subject: [PATCH 21/25] Check ESP-NN kernel extents when exporting The device loader rejects a convolution, pooling or fully connected operation whose input, output or weight dimensions exceed 65535, because ESP-NN takes them as uint16_t, but check_runtime_limits() did not, so such a program exported and then failed at init() on the device. check_runtime_limits() now applies the same limits as the loader, with tests on both sides of 65535 for each operation. --- backends/espressif/serialization/xts3.py | 21 +++++++++++++ backends/espressif/test/test_contracts.py | 31 +++++++++++++++++++ .../espressif/test/test_xts3_parser_host.py | 6 ++-- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index 2c39e33d67e..e867e00d122 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -62,6 +62,9 @@ UPSAMPLE_WEIGHT_ONE = 1 << UPSAMPLE_WEIGHT_BITS TRANSPOSE_CONV_PARAMS = "<13i" UPSAMPLE_MAX_EXTENT = 65535 +# ESP-NN's convolution, pooling and fully connected kernels take dimensions as +# uint16_t. +KERNEL_MAX_EXTENT = 65535 MAX_GRAPH_IO = 16 MAX_GRAPH_STATES = 16 MAX_GRAPH_TENSORS = 1024 @@ -2056,6 +2059,24 @@ def check_runtime_limits(graph: XTS3Graph) -> None: # noqa: C901 f"Upsample output {out_h}x{out_w} exceeds " f"{UPSAMPLE_MAX_EXTENT} in one dimension" ) + if operation.opcode in ( + XTS3OpCode.CONV2D, + XTS3OpCode.DEPTHWISE_CONV2D, + XTS3OpCode.MAX_POOL2D, + XTS3OpCode.AVG_POOL2D, + ): + source = graph.tensors[operation.inputs[0]].shape + target = graph.tensors[operation.outputs[0]].shape + extents = [*source[1:], *target[1:]] + elif operation.opcode == XTS3OpCode.LINEAR: + extents = list(graph.tensors[operation.inputs[1]].shape) + else: + extents = [] + if extents and max(extents) > KERNEL_MAX_EXTENT: + raise ValueError( + f"{XTS3OpCode(operation.opcode).name} has a dimension above " + f"{KERNEL_MAX_EXTENT}, which ESP-NN takes as uint16_t" + ) for tensor in graph.tensors: if ( tensor.flags & TENSOR_FLAG_CONSTANT diff --git a/backends/espressif/test/test_contracts.py b/backends/espressif/test/test_contracts.py index 4d68ae301d2..729b14d4c32 100644 --- a/backends/espressif/test/test_contracts.py +++ b/backends/espressif/test/test_contracts.py @@ -43,7 +43,10 @@ MAX_GRAPH_OPERATIONS, TENSOR_FLAG_CONSTANT, XTS3DType, + XTS3Graph, XTS3OpCode, + XTS3Operation, + XTS3Tensor, ) from executorch.backends.espressif.test.test_int8_ops import quantize # noqa: E402 from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( # noqa: E402 @@ -175,6 +178,34 @@ def test_runtime_limits_reject_misaligned_int32_constants(): check_runtime_limits(dataclasses.replace(graph, tensors=tensors)) +@pytest.mark.parametrize( + "opcode, shapes", + [ + (XTS3OpCode.CONV2D, lambda n: [(1, n, 1, 1), (1, 1, 1, 1)]), + (XTS3OpCode.DEPTHWISE_CONV2D, lambda n: [(1, 1, 1, 1), (1, 1, n, 1)]), + (XTS3OpCode.MAX_POOL2D, lambda n: [(1, 1, 1, n - 1), (1, 1, 1, n)]), + (XTS3OpCode.AVG_POOL2D, lambda n: [(1, 1, n, 1), (1, 1, 1, 1)]), + (XTS3OpCode.LINEAR, lambda n: [(1, n), (1, n), (1, 1)]), + ], + ids=["conv", "depthwise", "max-pool", "avg-pool", "linear"], +) +def test_runtime_limits_follow_esp_nn_extents(opcode, shapes): + def graph(extent): + tensors = [ + XTS3Tensor(k, shape, XTS3DType.INT8) + for k, shape in enumerate(shapes(extent)) + ] + output = len(tensors) - 1 + operation = XTS3Operation(0, opcode, list(range(output)), [output], b"") + return XTS3Graph( + tensors=tensors, operations=[operation], inputs=[0], outputs=[output] + ) + + check_runtime_limits(graph(65535)) + with pytest.raises(ValueError, match="uint16_t"): + check_runtime_limits(graph(65536)) + + class DropoutClassifier(nn.Module): """MobileNetV3's classifier: Linear, Hardswish, in-place Dropout, Linear.""" diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py index f5f036ae5f2..4ef71fd7ddd 100644 --- a/backends/espressif/test/test_xts3_parser_host.py +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -188,9 +188,11 @@ def _wide_graph(opcode, width, params, weight=False): return xts3.serialize_graph(graph) -def test_parser_rejects_outputs_wider_than_esp_nn_supports(parser): +def test_parser_rejects_outputs_wider_than_esp_nn_supports(parser, monkeypatch): # ESP-NN's pooling and convolution kernels take uint16_t dimensions: a - # (1, 2) window with padding (0, 1) widens 65535 to 65536. + # (1, 2) window with padding (0, 1) widens 65535 to 65536. The serializer + # refuses such graphs too, so its check is skipped to build them. + monkeypatch.setattr(xts3, "check_runtime_limits", lambda graph: None) pool = struct.pack(xts3.INT8_POOL_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, -128, 127) conv = struct.pack( xts3.INT8_CONV_BASE_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, 1, -128, 127 From 3eb5858bef8230d3cf52864efbb6e75f0957b913 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:40:23 -0800 Subject: [PATCH 22/25] Give a method's ESP-NN state to one delegate or none With delegate_mutable_buffers, the partitioner gave each mutable buffer to whichever partition held its accesses. A method whose buffers fell in two partitions then had its state in two delegates, and since each delegate applies a reset at the start of its own execute(), a reset could reach one part of the state and not yet the other. A partition with more than 16 buffers failed to export, and owning a buffer could leave a partition with no runtime input or output, which an XTS3 graph needs. The partitioner now gives a method's buffers to the delegate only when one partition holds all of them and still has a runtime input and output once it owns them; constants do not count, since a shared one is duplicated into the partition. Otherwise the buffers stay in the program. A partition takes at most 16 buffers and leaves the rest in the program. Each case logs a warning. Python tests cover two partitions, 17 buffers, a state that is the only input or output, and a constant shared with the CPU, and a host test runs the two-partition program. --- backends/espressif/README.md | 6 +- .../espressif/partition/espnn_partitioner.py | 102 ++++++++- backends/espressif/test/test_host_runtime.py | 16 ++ .../espressif/test/test_mutable_buffers.py | 208 +++++++++++++++++- 4 files changed, 309 insertions(+), 23 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 2d625c312ad..2ad7ad81308 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -186,8 +186,10 @@ print(report["delegated_buffers"]) # ['frames'] The delegate takes a buffer only when one partition holds every access to it and the buffer is read and written with the same int8 quantization, as a -`cat` and slice shift register is with `EspNNQuantizer`. Other buffers stay -in the program, with a warning. +`cat` and slice shift register is with `EspNNQuantizer`. It takes a method's +buffers only when they all end up in one partition that keeps another input +and output, and at most 16 of them. Other buffers stay in the program, with a +warning. On the device, each loaded method has its own state. It starts from the buffer's initial value when the method is loaded and persists across diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index d4ff36c2d5c..657043608bf 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -33,6 +33,7 @@ INT8_PASSTHROUGH_MARKERS, lower_int8_node, LUT_KINDS, + MAX_GRAPH_STATES, TensorRegistry, ) from executorch.exir.backend.compile_spec_schema import CompileSpec @@ -303,11 +304,59 @@ def _int8_state_compatible(buffer: torch.fx.Node, writer: torch.fx.Node) -> bool return len(reads) == 1 and None not in reads and _qparams(writer, producer) in reads +def _keeps_runtime_io( + exported_program: ExportedProgram, kept, released, writers +) -> bool: + """ + Whether the partition that would own the kept buffers still has a runtime + input and a runtime output, which its XTS3 graph needs. Constants do not + count, since a shared one is duplicated into the partition, and the + released buffers become runtime inputs. + """ + signature = exported_program.graph_signature + constants = { + *signature.inputs_to_parameters, + *signature.inputs_to_lifted_tensor_constants, + *( + name + for name, buffer in signature.inputs_to_buffers.items() + if buffer not in writers + ), + } + tag = kept[0][0].meta["delegation_tag"] + released_nodes = {node for node, _ in released} + nodes = { + node + for node in exported_program.graph.nodes + if node.meta.get("delegation_tag") == tag and node not in released_nodes + } + updates = {writers[buffer] for _, buffer in kept} + + def runtime(node): + return node.op != "get_attr" and node.name not in constants + + has_input = any( + source not in nodes and runtime(source) + for node in nodes + for source in node.all_input_nodes + ) + has_output = any( + runtime(node) + and user not in nodes + and not (user.op == "output" and node in updates) + for node in nodes + for user in node.users + ) + return has_input and has_output + + def _tag_quantized_mutable_buffers(exported_program: ExportedProgram) -> list[str]: """ - Give each mutable buffer to the partition that holds all its accesses, - unless the delegate could not keep it as int8. Returns the delegated - buffers' names. + Give the mutable buffers to the partition that holds all their accesses, + unless the delegate could not keep one as int8, they would end up in + several partitions or leave it without a runtime input or output, or there + are more than MAX_GRAPH_STATES of them. Returns the delegated buffers' + names. """ tag_mutated_buffer(exported_program) signature = exported_program.graph_signature @@ -321,7 +370,7 @@ def _tag_quantized_mutable_buffers(exported_program: ExportedProgram) -> list[st if buffer not in writers or "delegation_tag" not in node.meta: continue if _int8_state_compatible(node, writers[buffer]): - delegated.append(buffer) + delegated.append((node, buffer)) continue del node.meta["delegation_tag"] logging.warning( @@ -329,7 +378,35 @@ def _tag_quantized_mutable_buffers(exported_program: ExportedProgram) -> list[st "owns a buffer only when it is read and written with the same " "int8 quantization." ) - return delegated + # A reset reaches each delegate at the start of its own execute(), so + # state split across partitions could be reset between them. + if len({node.meta["delegation_tag"] for node, _ in delegated}) > 1: + logging.warning( + "Mutable buffers stay in the program: the ESP-NN delegate owns a " + "method's buffers only when one partition holds all of them." + ) + left = delegated + elif delegated and not _keeps_runtime_io( + exported_program, + delegated[:MAX_GRAPH_STATES], + delegated[MAX_GRAPH_STATES:], + writers, + ): + logging.warning( + "Mutable buffers stay in the program: owning them would leave the " + "ESP-NN partition without a runtime input or output." + ) + left = delegated + else: + left = delegated[MAX_GRAPH_STATES:] + for _, buffer in left: + logging.warning( + f"Mutable buffer {buffer} stays in the program: an ESP-NN " + f"delegate owns at most {MAX_GRAPH_STATES} buffers." + ) + for node, _ in left: + del node.meta["delegation_tag"] + return [buffer for _, buffer in delegated[: len(delegated) - len(left)]] class EspNNPartitioner(Partitioner): @@ -343,13 +420,14 @@ class EspNNPartitioner(Partitioner): error, whether it is unsupported or its region could not be delegated; the standalone XTS3 firmware path requires this. - With delegate_mutable_buffers, a quantized mutable buffer (streaming state) - whose readers and writer all fall in one partition is owned by that - delegate: it leaves the program's inputs and outputs, and the delegate - keeps it as int8 between executions. The buffer must be float32 and read - only through quantize ops, or int8 and read only through dequantize ops, - with the same quantization as its new value. Any other buffer stays in the - program. + With delegate_mutable_buffers, the quantized mutable buffers (streaming + state) whose readers and writers all fall in one partition are owned by + that delegate: they leave the program's inputs and outputs, and the + delegate keeps them as int8 between executions. Each buffer must be + float32 and read only through quantize ops, or int8 and read only through + dequantize ops, with the same quantization as its new value. The delegate + takes a method's buffers only when they all fall in one partition, and at + most MAX_GRAPH_STATES of them. Any other buffer stays in the program. """ def __init__( diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index df7c40d033b..a8ba3185ec8 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -47,7 +47,9 @@ lower_input_as_state, lower_stateful, ShiftRegister, + split_streams, StateReadLate, + TwoStreams, ) from executorch.exir import ExecutorchBackendConfig from executorch.exir.backend.compile_spec_schema import CompileSpec @@ -738,6 +740,20 @@ def test_state_reset_and_instances(tmp_path): assert_matches(second[step], expected) +def test_state_in_two_partitions_stays_in_the_program(tmp_path, monkeypatch): + split_streams(monkeypatch) + exported, program, report = lower_stateful( + TwoStreams(), inputs=2, allow_cpu_fallback=True + ) + assert report["partitions"] == 2 and not report["delegated_buffers"] + steps = random_steps([(1, 4, 1, 1)] * 2, 5) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["delegates"] == 2 and stats["states"] == 0 + + def test_streaming_example_runs_without_copies(tmp_path): sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "examples/espressif")) import export_streaming_model as example diff --git a/backends/espressif/test/test_mutable_buffers.py b/backends/espressif/test/test_mutable_buffers.py index 7f00188b7e3..74d1a052c45 100644 --- a/backends/espressif/test/test_mutable_buffers.py +++ b/backends/espressif/test/test_mutable_buffers.py @@ -30,6 +30,7 @@ from executorch.backends.espressif.reference import ProgramEmulator from executorch.backends.espressif.serialization import xts3 from executorch.exir.lowered_backend_module import LoweredBackendModule +from torch.fx.passes.infra.partitioner import Partition from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e @@ -80,6 +81,47 @@ def forward(self, x): return y +class StateOnlyInput(torch.nn.Module): + """Explicit int8 Q/DQ recurrence whose only input is its state.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + def forward(self): + qdq = InputAsState.qdq + state = qdq(self.state, 0.1) + update = qdq(state + state, 0.1) + self.state.copy_(update) + return qdq(update + state, 0.1) + + +class StateOnlyOutput(torch.nn.Module): + """Explicit int8 Q/DQ accumulator whose only delegated output is its state.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + def forward(self, x): + qdq = InputAsState.qdq + self.state.copy_(qdq(qdq(self.state, 0.1) + qdq(x, 0.1), 0.1)) + return torch.cumsum(x, dim=1) + + +class StateOnlyOutputWithConstant(StateOnlyOutput): + """StateOnlyOutput whose CPU operator reads a constant.""" + + def __init__(self): + super().__init__() + self.register_buffer("offset", torch.full((1, 4, 1, 1), 0.2)) + + def forward(self, x): + qdq = InputAsState.qdq + self.state.copy_(qdq(qdq(self.state, 0.1) + qdq(x, 0.1), 0.1)) + return torch.cumsum(x + self.offset, dim=1) + + def lower_input_as_state(): program = torch.export.export( InputAsState().eval(), (torch.zeros(1, 4, 1, 1),), strict=True @@ -99,30 +141,107 @@ def forward(self, x): return self.conv(window) -def quantize_stateful(model): +class TwoStreams(torch.nn.Module): + """Two shift registers on separate inputs, which partitions may split.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 2, 1)) + self.register_buffer("other", torch.zeros(1, 4, 2, 1)) + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + self.other_conv = torch.nn.Conv2d(4, 8, (3, 1)) + + def forward(self, x, y): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:]) + other = torch.cat([self.other, y], dim=2) + self.other.copy_(other[:, :, 1:]) + return self.conv(window), self.other_conv(other) + + +def split_streams(monkeypatch): + """Makes EspNNPartitioner propose each connected part as a partition.""" + propose = partitioner.CapabilityBasedPartitioner.propose_partitions + + def propose_streams(self): + streams = [] + for partition in propose(self): + nodes = set(partition.nodes) + while nodes: + stream, frontier = set(), [nodes.pop()] + while frontier: + node = frontier.pop() + stream.add(node) + for other in [*node.all_input_nodes, *node.users]: + if other in nodes: + nodes.discard(other) + frontier.append(other) + streams.append(Partition(nodes=stream)) + return streams + + monkeypatch.setattr( + partitioner.CapabilityBasedPartitioner, "propose_partitions", propose_streams + ) + + +class ManyStates(torch.nn.Module): + """More shift registers than one delegate can own.""" + + def __init__(self, count: int): + super().__init__() + for k in range(count): + self.register_buffer(f"state_{k}", torch.zeros(1, 4, 2, 1)) + self.convs = torch.nn.ModuleList( + torch.nn.Conv2d(4, 2, (3, 1)) for _ in range(count) + ) + + def forward(self, x): + outputs = [] + for k, conv in enumerate(self.convs): + state = getattr(self, f"state_{k}") + window = torch.cat([state, x], dim=2) + state.copy_(window[:, :, 1:]) + outputs.append(conv(window)) + return torch.cat(outputs, dim=1) + + +def quantize_stateful(model, inputs: int = 1): shape = (1, 4, 1, 1) generator = torch.Generator().manual_seed(5) + example = tuple(torch.zeros(shape) for _ in range(inputs)) prepared = prepare_pt2e( - torch.export.export(model.eval(), (torch.zeros(shape),), strict=True).module(), + torch.export.export(model.eval(), example, strict=True).module(), EspNNQuantizer(), ) for _ in range(16): - prepared(torch.randn(shape, generator=generator)) + prepared(*(torch.randn(shape, generator=generator) for _ in example)) converted = convert_pt2e(prepared) # Calibration ran the model, so start again from the initial state. - converted.state.zero_() + for name, _ in model.named_buffers(): + getattr(converted, name).zero_() return converted -def lower_stateful(model, delegate_mutable_buffers: bool = True, quantized=None): - converted = copy.deepcopy(quantized) if quantized else quantize_stateful(model) - program = torch.export.export(converted, (torch.zeros(1, 4, 1, 1),), strict=True) +def lower_stateful( + model, + delegate_mutable_buffers: bool = True, + quantized=None, + allow_cpu_fallback: bool = False, + inputs: int = 1, +): + converted = ( + copy.deepcopy(quantized) if quantized else quantize_stateful(model, inputs) + ) + example = tuple(torch.zeros(1, 4, 1, 1) for _ in range(inputs)) + program = torch.export.export(converted, example, strict=True) edge, report = to_espnn_edge( program, - allow_cpu_fallback=False, + allow_cpu_fallback=allow_cpu_fallback, delegate_mutable_buffers=delegate_mutable_buffers, ) - return edge.exported_program(), edge.to_executorch().buffer, report + # to_executorch() rewrites the mutations of buffers left in the program. + exported = copy.deepcopy(edge.exported_program()) + return exported, edge.to_executorch().buffer, report def delegate_graphs(exported) -> list[xts3.XTS3Graph]: @@ -224,3 +343,74 @@ def test_state_updated_from_a_graph_input(): ) torch.testing.assert_close(emulator(frame), expected) previous = frame + + +def test_state_in_two_partitions_stays_in_the_program(monkeypatch): + exported, _, report = lower_stateful(TwoStreams(), inputs=2) + assert sorted(report["delegated_buffers"]) == ["other", "state"] + # A reset could reach one partition's state and not yet the other's, so + # neither delegate owns its buffer. + split_streams(monkeypatch) + exported, _, report = lower_stateful( + TwoStreams(), inputs=2, allow_cpu_fallback=True + ) + assert report["partitions"] == 2 + assert report["delegated_buffers"] == [] + assert sorted(exported.graph_signature.buffers_to_mutate.values()) == [ + "other", + "state", + ] + assert not any(graph.states for graph in delegate_graphs(exported)) + + +def test_a_delegate_owns_at_most_16_buffers(): + model = ManyStates(xts3.MAX_GRAPH_STATES + 1) + exported, _, report = lower_stateful(model) + assert len(report["delegated_buffers"]) == xts3.MAX_GRAPH_STATES + assert len(exported.graph_signature.buffers_to_mutate) == 1 + (graph,) = delegate_graphs(exported) + assert len(graph.states) == xts3.MAX_GRAPH_STATES + + +@pytest.mark.parametrize( + "model, example", + [ + (StateOnlyInput(), ()), + (StateOnlyOutput(), (torch.zeros(1, 4, 1, 1),)), + ], + ids=["only-input", "only-output"], +) +def test_state_that_is_a_partition_boundary_stays_in_the_program(model, example): + # Owning the buffer would leave the partition without a runtime input or + # output, which an XTS3 graph needs. + program = torch.export.export(model.eval(), example, strict=True) + edge, report = to_espnn_edge(program, delegate_mutable_buffers=True) + assert report["partitions"] == 1 and report["delegated_buffers"] == [] + edge.to_executorch() + + +def test_constant_shared_with_the_cpu_is_not_a_runtime_output(monkeypatch): + # tag_constant_data gives a constant that the CPU also reads to the + # partition, as it would one that the partition reads too, and lowering + # then duplicates it for the CPU, so it is not a runtime output. + tag_constants = partitioner.tag_constant_data + + def tag_with_offset(program): + tag_constants(program) + buffers = program.graph_signature.inputs_to_buffers + nodes = list(program.graph.nodes) + (tag,) = { + node.meta["delegation_tag"] + for node in nodes + if "delegation_tag" in node.meta + } + (offset,) = [node for node in nodes if buffers.get(node.name) == "offset"] + offset.meta["delegation_tag"] = tag + + monkeypatch.setattr(partitioner, "tag_constant_data", tag_with_offset) + program = torch.export.export( + StateOnlyOutputWithConstant().eval(), (torch.zeros(1, 4, 1, 1),), strict=True + ) + edge, report = to_espnn_edge(program, delegate_mutable_buffers=True) + assert report["delegated_buffers"] == [] + edge.to_executorch() From 7b123235e42d1d0af12944bf3f1df08649007481 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:40:34 -0800 Subject: [PATCH 23/25] Return a copy of an ESP-NN state that the partition also returns A delegate-owned state whose old value the partition also returned, directly or through an operation lowering treats as an alias such as clone, was serialized as both a state and a graph output. The runtime rejects that, so the program exported and then failed at init(). The serializer now returns such a state through a COPY after every other operation, the same way it separates a state's update from a graph output. A Python test checks the graph, and a host test runs both forms against the reference. --- backends/espressif/serialization/xts3.py | 51 ++++++++++++------- backends/espressif/test/test_host_runtime.py | 13 +++++ .../espressif/test/test_mutable_buffers.py | 42 +++++++++++++-- 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index e867e00d122..99069320356 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -1832,12 +1832,13 @@ def _mark_int8_graph_io( # noqa: C901 def _mark_int8_state( - exported_program, registry: TensorRegistry + exported_program, registry: TensorRegistry, graph_outputs: list[int] ) -> list[tuple[int, int]]: """ Pair each mutable buffer of the partition with the int8 tensor holding its new value. The buffer's tensor becomes TENSOR_FLAG_STATE, with its - quantized initial value in the constant region. + quantized initial value in the constant region. A state cannot be a graph + output, so an old value that the partition returns is returned as a copy. """ signature = exported_program.graph_signature if not signature.buffers_to_mutate: @@ -1894,9 +1895,36 @@ def _mark_int8_state( registry.constant_data.extend(raw) state.flags |= TENSOR_FLAG_STATE states.append((state.tensor_id, update.tensor_id)) + if state.flags & TENSOR_FLAG_GRAPH_OUTPUT: + old_value = _copy_of(state, registry) + state.flags &= ~TENSOR_FLAG_GRAPH_OUTPUT + old_value.flags |= TENSOR_FLAG_GRAPH_OUTPUT + graph_outputs[:] = [ + old_value.tensor_id if output == state.tensor_id else output + for output in graph_outputs + ] return _separate_state_updates(states, registry) +def _copy_of(tensor: XTS3Tensor, registry: TensorRegistry) -> XTS3Tensor: + """A new int8 tensor that a COPY after every other operation fills.""" + from .int8_ops import copy_operation, copy_segment + + copy = XTS3Tensor( + tensor_id=len(registry.tensors), + shape=tuple(tensor.shape), + dtype=XTS3DType.INT8, + quant_scale=tensor.quant_scale, + quant_zero_point=tensor.quant_zero_point, + ) + registry.tensors.append(copy) + segment = copy_segment(0, [prod(tensor.shape)], [1], 0, [1], 0, tensor, copy) + registry.extra_operations.append( + copy_operation([tensor.tensor_id], copy.tensor_id, [segment]) + ) + return copy + + def _separate_state_updates( states: list[tuple[int, int]], registry: TensorRegistry ) -> list[tuple[int, int]]: @@ -1905,8 +1933,6 @@ def _separate_state_updates( state's slot: a graph input or output, a constant, a state, or an update shared by several states. The copies run after every other operation. """ - from .int8_ops import copy_operation, copy_segment - shared = ( TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT @@ -1917,20 +1943,7 @@ def _separate_state_updates( for index, (state_id, update_id) in enumerate(states): update = registry.tensors[update_id] if update.flags & shared or update_id in used: - copy = XTS3Tensor( - tensor_id=len(registry.tensors), - shape=tuple(update.shape), - dtype=XTS3DType.INT8, - quant_scale=update.quant_scale, - quant_zero_point=update.quant_zero_point, - ) - registry.tensors.append(copy) - elements = prod(update.shape) - segment = copy_segment(0, [elements], [1], 0, [1], 0, update, copy) - registry.extra_operations.append( - copy_operation([update_id], copy.tensor_id, [segment]) - ) - update_id = copy.tensor_id + update_id = _copy_of(update, registry).tensor_id states[index] = (state_id, update_id) used.add(update_id) return states @@ -1958,7 +1971,7 @@ def build_graph(exported_program) -> XTS3Graph: lowered.op_id = len(operations) operations.append(lowered) graph_inputs, graph_outputs = _mark_int8_graph_io(exported_program, registry) - states = _mark_int8_state(exported_program, registry) + states = _mark_int8_state(exported_program, registry, graph_outputs) for extra in registry.extra_operations: extra.op_id = len(operations) operations.append(extra) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index a8ba3185ec8..5a2f1ba2518 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -46,6 +46,8 @@ InputAsState, lower_input_as_state, lower_stateful, + ReturnsClonedOldState, + ReturnsOldState, ShiftRegister, split_streams, StateReadLate, @@ -725,6 +727,17 @@ def test_state_updated_from_a_graph_input(tmp_path): assert stats["states"] == 1 +@pytest.mark.parametrize("model_class", [ReturnsOldState, ReturnsClonedOldState]) +def test_returned_old_state(tmp_path, model_class): + exported, program, _ = lower_input_as_state(model_class) + steps = stateful_steps(4) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + + def test_state_reset_and_instances(tmp_path): exported, program, _ = lower_stateful(ShiftRegister()) steps = stateful_steps(6) diff --git a/backends/espressif/test/test_mutable_buffers.py b/backends/espressif/test/test_mutable_buffers.py index 74d1a052c45..a97e1e5a332 100644 --- a/backends/espressif/test/test_mutable_buffers.py +++ b/backends/espressif/test/test_mutable_buffers.py @@ -81,6 +81,28 @@ def forward(self, x): return y +class ReturnsOldState(InputAsState): + """InputAsState that also returns the state it read.""" + + def forward(self, x): + old = self.qdq(self.state, 0.05) + frame = self.qdq(x, 0.05) + y = self.qdq(old + frame, 0.1) + self.state.copy_(frame) + return y, old + + +class ReturnsClonedOldState(InputAsState): + """ReturnsOldState with the old state returned through a clone.""" + + def forward(self, x): + old = self.qdq(self.state, 0.05) + frame = self.qdq(x, 0.05) + y = self.qdq(old + frame, 0.1) + self.state.copy_(frame) + return y, self.qdq(old.clone(), 0.05) + + class StateOnlyInput(torch.nn.Module): """Explicit int8 Q/DQ recurrence whose only input is its state.""" @@ -122,14 +144,16 @@ def forward(self, x): return torch.cumsum(x + self.offset, dim=1) -def lower_input_as_state(): +def lower_input_as_state(model_class=InputAsState): program = torch.export.export( - InputAsState().eval(), (torch.zeros(1, 4, 1, 1),), strict=True + model_class().eval(), (torch.zeros(1, 4, 1, 1),), strict=True ) edge, report = to_espnn_edge( program, allow_cpu_fallback=False, delegate_mutable_buffers=True ) - return edge.exported_program(), edge.to_executorch().buffer, report + # to_executorch() rewrites the mutations of buffers left in the program. + exported = copy.deepcopy(edge.exported_program()) + return exported, edge.to_executorch().buffer, report class ScaledState(ShiftRegister): @@ -323,6 +347,18 @@ def test_int8_state_compatibility(dtype, reader, writer, writer_scale, owned): assert partitioner._int8_state_compatible(buffer, new_value) == owned +@pytest.mark.parametrize("model_class", [ReturnsOldState, ReturnsClonedOldState]) +def test_returned_old_state_is_a_copy(model_class): + # A state cannot also be a graph output, so the graph returns a copy of it. + exported, _, report = lower_input_as_state(model_class) + assert report["delegated_buffers"] == ["state"] + (graph,) = delegate_graphs(exported) + ((state, _),) = graph.states + assert state not in graph.outputs + (copy,) = [op for op in graph.operations if op.inputs == [state]] + assert copy.opcode == xts3.XTS3OpCode.COPY and copy.outputs[0] in graph.outputs + + def test_state_updated_from_a_graph_input(): exported, _, report = lower_input_as_state() assert report["delegated_buffers"] == ["state"] From f20454ad5603ebd7a24e8334a6f19673408714f4 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:41:00 -0800 Subject: [PATCH 24/25] Reset ESP-NN delegate state by group reset_delegate_states() and the reset_states backend option reset the state of every EspNNBackend delegate, so an application with several stateful methods could reset one of them only by loading it again. A method can now be loaded with a reset_group runtime spec from 0 to 7, passed through the existing LoadBackendOptionsMap, and reset_states accepts that group as an int, as does the new reset_delegate_states(int). Each group keeps its own generation counter, so a delegate restores its initial state at the start of its next execute() only when its own group was reset. Loading with an invalid group fails with Error::InvalidArgument. The host runner loads instance k in group k, and a host test resets one of two instances. --- backends/espressif/README.md | 22 ++++++-- .../include/espnn/espnn_backend.h | 22 ++++++-- .../espnn_runtime/src/espnn_backend.cpp | 51 ++++++++++++++---- .../espressif/test/host/espnn_host_runner.cpp | 52 ++++++++++++++----- backends/espressif/test/test_host_runtime.py | 22 ++++++++ 5 files changed, 138 insertions(+), 31 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 2ad7ad81308..d37e1996882 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -193,11 +193,13 @@ warning. On the device, each loaded method has its own state. It starts from the buffer's initial value when the method is loaded and persists across -`execute()` calls. `reset_delegate_states()` (from `espnn/espnn_backend.h`) -or the `reset_states` backend option restores the initial state of every -EspNNBackend delegate before its next `execute()`; to reset one method alone, -load it again. After an `execute()` that fails, the state is undefined until -it is reset. The reset may be requested from any task. +`execute()` calls. `reset_delegate_states()` (from `espnn/espnn_backend.h`), +or the `reset_states` backend option set to `true`, restores the initial +state of every EspNNBackend delegate before its next `execute()`. To reset +some methods alone, load them with a `reset_group` from 0 to 7 and pass that +group to `reset_states`, or load the method again. Buffers that stay in the +program are not reset this way. After an `execute()` that fails, the state is +undefined until it is reset. The reset may be requested from any task. ```cpp executorch::backends::espressif::reset_delegate_states(); @@ -205,6 +207,16 @@ executorch::backends::espressif::reset_delegate_states(); executorch::runtime::BackendOptions<1> options; options.set_option("reset_states", true); executorch::runtime::set_option("EspNNBackend", options.view()); + +// A method in reset group 1, reset alone: +executorch::runtime::BackendOptions<1> group; +group.set_option("reset_group", 1); +executorch::runtime::LoadBackendOptionsMap load_options; +load_options.set_options("EspNNBackend", group.view()); +auto method = program->load_method( + "forward", &memory_manager, nullptr, nullptr, &load_options); +options.set_option("reset_states", 1); +executorch::runtime::set_option("EspNNBackend", options.view()); ``` ### Streaming keyword spotting diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index 16842dd6942..822b8a949b3 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -20,10 +20,18 @@ namespace espressif { /// Backend ID that `EspNNBackend.preprocess()` writes into lowered programs. constexpr const char* kEspNNBackendId = "EspNNBackend"; -/// Backend option (bool) that calls reset_delegate_states() when true: -/// `executorch::runtime::set_option(kEspNNBackendId, options)`. +/// Backend option that resets delegate-owned state, through +/// `executorch::runtime::set_option(kEspNNBackendId, options)`: true calls +/// reset_delegate_states(), and a reset group (int) calls +/// reset_delegate_states(group). constexpr char kResetStatesOption[] = "reset_states"; +/// Load-time backend option (int, 0 by default) that puts the delegates of a +/// method in a reset group from 0 to kResetGroups - 1. Pass it to +/// `Program::load_method()` in a LoadBackendOptionsMap. +constexpr char kResetGroupOption[] = "reset_group"; +constexpr int kResetGroups = 8; + /** * Counters accumulated by every EspNNBackend delegate since startup or the * last reset_delegate_stats() call. @@ -81,10 +89,18 @@ void reset_delegate_stats(); * * Each loaded method instance has its own state, kept between executions * until this is called or the method is destroyed; loading the method again - * also starts from the initial state. Safe to call from any task. + * also starts from the initial state. Buffers that stay in the program are + * not reset. Safe to call from any task. */ void reset_delegate_states(); +/** + * Same, for the delegates loaded with kResetGroupOption set to @p group. + * + * @retval false @p group is not from 0 to kResetGroups - 1. + */ +bool reset_delegate_states(int group); + } // namespace espressif } // namespace backends } // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index b27b999f271..f1e0de90707 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -53,9 +53,9 @@ namespace espressif { namespace { DelegateStats g_stats{}; -// Incremented by each reset request; a delegate whose state is older restores -// its initial state before its next execute. -std::atomic g_state_generation{0}; +// Incremented by each reset of a group; a delegate whose state is older than +// its group restores its initial state before its next execute. +std::atomic g_state_generations[kResetGroups]{}; constexpr float kHalfEvenRoundingBias = 12582912.0f; @@ -202,6 +202,7 @@ struct DelegateState { Int8Interpreter interpreter; uint32_t conversion_slots = 0; bool profiled = false; + int reset_group = 0; uint32_t state_generation = 0; // Copies of outputs outside the arena whose memory is misaligned, in output // order. @@ -287,6 +288,16 @@ class EspNNBackendInterface final : public BackendInterface { BackendInitContext& context, FreeableBuffer* processed, ArrayRef compile_specs) const override { + const Result group = context.get_runtime_spec(kResetGroupOption); + const int reset_group = group.ok() ? group.get() : 0; + if ((!group.ok() && group.error() != Error::NotFound) || reset_group < 0 || + reset_group >= kResetGroups) { + std::printf( + "EspNN delegate: %s must be an int from 0 to %d\n", + kResetGroupOption, + kResetGroups - 1); + return Error::InvalidArgument; + } Int8RuntimeOptions options{}; for (const auto& spec : compile_specs) { if (std::strcmp(spec.key, "espnn_io_conversion_slots") == 0) { @@ -302,6 +313,7 @@ class EspNNBackendInterface final : public BackendInterface { if (memory == nullptr) return Error::MemoryAllocationFailed; DelegateState* state = new (memory) DelegateState(); + state->reset_group = reset_group; #if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM options.copy_constants_to_psram = true; #endif @@ -332,7 +344,7 @@ class EspNNBackendInterface final : public BackendInterface { if (state->interpreter.constants_copied()) processed->Free(); state->state_generation = - g_state_generation.load(std::memory_order_relaxed); + g_state_generations[reset_group].load(std::memory_order_relaxed); ++g_stats.delegates; g_stats.arena_bytes += state->interpreter.arena_bytes(); g_stats.scratch_bytes += state->interpreter.scratch_bytes(); @@ -361,8 +373,8 @@ class EspNNBackendInterface final : public BackendInterface { } const int64_t started = esp_timer_get_time(); if (interpreter.state_count() != 0) { - const uint32_t generation = - g_state_generation.load(std::memory_order_relaxed); + const uint32_t generation = g_state_generations[state->reset_group].load( + std::memory_order_relaxed); if (generation != state->state_generation) { interpreter.reset_states(); state->state_generation = generation; @@ -459,14 +471,23 @@ class EspNNBackendInterface final : public BackendInterface { BackendOptionContext&, const Span& backend_options) override { for (const BackendOption& option : backend_options) { - const bool* value = std::get_if(&option.value); - if (std::strcmp(option.key, kResetStatesOption) != 0 || - value == nullptr) { + if (std::strcmp(option.key, kResetStatesOption) != 0) { std::printf("EspNN delegate: unknown option %s\n", option.key); return Error::NotSupported; } - if (*value) + const bool* all = std::get_if(&option.value); + const int* group = std::get_if(&option.value); + if (all != nullptr && *all) { reset_delegate_states(); + } else if ( + all == nullptr && + (group == nullptr || !reset_delegate_states(*group))) { + std::printf( + "EspNN delegate: %s takes true or a reset group from 0 to %d\n", + kResetStatesOption, + kResetGroups - 1); + return Error::InvalidArgument; + } } return Error::Ok; } @@ -500,7 +521,15 @@ DelegateStats delegate_stats() { } void reset_delegate_states() { - g_state_generation.fetch_add(1, std::memory_order_relaxed); + for (std::atomic& generation : g_state_generations) + generation.fetch_add(1, std::memory_order_relaxed); +} + +bool reset_delegate_states(int group) { + if (group < 0 || group >= kResetGroups) + return false; + g_state_generations[group].fetch_add(1, std::memory_order_relaxed); + return true; } } // namespace espressif diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp index d5fc1001c68..7aa851bcc96 100644 --- a/backends/espressif/test/host/espnn_host_runner.cpp +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -15,7 +15,8 @@ * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] * [--alias INPUT:OUTPUT]... * [--share-output FIRST:SECOND]... - * [--misalign] [--reset-before STEP]... + * [--misalign] [--reset-before STEP[:GROUP]]... + * [--reset-groups] * [--unbound-output OUTPUT] * * Step s of instance k reads input i from IO_DIR/in__.bin and writes @@ -30,8 +31,9 @@ * --misalign places every runner buffer one byte past a 16-byte boundary. * --unbound-output never gives that unplanned output memory. * --reset-before asks EspNNBackend, through its reset_states backend option, - * to restore its delegates' state before that step. At the end, the runner - * prints the delegates' memory as a STATS line. + * to restore the state of its delegates, or of one reset group, before that + * step. --reset-groups loads instance k in reset group k. At the end, the + * runner prints the delegates' memory as a STATS line. */ #include @@ -45,6 +47,7 @@ #include #include +#include #include #include #include @@ -129,6 +132,8 @@ struct HostTensor { }; struct Instance { + executorch::runtime::BackendOptions<1> reset_group; + executorch::runtime::LoadBackendOptionsMap load_options; std::vector> planned; std::vector> spans; std::unique_ptr planned_allocator; @@ -153,13 +158,20 @@ int main(int argc, char** argv) { size_t unbound_output = SIZE_MAX; std::vector> aliases; std::vector> shared_outputs; - std::vector resets; + // (step, reset group or -1 for every delegate) + std::vector> resets; + bool reset_groups = false; for (int a = 4; a < argc; ++a) { const std::string arg = argv[a]; if (arg == "--instances" && a + 1 < argc) { instance_count = std::atoi(argv[++a]); } else if (arg == "--reset-before" && a + 1 < argc) { - resets.push_back(std::atoi(argv[++a])); + int step = 0, group = -1; + if (std::sscanf(argv[++a], "%d:%d", &step, &group) < 1) + fail("--reset-before takes STEP[:GROUP]"); + resets.emplace_back(step, group); + } else if (arg == "--reset-groups") { + reset_groups = true; } else if (arg == "--share-io") { share_io = true; } else if (arg == "--misalign") { @@ -195,7 +207,16 @@ int main(int argc, char** argv) { fail("no forward method", static_cast(meta.error())); std::vector instances(instance_count); - for (Instance& instance : instances) { + for (int k = 0; k < instance_count; ++k) { + Instance& instance = instances[k]; + if (reset_groups && + (instance.reset_group.set_option( + executorch::backends::espressif::kResetGroupOption, k) != + Error::Ok || + instance.load_options.set_options( + executorch::backends::espressif::kEspNNBackendId, + instance.reset_group.view()) != Error::Ok)) + fail("reset group options"); for (size_t b = 0; b < meta->num_memory_planned_buffers(); ++b) { instance.planned.emplace_back(meta->memory_planned_buffer_size(b).get()); } @@ -208,8 +229,12 @@ int main(int argc, char** argv) { instance.method_pool.size(), instance.method_pool.data()); instance.memory = std::make_unique( instance.method_allocator.get(), instance.planned_allocator.get()); - Result method = - program->load_method("forward", instance.memory.get()); + Result method = program->load_method( + "forward", + instance.memory.get(), + nullptr, + nullptr, + &instance.load_options); if (!method.ok()) fail("cannot load method", static_cast(method.error())); instance.method = std::make_unique(std::move(method.get())); @@ -269,13 +294,16 @@ int main(int argc, char** argv) { } for (int step = 0; step < steps; ++step) { - for (int reset : resets) { + for (const auto& [reset, group] : resets) { if (reset != step) continue; executorch::runtime::BackendOptions<1> options; - if (options.set_option( - executorch::backends::espressif::kResetStatesOption, true) != - Error::Ok) + const Error set = group < 0 + ? options.set_option( + executorch::backends::espressif::kResetStatesOption, true) + : options.set_option( + executorch::backends::espressif::kResetStatesOption, group); + if (set != Error::Ok) fail("BackendOptions::set_option"); Error error = executorch::runtime::set_option( executorch::backends::espressif::kEspNNBackendId, options.view()); diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py index 5a2f1ba2518..abb56d3daef 100644 --- a/backends/espressif/test/test_host_runtime.py +++ b/backends/espressif/test/test_host_runtime.py @@ -753,6 +753,28 @@ def test_state_reset_and_instances(tmp_path): assert_matches(second[step], expected) +def test_state_reset_by_group(tmp_path): + # Instance k is in reset group k, and only group 1 is reset. + exported, program, _ = lower_stateful(ShiftRegister()) + steps = stateful_steps(6) + kept, reset = run_host( + tmp_path, + program, + steps, + "--instances", + 2, + "--reset-groups", + "--reset-before", + "3:1", + )[0] + continuous, restarted = ProgramEmulator(exported), ProgramEmulator(exported) + for step, inputs in enumerate(steps): + if step == 3: + restarted.reset_states() + assert_matches(kept[step], as_list(continuous(*inputs))) + assert_matches(reset[step], as_list(restarted(*inputs))) + + def test_state_in_two_partitions_stays_in_the_program(tmp_path, monkeypatch): split_streams(monkeypatch) exported, program, report = lower_stateful( From dc0504fde52246de6e23933c48a4aeb7ba44e2d4 Mon Sep 17 00:00:00 2001 From: Anant Narayanan Date: Mon, 16 Nov 2026 21:41:17 -0800 Subject: [PATCH 25/25] Document ESP-NN program compatibility and buffer alignment The README did not say which runtimes run a program, and its examples bound buffers without the 16-byte alignment that lets the delegate write outputs in place. It gains a "Program compatibility" section: programs need a runtime from the same ExecuTorch version or later, since older runtimes reject XTS3 version 7 and 8 graphs at init(), while older programs still run. The binding examples declare their buffers alignas(16). The DelegateStats and OpState comments now say what they count and share. --- backends/espressif/README.md | 21 ++++++++++++++----- .../include/espnn/espnn_backend.h | 5 +++-- .../espnn_runtime/src/int8_interpreter.cpp | 4 ++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index d37e1996882..e260c3afadb 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -149,9 +149,9 @@ pte = edge.to_executorch(config).buffer The application then binds its buffers once, with `Method::set_input()` and `Method::set_output_data_ptr()`, and each step is a call to -`Method::execute()`. An output may share memory with an input (for example, -to update a state tensor in place), since the inputs are copied before any -operator runs. +`Method::execute()`. Declare them `alignas(16)`. An output may share memory +with an input, for example to update a state tensor in place, since the +inputs are copied in before any operator runs. ### Streaming state @@ -249,11 +249,14 @@ for each frame: #include // After Program::load() and load_method() with a method allocator sized by -// method_meta(), and no planned buffers (the program has none): +// method_meta(), and no planned buffers (the program has none). The buffers +// are 16-byte aligned so that the delegate writes the score in place. +alignas(16) int8_t features[40]; +alignas(16) int8_t score; method.set_input(EValue(Tensor(&features_impl)), 0); // int8[1, 40, 1, 1] method.set_output_data_ptr(&score, sizeof(score), 0); // int8[1, 1] for (;;) { - compute_features(features); // writes the bound buffer in place + compute_features(features); // writes the bound input buffer method.execute(); if (score > threshold) { on_detection(); @@ -405,6 +408,14 @@ The code follows the ExecuTorch coding style and passes the repository's lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" ``` +## Program compatibility + +A program runs on a delegate runtime from the same ExecuTorch version as the +exporter or a later one. Programs exported now carry XTS3 version 7 graphs, +or version 8 for delegates that own state, which older runtimes reject at +`init()` with `DelegateInvalidCompatibility`. The runtime still runs +programs from older exporters (XTS3 versions 5 and 6). + ## Limitations - Only int8 is delegated. Float32 operators run on ExecuTorch's portable diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index 822b8a949b3..8e85a6b95c8 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -43,10 +43,11 @@ struct DelegateStats { uint32_t calls; /// Total time spent in execute(), including input and output conversion. int64_t total_us; - /// Sum of the activation arenas held by the initialized delegates. + /// Sum of the activation arenas allocated by the delegates initialized in + /// that time. Destroying a delegate does not subtract its memory. size_t arena_bytes; /// Sums of the separate ESP-NN scratch buffers, the requantization tables - /// and the per-operation and per-tensor tables of the initialized delegates. + /// and the per-operation and per-tensor tables they allocated. size_t scratch_bytes; size_t parameter_bytes; size_t table_bytes; diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 50b4bec5296..515635e7eeb 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -222,8 +222,8 @@ bool pointwise_beyond_uint16( } // namespace // Fields that no operation uses together share storage: the input list -// (CONCAT, COPY) with the pooling parameters, and the convolution, -// depthwise-convolution and add parameters with each other. +// (CONCAT, COPY) with the pooling and convolution windows, and the +// convolution, depthwise-convolution and add parameters with each other. struct Int8Interpreter::OpState { OpCode opcode; uint32_t input0;