diff --git a/.ci/scripts/test_espressif_host.sh b/.ci/scripts/test_espressif_host.sh new file mode 100755 index 00000000000..ea19fd6093a --- /dev/null +++ b/.ci/scripts/test_espressif_host.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Builds the ESP-NN delegate for the host and checks fully delegated programs +# against the bit-exact Python reference: plain, and with ASan/UBSan over +# esp-nn's portable kernels, over its optimized C kernels, and with the +# delegate's non-default options. The sanitized builds make the kernels use +# scratch, in a separate buffer or in the arena. Needs no ESP-IDF toolchain or +# emulator. + +set -euo pipefail + +et_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +build_dir="${ESPRESSIF_HOST_BUILD_DIR:-${et_root}/cmake-out-espnn-host}" +cd "${et_root}" + +run_variant() { + local variant=$1 + shift + cmake -S backends/espressif/test/host -B "${build_dir}/${variant}" -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DPYTHON_EXECUTABLE="$(command -v python)" "$@" + cmake --build "${build_dir}/${variant}" + ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \ + ESPNN_HOST_INTERPRETER="${build_dir}/${variant}/espnn_host_interpreter" \ + python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py \ + backends/espressif/test/test_host_interpreter.py +} + +run_variant plain +scratch=(-DESPNN_HOST_SANITIZE=ON -DESPNN_HOST_SCRATCH=ON) +run_variant sanitized "${scratch[@]}" +run_variant sanitized-optimized "${scratch[@]}" -DESPNN_HOST_OPTIMIZED=ON +run_variant sanitized-options "${scratch[@]}" -DESPNN_HOST_OPTIMIZED=ON \ + -DESPNN_HOST_PSRAM=ON -DESPNN_DELEGATE_CONSTANTS_TO_PSRAM=ON \ + -DESPNN_DELEGATE_PROFILE=ON -DESPNN_DELEGATE_SCRATCH_IN_ARENA=ON diff --git a/.github/workflows/test-espressif.yml b/.github/workflows/test-espressif.yml index 055c5b3a813..e8c5f891ca7 100644 --- a/.github/workflows/test-espressif.yml +++ b/.github/workflows/test-espressif.yml @@ -32,6 +32,7 @@ on: - tools/cmake/preset/esp_baremetal.cmake - CMakePresets.json - .ci/scripts/test_espressif.sh + - .ci/scripts/test_espressif_host.sh - .ci/scripts/test_espressif_idf_component.sh - .ci/scripts/test_espressif_idf_consumer.sh - .ci/scripts/esp_idf_mirror.py @@ -75,6 +76,27 @@ jobs: install_executorch --use-pt-pinned-commit --minimal ESPRESSIF_NN_KERNELS="${{ matrix.kernels }}" bash .ci/scripts/test_espressif.sh + test-host: + name: ESP-NN delegate on the host + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read + with: + runner: mt-l-x86iavx512-8-64 + docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 45 + script: | + set -euo pipefail + CONDA_ENV=$(conda env list --json | jq -r '.envs | .[-1]') + conda activate "${CONDA_ENV}" + source .ci/scripts/utils.sh + install_executorch --use-pt-pinned-commit --minimal + bash .ci/scripts/test_espressif_host.sh + test-idf-component: name: ESP-IDF component needs: docker-image diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 48fcf2cb062..e260c3afadb 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -29,6 +29,7 @@ backends/espressif/ test/ pytest suite (lowering, operators, CPU fallback, parser) examples/espressif/ quantize_and_lower.py minimal quantize -> lower -> .pte example + export_streaming_model.py streaming model with delegate-owned state export_espnn_test_programs.py the test models as BundleIO programs for the runner camera_stream/ live camera classification on the ExecuTorch runtime ``` @@ -100,6 +101,170 @@ wrote mobilenet_v2_espnn.pte Calibrate on a few hundred real samples; random data gives useless scales. +### Inputs and outputs + +The delegate keeps rank-4 tensors in NHWC order. Inputs are always copied +into slots in the delegate's activation arena, quantized or transposed if +needed, because ESP-NN kernels may load whole vectors past the end of a +tensor they read. For the same reason, an output that an operator reads keeps +its arena slot and is copied out at the end. The delegate writes any other +int8 output directly when its buffer is 16-byte aligned and already in this +layout: the tensor is not rank 4, has one channel or one pixel, or has a +channels-last dim order. Each output is handled independently, whether its +buffer is memory-planned by ExecuTorch or supplied by the caller. + +Float outputs and NCHW outputs with several channels and pixels need dtype +or layout conversion. The exporter records which outputs need conversion, +so their int8 buffers can share the arena with intermediate tensors. Older +exports without this metadata use the arena for all outputs. An output +outside the arena whose buffer turns out misaligned is written to a staging +buffer and copied out. The delegate allocates that buffer at `init()`, sized +for every such output, in PSRAM when there is any. + +For the smallest and fastest program, give the delegate int8, channels-last +inputs and outputs and leave them out of ExecuTorch's memory plan, so that +the application's own buffers are used without conversion: + +```python +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass +from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output + +# Example inputs (and calibration data) in channels-last memory format. An +# output that PyTorch would make contiguous, such as a slice of a cat, needs +# .clone(memory_format=torch.channels_last) at the end of forward(). +example_inputs = (torch.randn(1, 3, 96, 96).to(memory_format=torch.channels_last),) +# ... quantize and calibrate as above ... +program = torch.export.export(quantized, example_inputs) +quantize_input(program, 0) +quantize_output(program, 0) +edge, report = to_espnn_edge(program) +config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) +) +pte = edge.to_executorch(config).buffer +``` + +The application then binds its buffers once, with `Method::set_input()` and +`Method::set_output_data_ptr()`, and each step is a call to +`Method::execute()`. Declare them `alignas(16)`. An output may share memory +with an input, for example to update a state tensor in place, since the +inputs are copied in before any operator runs. + +### Streaming state + +A streaming model keeps a few frames of history between calls in a mutable +buffer. By default the buffer stays in the program, which passes it to the +delegate and back on every call. With `delegate_mutable_buffers=True`, the +delegate that reads and updates a buffer owns it instead: the program loses +that input and output, and the delegate keeps the buffer as int8 in its arena, +written in place when no operator reads the old value after the new one is +produced. + +```python +class Streaming(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("frames", torch.zeros(1, 40, 2, 1)) + self.conv = torch.nn.Conv2d(40, 32, (3, 1)) + + def forward(self, x): # x: (1, 40, 1, 1), one new frame + window = torch.cat([self.frames, x], dim=2) + self.frames.copy_(window[:, :, 1:]) + return self.conv(window) + +# ... prepare_pt2e, calibrate, convert_pt2e as above, then reset the buffer +# that calibration updated before exporting. +edge, report = to_espnn_edge( + torch.export.export(quantized, (torch.zeros(1, 40, 1, 1),)), + delegate_mutable_buffers=True, +) +print(report["delegated_buffers"]) # ['frames'] +``` + +The delegate takes a buffer only when one partition holds every access to it +and the buffer is read and written with the same int8 quantization, as a +`cat` and slice shift register is with `EspNNQuantizer`. It takes a method's +buffers only when they all end up in one partition that keeps another input +and output, and at most 16 of them. Other buffers stay in the program, with a +warning. + +On the device, each loaded method has its own state. It starts from the +buffer's initial value when the method is loaded and persists across +`execute()` calls. `reset_delegate_states()` (from `espnn/espnn_backend.h`), +or the `reset_states` backend option set to `true`, restores the initial +state of every EspNNBackend delegate before its next `execute()`. To reset +some methods alone, load them with a `reset_group` from 0 to 7 and pass that +group to `reset_states`, or load the method again. Buffers that stay in the +program are not reset this way. After an `execute()` that fails, the state is +undefined until it is reset. The reset may be requested from any task. + +```cpp +executorch::backends::espressif::reset_delegate_states(); +// or, through the generic backend-option API: +executorch::runtime::BackendOptions<1> options; +options.set_option("reset_states", true); +executorch::runtime::set_option("EspNNBackend", options.view()); + +// A method in reset group 1, reset alone: +executorch::runtime::BackendOptions<1> group; +group.set_option("reset_group", 1); +executorch::runtime::LoadBackendOptionsMap load_options; +load_options.set_options("EspNNBackend", group.view()); +auto method = program->load_method( + "forward", &memory_manager, nullptr, nullptr, &load_options); +options.set_option("reset_states", 1); +executorch::runtime::set_option("EspNNBackend", options.view()); +``` + +### Streaming keyword spotting + +[`examples/espressif/export_streaming_model.py`](../../examples/espressif/export_streaming_model.py) +puts the pieces above together for a streaming keyword-spotting model: one +frame of features in, one score out, the frame history in delegate-owned +buffers, int8 input and output in the application's memory. The resulting +program has one input and one output, no CPU operators, and needs no +planned memory. + +Quantization choices that matter for such models: + +- `EspNNQuantizer` observes activations with histograms. For small streaming + models, min/max ranges (`minmax_quantizer()` in the example) often track a + float model, or a TensorFlow Lite reference, more closely. +- To reproduce an existing int8 model exactly, give each activation the + reference model's scale and zero point: after `prepare_pt2e`, replace its + observers with `FixedQParamsObserver(scale, zero_point, ...)`. +- Weight scales are `max|w| / 127` per output channel; check that a layer with + very small weights is not clamped by the weight observer's `eps`. +- Calibrate on consecutive frames, so the histories see realistic values, and + zero the buffers before exporting, since calibration updated them. + +On the device, the application binds its buffers once and calls `execute()` +for each frame: + +```cpp +#include +#include + +// After Program::load() and load_method() with a method allocator sized by +// method_meta(), and no planned buffers (the program has none). The buffers +// are 16-byte aligned so that the delegate writes the score in place. +alignas(16) int8_t features[40]; +alignas(16) int8_t score; +method.set_input(EValue(Tensor(&features_impl)), 0); // int8[1, 40, 1, 1] +method.set_output_data_ptr(&score, sizeof(score), 0); // int8[1, 1] +for (;;) { + compute_features(features); // writes the bound input buffer + method.execute(); + if (score > threshold) { + on_detection(); + executorch::backends::espressif::reset_delegate_states(); + } +} +``` + ## Running on the ESP32-S3 You need ESP-IDF and an activated ESP-IDF shell. @@ -174,7 +339,8 @@ Other options, in menuconfig under *ExecuTorch ESP-NN backend*: |---|---|---| | `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | | `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | -| `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | +| `ESPNN_DELEGATE_SCRATCH_IN_ARENA` | n | Plans each operator's ESP-NN scratch in the activation arena instead of a separate buffer sized for the largest operator. Saves internal RAM when the arena fits there; for a large model it moves the scratch to PSRAM with the arena | +| `ESPNN_DELEGATE_PROFILE` | n | Times every operator and prints a per-operator time table after a delegate's first run. The timer reads cost about 1 µs per operator on the ESP32-S3, so leave it off in production | ### The original ESP32 @@ -206,6 +372,35 @@ ANSI C reference kernels against the same bundled outputs. Both jobs target ESP32-S3. Logs include the kernel configuration in their filenames; artifacts retain the bundled fixtures, SDK configuration, and resolved component versions. +### On the host + +The delegate also builds outside ESP-IDF, for Linux and macOS, over ESP-NN's +portable C kernels. `host/include` stands in for the few ESP-IDF APIs it uses, +and ESP-NN is fetched from GitHub (or taken from `-DESPNN_LOCAL_PATH=...`). +`test_host_runtime.py` runs programs through the ExecuTorch runtime, this +build and the portable kernels, and checks every output against the host +reference (bit-exactly for fully delegated programs), with no board or +emulator: + +```bash +cmake -S backends/espressif/test/host -B cmake-out-espnn-host -G Ninja +cmake --build cmake-out-espnn-host +ESPNN_HOST_RUNNER=cmake-out-espnn-host/espnn_host_runner \ + pytest backends/espressif/test/test_host_runtime.py +``` + +`-DESPNN_HOST_SANITIZE=ON` builds with AddressSanitizer and +UndefinedBehaviorSanitizer, `-DESPNN_HOST_OPTIMIZED=ON` uses ESP-NN's optimized +C kernels instead of the reference ones, `-DESPNN_HOST_PSRAM=ON` makes the +heap stand-in serve PSRAM allocations, and `-DESPNN_HOST_SCRATCH=ON` makes +the convolution and mean kernels request scratch and overwrite all of it, as +a check on where the delegate puts it. `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM`, +`ESPNN_DELEGATE_PROFILE` and `ESPNN_DELEGATE_SCRATCH_IN_ARENA` are CMake +options here, off by default. `.ci/scripts/test_espressif_host.sh` +runs a plain build and three sanitized ones that cover these options. The host +build checks the delegate's logic, not ESP-NN's S3 assembly kernels, which only +the emulator and the board run. + The code follows the ExecuTorch coding style and passes the repository's `lintrunner` checks: @@ -213,6 +408,14 @@ The code follows the ExecuTorch coding style and passes the repository's lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" ``` +## Program compatibility + +A program runs on a delegate runtime from the same ExecuTorch version as the +exporter or a later one. Programs exported now carry XTS3 version 7 graphs, +or version 8 for delegates that own state, which older runtimes reject at +`init()` with `DelegateInvalidCompatibility`. The runtime still runs +programs from older exporters (XTS3 versions 5 and 6). + ## Limitations - Only int8 is delegated. Float32 operators run on ExecuTorch's portable @@ -223,15 +426,17 @@ lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" norm fall back to the CPU. - Dual-core execution only helps large convolutions, and it is off by default. - The delegate handle comes from ExecuTorch's runtime allocator, but each - delegate allocates its activation arena, scratch buffers and optional PSRAM - weight copy with ESP-IDF's `heap_caps` allocator at `init()`, so it can - choose between internal RAM and PSRAM. Nothing is allocated during - `execute()`. + delegate allocates its activation arena, scratch and staging buffers and + optional PSRAM weight copy with ESP-IDF's `heap_caps` allocator at + `init()`, so it can choose between internal RAM and PSRAM. Nothing is + allocated during `execute()`, except that with `ESPNN_DELEGATE_DUAL_CORE` + ESP-NN's worker task grows its own scratch buffer the first time a large + convolution needs it. ## License Every source file in this backend and its examples is released under the BSD 3-Clause license (`SPDX-License-Identifier: BSD-3-Clause`), the same license as ExecuTorch. See the `LICENSE` file at the repository root. ESP-NN -is Apache-2.0 and is fetched from the Espressif component registry, not -vendored. +is Apache-2.0 and is fetched from the Espressif component registry (or, for +the host build, from GitHub), not vendored. diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py index 1b7f7f32376..552d05451a8 100644 --- a/backends/espressif/espnn_preprocess.py +++ b/backends/espressif/espnn_preprocess.py @@ -11,6 +11,8 @@ import logging from typing import List +import torch + from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult from executorch.exir.backend.compile_spec_schema import CompileSpec @@ -20,6 +22,41 @@ BACKEND_ID = "EspNNBackend" INT8_FORMAT_SPEC = CompileSpec("espnn_format", b"int8") +_IO_CONVERSION_SPEC_KEY = "espnn_io_conversion_slots" + + +def _io_conversion_slots(program, graph) -> int: + signature = program.graph_signature + nodes = {node.name: node for node in program.graph.nodes} + io_nodes = [ + node + for node in program.graph.nodes + if node.op == "placeholder" and node.name in signature.user_inputs + ] + [nodes[name] for name in signature.user_outputs] + io_ids = graph.inputs + graph.outputs + if len(io_nodes) != len(io_ids): + raise ValueError("ESP-NN boundary tensors do not match the serialized I/O") + slots = 0 + seen = set() + for slot, (node, tensor_id) in enumerate(zip(io_nodes, io_ids)): + if tensor_id in seen: + continue + seen.add(tensor_id) + tensor = node.meta["val"] + # Kernels may load whole vectors past the end of a tensor they read, so + # inputs are always copied into the arena. + if ( + slot < len(graph.inputs) + or tensor.dtype != torch.int8 + or ( + tensor.dim() == 4 + and tensor.shape[1] != 1 + and tensor.shape[2] * tensor.shape[3] != 1 + and not tensor.is_contiguous(memory_format=torch.channels_last) + ) + ): + slots |= 1 << slot + return slots class EspNNBackend(BackendDetails): @@ -48,6 +85,15 @@ def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResu ) graph = build_graph(edge_program) blob = serialize_graph(graph) + compile_specs[:] = [ + spec for spec in compile_specs if spec.key != _IO_CONVERSION_SPEC_KEY + ] + compile_specs.append( + CompileSpec( + _IO_CONVERSION_SPEC_KEY, + _io_conversion_slots(edge_program, graph).to_bytes(4, "little"), + ) + ) logger.info( "EspNN delegate: %d tensors, %d operations, %d constant bytes, %d byte blob", len(graph.tensors), diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt index d909069f942..3765d299d79 100644 --- a/backends/espressif/espnn_runtime/CMakeLists.txt +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -10,6 +10,17 @@ # is meant to sit next to examples/espressif/executor_runner, which imports the # cross-compiled ExecuTorch targets this component links against: build # examples/espressif/project with -DET_ESPNN_BACKEND=ON. +# +# Outside ESP-IDF, this file builds the same delegate for the host (Linux, +# macOS) as the espnn_backend library, over esp-nn's portable C kernels, with +# host/include standing in for the few ESP-IDF APIs it uses. Add it after +# ExecuTorch with add_subdirectory(); backends/espressif/test/host uses it to +# check the delegate against the bit-exact Python reference without a board. + +if(NOT ESP_PLATFORM) + include(${CMAKE_CURRENT_LIST_DIR}/host/espnn_host.cmake) + return() +endif() idf_component_register( SRCS diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index 8fa51b176b6..e09ef8fe8d9 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -17,8 +17,23 @@ menu "ExecuTorch ESP-NN backend" bool "Split ESP-NN kernels across both cores inside delegates" default n + config ESPNN_DELEGATE_SCRATCH_IN_ARENA + bool "Plan ESP-NN scratch inside each delegate's activation arena" + default n + help + Gives each operator's ESP-NN scratch a slot in the activation + arena that is live only while the operator runs, instead of a + separate buffer sized for the largest operator. This saves + internal RAM for small models whose arena fits there; for a model + whose arena goes to PSRAM, the scratch moves to PSRAM too, which + is slower. + config ESPNN_DELEGATE_PROFILE bool "Print a per-operator profile after each delegate's first run" default n + help + Times every operator, which adds a timer read per operator to + each execute. Without it, the delegate reads the timer only + every few milliseconds, for its cooperative yield. endmenu diff --git a/backends/espressif/espnn_runtime/host/espnn_host.cmake b/backends/espressif/espnn_runtime/host/espnn_host.cmake new file mode 100644 index 00000000000..4c93fa8dbb1 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/espnn_host.cmake @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Host build of the ESP-NN delegate; included from ../CMakeLists.txt outside +# ESP-IDF. Needs the executorch_core target. esp-nn comes from ESPNN_LOCAL_PATH +# or is fetched at ESPNN_GIT_TAG. + +include(FetchContent) + +# The version that ../../idf_component.yml pins for the device build. +set(ESPNN_GIT_TAG + "d9be5422a79f4115e4275452dcaba3fe363a697d" # v1.4.1 + CACHE STRING "esp-nn commit fetched when ESPNN_LOCAL_PATH is not set" +) +set(ESPNN_LOCAL_PATH + "" + CACHE PATH "Existing esp-nn checkout to build instead of fetching one" +) +option(ESPNN_HOST_OPTIMIZED "Use esp-nn's generic optimized C kernels" OFF) +option(ESPNN_HOST_PSRAM "Serve PSRAM allocations from the host heap" OFF) +option(ESPNN_HOST_SCRATCH "Make the kernels request and fill ESP-NN scratch" + OFF +) +option(ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + "Copy weights into PSRAM; needs ESPNN_HOST_PSRAM" OFF +) +option(ESPNN_DELEGATE_PROFILE "Time every delegate operator" OFF) +option(ESPNN_DELEGATE_SCRATCH_IN_ARENA + "Plan ESP-NN scratch in the activation arena" OFF +) + +if(ESPNN_LOCAL_PATH) + set(espnn_source_dir "${ESPNN_LOCAL_PATH}") +else() + # esp-nn's own CMakeLists.txt is an ESP-IDF component; SOURCE_SUBDIR keeps + # FetchContent from adding it. + FetchContent_Declare( + esp_nn + GIT_REPOSITORY https://github.com/espressif/esp-nn.git + GIT_TAG ${ESPNN_GIT_TAG} + SOURCE_SUBDIR none + ) + FetchContent_MakeAvailable(esp_nn) + set(espnn_source_dir "${esp_nn_SOURCE_DIR}") +endif() + +file(GLOB espnn_kernel_sources "${espnn_source_dir}/src/*/*_ansi.c") +if(ESPNN_HOST_OPTIMIZED) + file(GLOB espnn_optimized_sources "${espnn_source_dir}/src/*/*_opt.c") + list(APPEND espnn_kernel_sources ${espnn_optimized_sources}) +endif() +add_library(esp_nn_host STATIC ${espnn_kernel_sources}) +target_include_directories( + esp_nn_host + PUBLIC "${espnn_source_dir}/include" + PRIVATE "${espnn_source_dir}/src/common" +) +if(ESPNN_HOST_OPTIMIZED) + target_compile_definitions(esp_nn_host PUBLIC CONFIG_NN_OPTIMIZED=1) +endif() +set_target_properties(esp_nn_host PROPERTIES POSITION_INDEPENDENT_CODE ON) + +get_filename_component( + espnn_executorch_root "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE +) +add_library( + espnn_backend STATIC + ${CMAKE_CURRENT_LIST_DIR}/../src/xts3_graph.cpp + ${CMAKE_CURRENT_LIST_DIR}/../src/int8_interpreter.cpp + ${CMAKE_CURRENT_LIST_DIR}/../src/espnn_backend.cpp +) +target_include_directories( + espnn_backend + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../include + PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include "${espnn_executorch_root}/.." + "${espnn_executorch_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + espnn_backend PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +if(ESPNN_HOST_PSRAM) + target_compile_definitions(espnn_backend PUBLIC ESPNN_HOST_PSRAM=1) +endif() +if(ESPNN_HOST_SCRATCH) + target_compile_definitions(espnn_backend PRIVATE ESPNN_HOST_SCRATCH=1) +endif() +foreach(option ESPNN_DELEGATE_CONSTANTS_TO_PSRAM ESPNN_DELEGATE_PROFILE + ESPNN_DELEGATE_SCRATCH_IN_ARENA +) + if(${option}) + target_compile_definitions(espnn_backend PRIVATE CONFIG_${option}=1) + endif() +endforeach() +target_link_libraries(espnn_backend PUBLIC executorch_core esp_nn_host) +set_target_properties(espnn_backend PROPERTIES POSITION_INDEPENDENT_CODE ON) +executorch_target_link_options_shared_lib(espnn_backend) diff --git a/backends/espressif/espnn_runtime/host/include/esp_cpu.h b/backends/espressif/espnn_runtime/host/include/esp_cpu.h new file mode 100644 index 00000000000..9f3e889760b --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_cpu.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for esp_cpu_get_cycle_count(): a 240 MHz count of a monotonic +// clock, wrapping like the 32-bit cycle counter. + +#pragma once + +#include +#include + +inline uint32_t esp_cpu_get_cycle_count() { + const uint64_t ns = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + return static_cast(ns * 240 / 1000); +} diff --git a/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h new file mode 100644 index 00000000000..c58ff752fee --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_heap_caps.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for the ESP-IDF heap API the ESP-NN delegate uses: one heap, +// all of it "internal". With ESPNN_HOST_PSRAM, PSRAM allocations come from the +// same heap; otherwise there is no PSRAM. ESPNN_HOST_INTERNAL_BYTES in the +// environment limits the internal aligned allocations to that many bytes. + +#pragma once + +#include +#include +#include +#include + +#define MALLOC_CAP_8BIT (1U << 2) +#define MALLOC_CAP_SPIRAM (1U << 10) +#define MALLOC_CAP_INTERNAL (1U << 11) + +#ifdef ESPNN_HOST_PSRAM +constexpr bool kHostPsram = true; +#else +constexpr bool kHostPsram = false; +#endif + +namespace espnn_host { + +inline std::unordered_map g_internal_blocks; +inline size_t g_internal_used = 0; + +inline size_t internal_free() { + static const char* limit = std::getenv("ESPNN_HOST_INTERNAL_BYTES"); + const size_t capacity = + limit != nullptr ? std::strtoull(limit, nullptr, 10) : SIZE_MAX / 2; + return capacity > g_internal_used ? capacity - g_internal_used : 0; +} + +} // namespace espnn_host + +inline void* +heap_caps_aligned_alloc(size_t alignment, size_t size, uint32_t caps) { + const bool psram = (caps & MALLOC_CAP_SPIRAM) != 0; + if ((psram && !kHostPsram) || size == 0 || + (!psram && size > espnn_host::internal_free())) + return nullptr; + // Exactly size bytes, so that AddressSanitizer sees any overrun. + void* memory = nullptr; + if (posix_memalign(&memory, alignment, size) != 0) + return nullptr; + if (!psram) { + espnn_host::g_internal_blocks[memory] = size; + espnn_host::g_internal_used += size; + } + return memory; +} + +inline void* heap_caps_calloc(size_t count, size_t size, uint32_t) { + return std::calloc(count, size); +} + +inline void heap_caps_free(void* memory) { + const auto block = espnn_host::g_internal_blocks.find(memory); + if (block != espnn_host::g_internal_blocks.end()) { + espnn_host::g_internal_used -= block->second; + espnn_host::g_internal_blocks.erase(block); + } + std::free(memory); +} + +inline size_t heap_caps_get_largest_free_block(uint32_t caps) { + if ((caps & MALLOC_CAP_SPIRAM) != 0) + return kHostPsram ? SIZE_MAX / 2 : 0; + return espnn_host::internal_free(); +} + +inline size_t heap_caps_get_total_size(uint32_t caps) { + return heap_caps_get_largest_free_block(caps); +} diff --git a/backends/espressif/espnn_runtime/host/include/esp_nn.h b/backends/espressif/espnn_runtime/host/include/esp_nn.h new file mode 100644 index 00000000000..a44e19a0de8 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_nn.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// esp-nn's header. esp-nn's portable kernels need no scratch, unlike its +// ESP32-S3 kernels. With ESPNN_HOST_SCRATCH, the convolution and mean kernels +// request some and fill all of it before they run, so that a scratch buffer +// that is too small fails under AddressSanitizer, and one that overlaps live +// activations fails the bit-exact output checks. + +#pragma once + +#include_next + +#ifdef ESPNN_HOST_SCRATCH + +#include +#include +#include + +namespace espnn_host { + +inline void* g_conv_scratch = nullptr; +inline void* g_depthwise_scratch = nullptr; +inline void* g_mean_scratch = nullptr; + +// Each term is capped, so that no graph shape can overflow the sizes. +inline int capped(int64_t value) { + return static_cast(std::min(value, 4096)); +} + +inline int scratch_size(const data_dims_t* input, const data_dims_t* output) { + return 4 * capped(input->channels) + + capped(int64_t{output->width} * output->height) + 16; +} + +inline int conv_scratch_size( + const data_dims_t* input_dims, + const data_dims_t*, + const data_dims_t* output_dims, + const conv_params_t*) { + return scratch_size(input_dims, output_dims); +} + +inline int depthwise_conv_scratch_size( + const data_dims_t* input_dims, + const data_dims_t*, + const data_dims_t* output_dims, + const dw_conv_params_t*) { + return scratch_size(input_dims, output_dims); +} + +inline int32_t mean_scratch_size(int32_t, int32_t, int32_t channels) { + return 4 * capped(channels) + 16; +} + +inline void set_conv_scratch(const void* buffer) { + g_conv_scratch = const_cast(buffer); +} + +inline void set_depthwise_conv_scratch(const void* buffer) { + g_depthwise_scratch = const_cast(buffer); +} + +inline void set_mean_scratch(void* buffer) { + g_mean_scratch = buffer; +} + +inline void conv_s8( + const data_dims_t* input_dims, + const int8_t* input_data, + const data_dims_t* filter_dims, + const int8_t* filter_data, + const int32_t* bias, + const data_dims_t* output_dims, + int8_t* out_data, + const conv_params_t* conv_params, + const quant_data_t* quant_data) { + std::memset(g_conv_scratch, 0x5a, scratch_size(input_dims, output_dims)); + esp_nn_conv_s8( + input_dims, + input_data, + filter_dims, + filter_data, + bias, + output_dims, + out_data, + conv_params, + quant_data); +} + +inline void depthwise_conv_s8( + const data_dims_t* input_dims, + const int8_t* input_data, + const data_dims_t* filter_dims, + const int8_t* filter_data, + const int32_t* bias, + const data_dims_t* output_dims, + int8_t* out_data, + const dw_conv_params_t* conv_params, + const quant_data_t* quant_data) { + std::memset(g_depthwise_scratch, 0x5a, scratch_size(input_dims, output_dims)); + esp_nn_depthwise_conv_s8( + input_dims, + input_data, + filter_dims, + filter_data, + bias, + output_dims, + out_data, + conv_params, + quant_data); +} + +inline void mean_nhwc_s8( + const int8_t* input, + int8_t* output, + int32_t height, + int32_t width, + int32_t channels, + int32_t input_zero_point, + int32_t output_zero_point, + int32_t multiplier, + int32_t shift) { + std::memset(g_mean_scratch, 0x5a, mean_scratch_size(height, width, channels)); + esp_nn_mean_nhwc_s8( + input, + output, + height, + width, + channels, + input_zero_point, + output_zero_point, + multiplier, + shift); +} + +} // namespace espnn_host + +#undef esp_nn_get_conv_scratch_size +#undef esp_nn_set_conv_scratch_buf +#undef esp_nn_conv_s8 +#undef esp_nn_get_depthwise_conv_scratch_size +#undef esp_nn_set_depthwise_conv_scratch_buf +#undef esp_nn_depthwise_conv_s8 +#undef esp_nn_get_mean_scratch_size +#undef esp_nn_set_mean_scratch_buf +#undef esp_nn_mean_nhwc_s8 +#define esp_nn_get_conv_scratch_size espnn_host::conv_scratch_size +#define esp_nn_set_conv_scratch_buf espnn_host::set_conv_scratch +#define esp_nn_conv_s8 espnn_host::conv_s8 +#define esp_nn_get_depthwise_conv_scratch_size \ + espnn_host::depthwise_conv_scratch_size +#define esp_nn_set_depthwise_conv_scratch_buf \ + espnn_host::set_depthwise_conv_scratch +#define esp_nn_depthwise_conv_s8 espnn_host::depthwise_conv_s8 +#define esp_nn_get_mean_scratch_size espnn_host::mean_scratch_size +#define esp_nn_set_mean_scratch_buf espnn_host::set_mean_scratch +#define esp_nn_mean_nhwc_s8 espnn_host::mean_nhwc_s8 + +#endif diff --git a/backends/espressif/espnn_runtime/host/include/esp_timer.h b/backends/espressif/espnn_runtime/host/include/esp_timer.h new file mode 100644 index 00000000000..8166fbeccc4 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/esp_timer.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for esp_timer_get_time(): microseconds of a monotonic clock. + +#pragma once + +#include +#include + +inline int64_t esp_timer_get_time() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} diff --git a/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h b/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h new file mode 100644 index 00000000000..5c7e81325b7 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/FreeRTOS.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-ins for the FreeRTOS calls the ESP-NN delegate makes: a mutex +// backed by std::mutex, and a no-op task yield. + +#pragma once + +#include +#include + +typedef uint32_t TickType_t; +typedef int BaseType_t; +#define portMAX_DELAY (static_cast(0xffffffffU)) +#define pdTRUE 1 + +struct StaticSemaphore_t { + std::mutex mutex; +}; +typedef StaticSemaphore_t* SemaphoreHandle_t; diff --git a/backends/espressif/espnn_runtime/host/include/freertos/semphr.h b/backends/espressif/espnn_runtime/host/include/freertos/semphr.h new file mode 100644 index 00000000000..8010080f4f8 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/semphr.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +inline SemaphoreHandle_t xSemaphoreCreateMutexStatic( + StaticSemaphore_t* storage) { + return storage; +} + +inline BaseType_t xSemaphoreTake(SemaphoreHandle_t handle, TickType_t) { + handle->mutex.lock(); + return pdTRUE; +} + +inline BaseType_t xSemaphoreGive(SemaphoreHandle_t handle) { + handle->mutex.unlock(); + return pdTRUE; +} diff --git a/backends/espressif/espnn_runtime/host/include/freertos/task.h b/backends/espressif/espnn_runtime/host/include/freertos/task.h new file mode 100644 index 00000000000..7689b82b914 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/freertos/task.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +inline void vTaskDelay(TickType_t) {} diff --git a/backends/espressif/espnn_runtime/host/include/sdkconfig.h b/backends/espressif/espnn_runtime/host/include/sdkconfig.h new file mode 100644 index 00000000000..a32ebafedc4 --- /dev/null +++ b/backends/espressif/espnn_runtime/host/include/sdkconfig.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// Host stand-in for ESP-IDF's generated sdkconfig.h. The host CMake build +// passes the delegate's CONFIG_ESPNN_DELEGATE_* options as definitions. + +#pragma once diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h index d5293be3bfa..8e85a6b95c8 100644 --- a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -20,6 +20,18 @@ namespace espressif { /// Backend ID that `EspNNBackend.preprocess()` writes into lowered programs. constexpr const char* kEspNNBackendId = "EspNNBackend"; +/// Backend option that resets delegate-owned state, through +/// `executorch::runtime::set_option(kEspNNBackendId, options)`: true calls +/// reset_delegate_states(), and a reset group (int) calls +/// reset_delegate_states(group). +constexpr char kResetStatesOption[] = "reset_states"; + +/// Load-time backend option (int, 0 by default) that puts the delegates of a +/// method in a reset group from 0 to kResetGroups - 1. Pass it to +/// `Program::load_method()` in a LoadBackendOptionsMap. +constexpr char kResetGroupOption[] = "reset_group"; +constexpr int kResetGroups = 8; + /** * Counters accumulated by every EspNNBackend delegate since startup or the * last reset_delegate_stats() call. @@ -31,8 +43,23 @@ struct DelegateStats { uint32_t calls; /// Total time spent in execute(), including input and output conversion. int64_t total_us; - /// Sum of the activation arenas held by the initialized delegates. + /// Sum of the activation arenas allocated by the delegates initialized in + /// that time. Destroying a delegate does not subtract its memory. size_t arena_bytes; + /// Sums of the separate ESP-NN scratch buffers, the requantization tables + /// and the per-operation and per-tensor tables they allocated. + size_t scratch_bytes; + size_t parameter_bytes; + size_t table_bytes; + /// Sum of the buffers that delegates allocated at init() for outputs + /// whose memory turns out not to be 16-byte aligned, and the number of + /// outputs that execute() wrote through them. + size_t staging_bytes; + uint32_t staged_outputs; + /// Delegate-owned state tensors (mutable buffers), and how many of them + /// are updated in place rather than copied after the last operator. + uint32_t states; + uint32_t states_updated_in_place; }; /** @@ -56,6 +83,25 @@ DelegateStats delegate_stats(); /// Zeroes all delegate counters. void reset_delegate_stats(); +/** + * Restores the state that delegates own (the mutable buffers lowered with + * `delegate_mutable_buffers=True`) to its initial value, in every loaded + * EspNNBackend delegate, before that delegate's next execute(). + * + * Each loaded method instance has its own state, kept between executions + * until this is called or the method is destroyed; loading the method again + * also starts from the initial state. Buffers that stay in the program are + * not reset. Safe to call from any task. + */ +void reset_delegate_states(); + +/** + * Same, for the delegates loaded with kResetGroupOption set to @p group. + * + * @retval false @p group is not from 0 to kResetGroups - 1. + */ +bool reset_delegate_states(int group); + } // namespace espressif } // namespace backends } // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 9c18ac8e63b..6415c15f3a4 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -41,6 +41,23 @@ struct Int8RuntimeOptions { bool dual_core = false; /// Internal RAM left free when deciding whether the arena fits there. size_t internal_reserve_bytes = 48 * 1024; + /// Time every operation for op_time_us() and print_profile(). Otherwise + /// invoke() reads the timer only every few milliseconds. + bool profile_operations = false; + /// Plan each operation's ESP-NN scratch in the activation arena, live only + /// while the operation runs, instead of one buffer sized for the largest + /// operation. Saves memory, but the scratch then lives wherever the arena + /// does (PSRAM for a large model). + bool scratch_in_arena = false; + /// Bind graph outputs to caller-provided device-layout (NHWC) buffers, + /// outside this interpreter's arena, except the slots selected by + /// conversion_slots. Graph inputs and outputs that an operation reads + /// always have arena slots. + bool external_io = false; + /// With external_io, include these output slots in the arena for + /// conversion. Bits select slots in input-then-output order, determined at + /// export time. + uint32_t conversion_slots = 0; }; /** @@ -115,6 +132,18 @@ class Int8Interpreter { bool input_info(size_t index, Int8TensorInfo& info) const; /// Describes output @p index. @retval false @p index is out of range. bool output_info(size_t index, Int8TensorInfo& info) const; + /** + * With Int8RuntimeOptions::external_io, makes @p data, a device-layout + * buffer of output_info().bytes, the storage of output @p index until it is + * bound again. The buffer must be 16-byte aligned like the arena, since the + * kernels may rely on that. An output with an arena slot cannot be bound: + * output_data() returns the slot. + * + * @retval false external_io is off, @p index is out of range, @p data is + * null or misaligned, or the output has an arena slot (it is a graph + * input, an operation reads it, or conversion_slots selects it). + */ + bool bind_output(size_t index, int8_t* data); /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. int8_t* input_data(size_t index); /// Device-layout (NHWC) int8 buffer of output @p index, or nullptr. @@ -157,11 +186,17 @@ class Int8Interpreter { const GraphView& graph() const { return graph_; } + /// With external_io, the I/O slots given arena slots: conversion_slots plus + /// every output that an operation reads, in the same bit order. + uint32_t conversion_slots() const { + return options_.conversion_slots; + } /// Duration of the last invoke() in microseconds. int64_t last_invoke_us() const { return last_invoke_us_; } - /// Duration of operation @p op during the last invoke(), or 0. + /// Duration of operation @p op during the last invoke(), or 0 without + /// Int8RuntimeOptions::profile_operations. int64_t op_time_us(uint32_t op) const; /// Size of the activation arena. size_t arena_bytes() const { @@ -171,10 +206,24 @@ class Int8Interpreter { bool arena_in_internal_ram() const { return arena_internal_; } - /// Size of the shared ESP-NN scratch buffer. + /// Size of the separate ESP-NN scratch buffer (0 with scratch_in_arena). size_t scratch_bytes() const { return scratch_size_; } + /// Number of delegate-owned state tensors (mutable buffers). + size_t state_count() const { + return graph_.num_states; + } + /// Number of states whose new value is written in place over the old one + /// rather than copied after the last operation. + size_t states_updated_in_place() const; + /** + * Restores every state tensor to its initial value. load() does this too. + * Not thread-safe with respect to invoke() on the same interpreter. + */ + void reset_states(); + /// Size of the per-operation and per-tensor tables kept in RAM after load(). + size_t table_bytes() const; /// Size of the precomputed requantization tables. size_t parameter_bytes() const { return params_size_; @@ -193,9 +242,11 @@ class Int8Interpreter { bool plan_memory(); bool prepare_operations(); + bool allocate_operation_buffers(); void rebase_constants(const uint8_t* old_base, size_t size); bool tensor_info(uint32_t tensor_id, Int8TensorInfo& info) const; int8_t* activation(uint32_t tensor_id) const; + bool in_arena(const int8_t* data) const; const int8_t* operand(uint32_t tensor_id) const; void run_generic_conv(const OpState& s, const int8_t* input, int8_t* output) const; @@ -216,10 +267,11 @@ class Int8Interpreter { GraphView graph_{}; bool loaded_ = false; uint8_t* blob_copy_ = nullptr; - TensorRecord* tensors_ = nullptr; - OperationRecord* operations_ = nullptr; + const TensorRecord* tensors_ = nullptr; + /// The graph's tensor table and requantization arrays are used in place. + bool tables_in_place_ = false; OpState* op_states_ = nullptr; - size_t* tensor_offsets_ = nullptr; + int8_t** tensor_data_ = nullptr; uint8_t* arena_ = nullptr; size_t arena_size_ = 0; bool arena_internal_ = false; @@ -229,6 +281,8 @@ class Int8Interpreter { size_t phase_bytes_ = 0; int32_t* params_ = nullptr; size_t params_size_ = 0; + int64_t* op_us_ = nullptr; + uint16_t updates_in_place_ = 0; uint32_t inputs_[MAX_GRAPH_IO]{}; uint32_t outputs_[MAX_GRAPH_IO]{}; size_t input_count_ = 0; diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h index 4fac0173531..bfe194bfff3 100644 --- a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -19,19 +19,28 @@ * (backends/espressif/serialization/xts3.py). All multi-byte fields are * little-endian. A graph is a header, an optional input/output table * (version 6), fixed-size tensor and operation records, a parameter region - * and a constant region. parse_graph() bounds-checks every region and record - * before any accessor reads it. + * and a constant region. Version 7 pads the input/output table to a 4-byte + * boundary, so that the records and the parameter region are word-aligned and + * the runtime can use them where they are. Version 8 adds a state table after + * the input/output table: pairs of a state tensor, which persists across + * invokes, and the tensor holding its next value. parse_graph() bounds-checks + * every region and record before any accessor reads it. */ namespace executorch { namespace backends { namespace espressif { -/// Graph versions accepted by this runtime; 6 adds the input/output table. +/// Graph versions accepted by this runtime; 6 adds the input/output table, 7 +/// aligns the tables that follow it and 8 adds the state table. static constexpr uint32_t GRAPH_ESP_NN_INT8_VERSION = 5; static constexpr uint32_t GRAPH_ESP_NN_INT8_IO_VERSION = 6; +static constexpr uint32_t GRAPH_ESP_NN_INT8_ALIGNED_VERSION = 7; +static constexpr uint32_t GRAPH_ESP_NN_INT8_STATE_VERSION = 8; static constexpr size_t GRAPH_HEADER_SIZE = 22; static constexpr size_t GRAPH_IO_TABLE_HEADER_SIZE = 4; +static constexpr size_t GRAPH_STATE_TABLE_HEADER_SIZE = 4; +static constexpr size_t MAX_GRAPH_STATES = 16; static constexpr size_t MAX_GRAPH_IO = 16; static constexpr size_t TENSOR_RECORD_SIZE = 36; static constexpr size_t OPERATION_RECORD_SIZE = 44; @@ -43,6 +52,10 @@ static constexpr uint8_t TENSOR_FLAG_CONSTANT = 1U << 0; static constexpr uint8_t TENSOR_FLAG_QUANTIZED = 1U << 1; static constexpr uint8_t TENSOR_FLAG_GRAPH_INPUT = 1U << 2; static constexpr uint8_t TENSOR_FLAG_GRAPH_OUTPUT = 1U << 3; +/// Delegate-owned int8 state (version 8): its initial value is in the constant +/// region at constant_offset, and its state-table pair replaces it after every +/// invoke. +static constexpr uint8_t TENSOR_FLAG_STATE = 1U << 4; /// Serialized parameter block sizes and modes, mirrored in xts3.py. static constexpr size_t INT8_CONV_BASE_PARAMS_SIZE = 20; @@ -148,6 +161,9 @@ struct GraphView { uint16_t num_inputs; uint16_t num_outputs; const uint8_t* io_ids; + uint16_t num_states; + /// num_states (state, update) tensor ID pairs, little-endian u32. + const uint8_t* state_ids; }; /// Decoded convolution parameters; multipliers and shifts are per channel. @@ -207,6 +223,13 @@ uint32_t graph_output_count(const GraphView& graph); uint32_t graph_input_id(const GraphView& graph, uint32_t index); /// Tensor ID of output @p index, or UINT32_MAX if out of range. uint32_t graph_output_id(const GraphView& graph, uint32_t index); +/// State pair @p index: the persistent tensor and the tensor holding its next +/// value. @retval false @p index is out of range. +bool graph_state_pair( + const GraphView& graph, + uint32_t index, + uint32_t& state_id, + uint32_t& update_id); /// Decodes tensor record @p index. @retval false Out of range. bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor); /// Decodes operation record @p index. @retval false Out of range. diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 06b21783611..f1e0de90707 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -10,7 +10,9 @@ #include +#include #include +#include #include #include #include @@ -35,6 +37,8 @@ using executorch::runtime::Backend; using executorch::runtime::BackendExecutionContext; using executorch::runtime::BackendInitContext; using executorch::runtime::BackendInterface; +using executorch::runtime::BackendOption; +using executorch::runtime::BackendOptionContext; using executorch::runtime::CompileSpec; using executorch::runtime::DelegateHandle; using executorch::runtime::Error; @@ -49,6 +53,9 @@ namespace espressif { namespace { DelegateStats g_stats{}; +// Incremented by each reset of a group; a delegate whose state is older than +// its group restores its initial state before its next execute. +std::atomic g_state_generations[kResetGroups]{}; constexpr float kHalfEvenRoundingBias = 12582912.0f; @@ -73,29 +80,50 @@ int8_t quantize(float value, float inverse_scale, int32_t zero_point) { return static_cast(q); } -bool same_layout(const Tensor& tensor, const Int8TensorInfo& info) { +// How an ExecuTorch tensor's bytes relate to the delegate's device layout +// (NHWC for rank 4). kDevice: identical, because the tensor is not rank 4, its +// transpose is trivial (one channel, or one pixel), or it is channels-last. +// kNchw: a contiguous rank-4 tensor that needs transposing. +enum class Layout : uint8_t { kInvalid, kNchw, kDevice }; + +Layout tensor_layout(const Tensor& tensor, const Int8TensorInfo& info) { if (tensor.dim() != info.rank || static_cast(tensor.numel()) != info.bytes) - return false; + return Layout::kInvalid; for (uint8_t d = 0; d < info.rank; ++d) { if (tensor.size(d) != info.shape[d]) - return false; + return Layout::kInvalid; } - return executorch::runtime::is_contiguous_dim_order( - tensor.dim_order().data(), tensor.dim_order().size()); + const auto dim_order = tensor.dim_order(); + if (executorch::runtime::is_contiguous_dim_order( + dim_order.data(), dim_order.size())) { + const bool trivial = info.rank != 4 || info.channels == 1 || + static_cast(info.height) * info.width == 1; + return trivial ? Layout::kDevice : Layout::kNchw; + } + if (info.rank == 4 && + executorch::runtime::is_channels_last_dim_order( + dim_order.data(), dim_order.size())) + return Layout::kDevice; + return Layout::kInvalid; +} + +Error missing_data(const char* kind, size_t index) { + std::printf( + "EspNN delegate: %s %u has no data\n", + kind, + static_cast(index)); + return Error::InvalidArgument; } bool load_input( - Int8Interpreter& interpreter, - size_t index, - const Tensor& tensor) { - Int8TensorInfo info{}; - if (!interpreter.input_info(index, info) || !same_layout(tensor, info)) - return false; - int8_t* destination = interpreter.input_data(index); + const Int8TensorInfo& info, + Layout layout, + const Tensor& tensor, + int8_t* destination) { const size_t plane = static_cast(info.height) * info.width; const size_t channels = static_cast(info.channels); - const bool nchw = info.rank == 4; + const bool nchw = layout == Layout::kNchw; if (tensor.scalar_type() == ScalarType::Float) { const float* source = tensor.const_data_ptr(); const float inverse = @@ -129,16 +157,13 @@ bool load_input( } bool store_output( - const Int8Interpreter& interpreter, - size_t index, + const Int8TensorInfo& info, + Layout layout, + const int8_t* source, Tensor& tensor) { - Int8TensorInfo info{}; - if (!interpreter.output_info(index, info) || !same_layout(tensor, info)) - return false; - const int8_t* source = interpreter.output_data(index); const size_t plane = static_cast(info.height) * info.width; const size_t channels = static_cast(info.channels); - const bool nchw = info.rank == 4; + const bool nchw = layout == Layout::kNchw; if (tensor.scalar_type() == ScalarType::Float) { float* destination = tensor.mutable_data_ptr(); if (!nchw) { @@ -160,7 +185,8 @@ bool store_output( if (tensor.scalar_type() == ScalarType::Char) { int8_t* destination = tensor.mutable_data_ptr(); if (!nchw) { - std::memcpy(destination, source, info.bytes); + if (destination != source) + std::memcpy(destination, source, info.bytes); return true; } for (size_t c = 0; c < channels; ++c) { @@ -174,7 +200,82 @@ bool store_output( struct DelegateState { Int8Interpreter interpreter; + uint32_t conversion_slots = 0; bool profiled = false; + int reset_group = 0; + uint32_t state_generation = 0; + // Copies of outputs outside the arena whose memory is misaligned, in output + // order. + int8_t* staging = nullptr; + + ~DelegateState() { + if (staging != nullptr) + heap_caps_free(staging); + } + + // A graph tensor returned more than once is written where its first + // output is bound, and a graph input returned as an output stays where + // the input is bound; both are copied to their outputs afterwards. + bool returned_again(size_t o) const { + Int8TensorInfo info{}; + Int8TensorInfo other{}; + if (!interpreter.output_info(o, info)) + return false; + for (size_t earlier = 0; earlier < o; ++earlier) { + if (interpreter.output_info(earlier, other) && + other.tensor_id == info.tensor_id) + return true; + } + for (size_t i = 0; i < interpreter.input_count(); ++i) { + if (interpreter.input_info(i, other) && other.tensor_id == info.tensor_id) + return true; + } + return false; + } + + // Offset of output end's staging copy, the staging buffer's size for + // end = output_count(), or SIZE_MAX if that overflows. + size_t staging_offset(size_t end) const { + const size_t inputs = interpreter.input_count(); + size_t total = 0; + for (size_t o = 0; o < end; ++o) { + Int8TensorInfo info{}; + if ((conversion_slots & (1U << (inputs + o))) != 0 || returned_again(o) || + !interpreter.output_info(o, info)) + continue; + if (info.bytes > SIZE_MAX - 15 - total) + return SIZE_MAX; + total += (info.bytes + 15) & ~static_cast(15); + } + return total; + } + + // Only misaligned outputs use the staging buffer, so it goes to PSRAM when + // there is any and leaves internal RAM to the arena. + bool allocate_staging() { + const size_t total = staging_offset(interpreter.output_count()); + if (total == 0) + return true; + void* memory = nullptr; + if (total != SIZE_MAX) { + memory = heap_caps_aligned_alloc( + 16, total, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (memory == nullptr) { + memory = heap_caps_aligned_alloc( + 16, total, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + } + } + if (memory == nullptr) { + std::printf( + "EspNN delegate: cannot allocate %u staging bytes\n", + static_cast(total)); + return false; + } + staging = static_cast(memory); + std::memset(staging, 0, total); + g_stats.staging_bytes += total; + return true; + } }; class EspNNBackendInterface final : public BackendInterface { @@ -186,18 +287,44 @@ class EspNNBackendInterface final : public BackendInterface { Result init( BackendInitContext& context, FreeableBuffer* processed, - ArrayRef) const override { + ArrayRef compile_specs) const override { + const Result group = context.get_runtime_spec(kResetGroupOption); + const int reset_group = group.ok() ? group.get() : 0; + if ((!group.ok() && group.error() != Error::NotFound) || reset_group < 0 || + reset_group >= kResetGroups) { + std::printf( + "EspNN delegate: %s must be an int from 0 to %d\n", + kResetGroupOption, + kResetGroups - 1); + return Error::InvalidArgument; + } + Int8RuntimeOptions options{}; + for (const auto& spec : compile_specs) { + if (std::strcmp(spec.key, "espnn_io_conversion_slots") == 0) { + if (spec.value.buffer == nullptr || spec.value.nbytes != 4) + return Error::DelegateInvalidCompatibility; + options.external_io = true; + options.conversion_slots = + read_le_u32(static_cast(spec.value.buffer)); + } + } void* memory = context.get_runtime_allocator()->allocate( sizeof(DelegateState), alignof(DelegateState)); if (memory == nullptr) return Error::MemoryAllocationFailed; DelegateState* state = new (memory) DelegateState(); - Int8RuntimeOptions options{}; + state->reset_group = reset_group; #if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM options.copy_constants_to_psram = true; #endif #if CONFIG_ESPNN_DELEGATE_DUAL_CORE options.dual_core = true; +#endif +#if CONFIG_ESPNN_DELEGATE_PROFILE + options.profile_operations = true; +#endif +#if CONFIG_ESPNN_DELEGATE_SCRATCH_IN_ARENA + options.scratch_in_arena = true; #endif if (!state->interpreter.load( static_cast(processed->data()), @@ -206,10 +333,26 @@ class EspNNBackendInterface final : public BackendInterface { state->~DelegateState(); return Error::DelegateInvalidCompatibility; } + // Older exports use the arena for every I/O slot. + state->conversion_slots = options.external_io + ? state->interpreter.conversion_slots() + : UINT32_MAX; + if (!state->allocate_staging()) { + state->~DelegateState(); + return Error::MemoryAllocationFailed; + } if (state->interpreter.constants_copied()) processed->Free(); + state->state_generation = + g_state_generations[reset_group].load(std::memory_order_relaxed); ++g_stats.delegates; g_stats.arena_bytes += state->interpreter.arena_bytes(); + g_stats.scratch_bytes += state->interpreter.scratch_bytes(); + g_stats.parameter_bytes += state->interpreter.parameter_bytes(); + g_stats.table_bytes += state->interpreter.table_bytes(); + g_stats.states += state->interpreter.state_count(); + g_stats.states_updated_in_place += + state->interpreter.states_updated_in_place(); return state; } @@ -229,28 +372,86 @@ class EspNNBackendInterface final : public BackendInterface { return Error::InvalidArgument; } const int64_t started = esp_timer_get_time(); + if (interpreter.state_count() != 0) { + const uint32_t generation = g_state_generations[state->reset_group].load( + std::memory_order_relaxed); + if (generation != state->state_generation) { + interpreter.reset_states(); + state->state_generation = generation; + } + } + // Inputs are copied into their arena slots, so outputs may be written over + // the caller's input buffers. Output conversion buffers share the arena; + // compatible outputs are bound directly, independently of ExecuTorch's + // output memory planning. + Int8TensorInfo info{}; for (size_t i = 0; i < inputs; ++i) { - if (!args[i]->isTensor() || - !load_input(interpreter, i, args[i]->toTensor())) { + const Layout layout = + args[i]->isTensor() && interpreter.input_info(i, info) + ? tensor_layout(args[i]->toTensor(), info) + : Layout::kInvalid; + if (layout != Layout::kInvalid && + args[i]->toTensor().const_data_ptr() == nullptr) + return missing_data("input", i); + const bool ok = + layout != Layout::kInvalid && + load_input( + info, layout, args[i]->toTensor(), interpreter.input_data(i)); + if (!ok) { std::printf( "EspNN delegate: input %u has an unsupported dtype, shape or dim order\n", static_cast(i)); return Error::InvalidArgument; } } - if (!interpreter.invoke()) - return Error::Internal; + uint32_t staged_outputs = 0; for (size_t o = 0; o < outputs; ++o) { - if (!args[inputs + o]->isTensor()) - return Error::InvalidArgument; - Tensor tensor = args[inputs + o]->toTensor(); - if (!store_output(interpreter, o, tensor)) { + const Layout layout = + args[inputs + o]->isTensor() && interpreter.output_info(o, info) + ? tensor_layout(args[inputs + o]->toTensor(), info) + : Layout::kInvalid; + bool ok = layout != Layout::kInvalid; + if (ok && args[inputs + o]->toTensor().const_data_ptr() == nullptr) + return missing_data("output", o); + if (ok && (state->conversion_slots & (1U << (inputs + o))) != 0) { + staged_outputs |= 1U << o; + continue; + } + if (ok && state->returned_again(o)) { + staged_outputs |= 1U << o; + } else if (ok) { + // bind_output() refuses a misaligned buffer. + Tensor tensor = args[inputs + o]->toTensor(); + if (tensor.scalar_type() != ScalarType::Char || + layout != Layout::kDevice || + !interpreter.bind_output(o, tensor.mutable_data_ptr())) { + ok = interpreter.bind_output( + o, state->staging + state->staging_offset(o)); + staged_outputs |= 1U << o; + ++g_stats.staged_outputs; + } + } + if (!ok) { std::printf( "EspNN delegate: output %u has an unsupported dtype, shape or dim order\n", static_cast(o)); return Error::InvalidArgument; } } + if (!interpreter.invoke()) + return Error::Internal; + for (size_t o = 0; o < outputs; ++o) { + if ((staged_outputs & (1U << o)) == 0) + continue; + Tensor tensor = args[inputs + o]->toTensor(); + interpreter.output_info(o, info); + if (!store_output( + info, + tensor_layout(tensor, info), + interpreter.output_data(o), + tensor)) + return Error::InvalidArgument; + } ++g_stats.calls; g_stats.total_us += esp_timer_get_time() - started; #if CONFIG_ESPNN_DELEGATE_PROFILE @@ -266,6 +467,31 @@ class EspNNBackendInterface final : public BackendInterface { return Error::Ok; } + Error set_option( + BackendOptionContext&, + const Span& backend_options) override { + for (const BackendOption& option : backend_options) { + if (std::strcmp(option.key, kResetStatesOption) != 0) { + std::printf("EspNN delegate: unknown option %s\n", option.key); + return Error::NotSupported; + } + const bool* all = std::get_if(&option.value); + const int* group = std::get_if(&option.value); + if (all != nullptr && *all) { + reset_delegate_states(); + } else if ( + all == nullptr && + (group == nullptr || !reset_delegate_states(*group))) { + std::printf( + "EspNN delegate: %s takes true or a reset group from 0 to %d\n", + kResetStatesOption, + kResetGroups - 1); + return Error::InvalidArgument; + } + } + return Error::Ok; + } + void destroy(DelegateHandle* handle) const override { if (handle != nullptr) static_cast(handle)->~DelegateState(); @@ -294,6 +520,18 @@ DelegateStats delegate_stats() { g_stats = {}; } +void reset_delegate_states() { + for (std::atomic& generation : g_state_generations) + generation.fetch_add(1, std::memory_order_relaxed); +} + +bool reset_delegate_states(int group) { + if (group < 0 || group >= kResetGroups) + return false; + g_state_generations[group].fetch_add(1, std::memory_order_relaxed); + return true; +} + } // namespace espressif } // namespace backends } // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index 391574781c2..515635e7eeb 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -11,10 +11,12 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -30,16 +32,20 @@ namespace { constexpr size_t ARENA_ALIGNMENT = 16; constexpr int64_t YIELD_INTERVAL_US = 1000000; +// Without profiling, invoke() reads the cycle counter, which takes a few +// cycles, after every operation, and the timer, which costs as much as a small +// operation, only once this many cycles (4 ms at 240 MHz) have passed. +constexpr uint32_t YIELD_CHECK_CYCLES = 1U << 20; -SemaphoreHandle_t invoke_mutex() { - static StaticSemaphore_t storage; - static SemaphoreHandle_t handle = xSemaphoreCreateMutexStatic(&storage); - return handle; -} +// Created by a static initializer rather than on first use, so that invoke() +// needs no thread-safe static guard (and no C++ runtime support for one). +StaticSemaphore_t g_invoke_mutex_storage; +const SemaphoreHandle_t g_invoke_mutex = + xSemaphoreCreateMutexStatic(&g_invoke_mutex_storage); class InvokeLock { public: - InvokeLock() : handle_(invoke_mutex()) { + InvokeLock() : handle_(g_invoke_mutex) { xSemaphoreTake(handle_, portMAX_DELAY); } ~InvokeLock() { @@ -52,6 +58,71 @@ class InvokeLock { SemaphoreHandle_t handle_; }; +// A version 7 graph keeps its tensor table and parameter region word-aligned. +// On a little-endian target its tensor records then have TensorRecord's +// layout, and per-channel multipliers and shifts are int32 arrays. +static_assert( + sizeof(TensorRecord) == TENSOR_RECORD_SIZE && + offsetof(TensorRecord, dtype) == 8 && + offsetof(TensorRecord, rank) == 10 && + offsetof(TensorRecord, shape) == 12 && + offsetof(TensorRecord, quant_scale) == 28 && + offsetof(TensorRecord, quant_zero_point) == 32, + "TensorRecord must have the XTS3 tensor record layout"); + +bool graph_tables_aligned(const GraphView& graph) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION && + (reinterpret_cast(graph.data) & 3U) == 0; +#else + return false; +#endif +} + +// An int32 array of the parameter region where it sits in the graph, or +// nullptr when it must be copied. +const int32_t* int32s_in_place(bool tables_in_place, const uint8_t* data) { + if (!tables_in_place || (reinterpret_cast(data) & 3U) != 0) + return nullptr; + return reinterpret_cast(data); +} + +// The per-channel requantization multipliers and shifts of a convolution or +// linear operation, little-endian int32 arrays in its parameter block. +bool requantization_arrays( + const GraphView& graph, + const TensorRecord* tensors, + const OperationRecord& op, + const uint8_t*& multipliers, + const uint8_t*& shifts, + uint32_t& count) { + const uint8_t* p = graph.parameter_data + op.parameter_offset; + switch (static_cast(op.opcode)) { + case OpCode::CONV2D: + case OpCode::DEPTHWISE_CONV2D: { + count = static_cast(tensors[op.outputs[0]].shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph, op, count, params)) + return false; + multipliers = params.multipliers; + shifts = params.shifts; + return true; + } + case OpCode::TRANSPOSE_CONV2D: + count = static_cast(tensors[op.outputs[0]].shape[1]); + multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + shifts = multipliers + 4 * count; + return true; + case OpCode::LINEAR: + count = static_cast(tensors[op.inputs[1]].shape[0]); + multipliers = p + INT8_PAIR_PARAMS_SIZE; + shifts = multipliers + 4 * count; + return true; + default: + return false; + } +} + size_t align_up(size_t value, size_t alignment) { return (value + alignment - 1) & ~(alignment - 1); } @@ -139,27 +210,47 @@ int8_t quantize_value(float value, float inverse_scale, int32_t zero_point) { static_cast(std::nearbyintf(scaled)) + zero_point); } +// esp-nn's ESP32-S3 pointwise convolution passes the number of output +// positions as a uint16_t width and leaves the rest unwritten. +bool pointwise_beyond_uint16( + const data_dims_t& filter, + const data_dims_t& output) { + return filter.width == 1 && filter.height == 1 && + static_cast(output.width) * output.height > UINT16_MAX; +} + } // namespace +// Fields that no operation uses together share storage: the input list +// (CONCAT, COPY) with the pooling and convolution windows, and the +// convolution, depthwise-convolution and add parameters with each other. struct Int8Interpreter::OpState { OpCode opcode; uint32_t input0; uint32_t input1; uint32_t output; uint8_t input_count; - uint32_t inputs[6]; - int32_t pool[6]; + // COPY: the first writer of an output in caller memory clears it first. + bool clear_output; + union { + uint32_t inputs[6]; + int32_t pool[6]; + }; const int8_t* weights; const int32_t* bias; data_dims_t input_dims; data_dims_t filter_dims; data_dims_t output_dims; - conv_params_t conv; - dw_conv_params_t dw; + union { + conv_params_t conv; + dw_conv_params_t dw; + Int8AddParams add; + }; quant_data_t quant; int32_t* multipliers; int32_t* shifts; - Int8AddParams add; + uint32_t scratch_bytes; + uint32_t scratch_offset; int32_t value0; int32_t value1; int32_t input_zero_point; @@ -173,9 +264,6 @@ struct Int8Interpreter::OpState { int32_t dilation_h; int32_t dilation_w; int32_t batches; - int64_t total_us; - int64_t last_us; - uint32_t calls; }; Int8Interpreter::~Int8Interpreter() { @@ -195,22 +283,23 @@ void Int8Interpreter::reset() { heap_caps_free(params_); if (blob_copy_ != nullptr) heap_caps_free(blob_copy_); - if (tensors_ != nullptr) - heap_caps_free(tensors_); - if (operations_ != nullptr) - heap_caps_free(operations_); + if (tensors_ != nullptr && !tables_in_place_) + heap_caps_free(const_cast(tensors_)); + tables_in_place_ = false; if (op_states_ != nullptr) heap_caps_free(op_states_); - if (tensor_offsets_ != nullptr) - heap_caps_free(tensor_offsets_); + if (op_us_ != nullptr) + heap_caps_free(op_us_); + if (tensor_data_ != nullptr) + heap_caps_free(tensor_data_); arena_ = nullptr; scratch_ = nullptr; params_ = nullptr; blob_copy_ = nullptr; tensors_ = nullptr; - operations_ = nullptr; op_states_ = nullptr; - tensor_offsets_ = nullptr; + op_us_ = nullptr; + tensor_data_ = nullptr; arena_size_ = 0; scratch_size_ = 0; params_size_ = 0; @@ -232,10 +321,10 @@ bool Int8Interpreter::load( } if (!is_int8_graph_version(graph_.header.version)) { std::printf( - "ERROR: model is XTS3 v%u; the int8 runtime requires v%u or v%u\n", + "ERROR: model is XTS3 v%u; the int8 runtime requires v%u to v%u\n", static_cast(graph_.header.version), static_cast(GRAPH_ESP_NN_INT8_VERSION), - static_cast(GRAPH_ESP_NN_INT8_IO_VERSION)); + static_cast(GRAPH_ESP_NN_INT8_STATE_VERSION)); return false; } const bool misaligned = @@ -258,22 +347,31 @@ bool Int8Interpreter::load( const uint32_t tensor_count = graph_.header.num_tensors; const uint32_t op_count = graph_.header.num_operations; - tensors_ = static_cast( - heap_caps_calloc(tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); - operations_ = static_cast( - heap_caps_calloc(op_count, sizeof(OperationRecord), MALLOC_CAP_8BIT)); + tables_in_place_ = graph_tables_aligned(graph_); + tensors_ = tables_in_place_ + ? reinterpret_cast( + graph_.data + graph_.tensor_table_offset) + : static_cast(heap_caps_calloc( + tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); op_states_ = static_cast( heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); - tensor_offsets_ = static_cast( - heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); - if (tensors_ == nullptr || operations_ == nullptr || op_states_ == nullptr || - tensor_offsets_ == nullptr) { + tensor_data_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(int8_t*), MALLOC_CAP_8BIT)); + if (options_.profile_operations) { + op_us_ = static_cast( + heap_caps_calloc(op_count, sizeof(int64_t), MALLOC_CAP_8BIT)); + } + if (tensors_ == nullptr || op_states_ == nullptr || tensor_data_ == nullptr || + (options_.profile_operations && op_us_ == nullptr)) { std::printf("ERROR: unable to allocate interpreter tables\n"); reset(); return false; } - for (uint32_t i = 0; i < tensor_count; ++i) - read_tensor(graph_, i, tensors_[i]); + if (!tables_in_place_) { + TensorRecord* tensors = const_cast(tensors_); + for (uint32_t i = 0; i < tensor_count; ++i) + read_tensor(graph_, i, tensors[i]); + } input_count_ = graph_input_count(graph_); output_count_ = graph_output_count(graph_); if (input_count_ > MAX_GRAPH_IO || output_count_ > MAX_GRAPH_IO) { @@ -283,17 +381,37 @@ bool Int8Interpreter::load( reset(); return false; } + if ((static_cast(options_.conversion_slots) >> + (input_count_ + output_count_)) != 0) { + std::printf("ERROR: conversion mask selects a nonexistent I/O slot\n"); + reset(); + return false; + } for (uint32_t i = 0; i < input_count_; ++i) inputs_[i] = graph_input_id(graph_, i); for (uint32_t i = 0; i < output_count_; ++i) outputs_[i] = graph_output_id(graph_, i); - for (uint32_t i = 0; i < op_count; ++i) - read_operation(graph_, i, operations_[i]); + // Like graph inputs, outputs that an operation reads keep their arena slots + // (see plan_memory()). + for (uint32_t op = 0; options_.external_io && op < op_count; ++op) { + OperationRecord record{}; + read_operation(graph_, op, record); + for (uint8_t j = 0; j < record.num_inputs; ++j) { + for (uint32_t o = 0; o < output_count_; ++o) { + if (outputs_[o] == record.inputs[j]) + options_.conversion_slots |= 1U << (input_count_ + o); + } + } + } - if (!plan_memory() || !prepare_operations()) { + // The planner places each operation's ESP-NN scratch, and the arena gets + // the first claim on internal RAM. + if (!prepare_operations() || !plan_memory() || + !allocate_operation_buffers()) { reset(); return false; } + reset_states(); if (options_.copy_constants_to_psram && blob_copy_ == nullptr && heap_caps_get_total_size(MALLOC_CAP_SPIRAM) != 0) { const size_t largest = @@ -314,6 +432,10 @@ bool Int8Interpreter::load( return false; } rebase_constants(old_base, size); + if (tables_in_place_) { + tensors_ = reinterpret_cast( + graph_.data + graph_.tensor_table_offset); + } } } if (options_.dual_core) { @@ -330,38 +452,85 @@ bool Int8Interpreter::load( bool Int8Interpreter::plan_memory() { const uint32_t tensor_count = graph_.header.num_tensors; const int32_t op_count = static_cast(graph_.header.num_operations); + // Entries below tensor_count are tensors. With scratch_in_arena, entry + // tensor_count + op is that operation's ESP-NN scratch, live only while the + // operation runs. + const uint32_t entries = tensor_count + + (options_.scratch_in_arena ? static_cast(op_count) : 0U); int32_t* first = static_cast( - heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(int32_t), MALLOC_CAP_8BIT)); int32_t* last = static_cast( - heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(int32_t), MALLOC_CAP_8BIT)); size_t* bytes = static_cast( - heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(size_t), MALLOC_CAP_8BIT)); + size_t* offsets = static_cast( + heap_caps_calloc(entries, sizeof(size_t), MALLOC_CAP_8BIT)); bool* planned = static_cast( - heap_caps_calloc(tensor_count, sizeof(bool), MALLOC_CAP_8BIT)); + heap_caps_calloc(entries, sizeof(bool), MALLOC_CAP_8BIT)); struct PlanBuffers { void* buffers[4]; - ~PlanBuffers() { - for (void* buffer : buffers) { + void release() { + for (void*& buffer : buffers) { if (buffer != nullptr) heap_caps_free(buffer); + buffer = nullptr; } } + ~PlanBuffers() { + release(); + } } plan_buffers{{first, last, bytes, planned}}; + struct OffsetBuffer { + size_t* offsets; + ~OffsetBuffer() { + heap_caps_free(offsets); + } + } offset_buffer{offsets}; if (first == nullptr || last == nullptr || bytes == nullptr || - planned == nullptr) { + offsets == nullptr || planned == nullptr) { std::printf("ERROR: unable to allocate memory-planner tables\n"); return false; } + // Kernels may load whole vectors past the end of a tensor they read, which + // stays within the arena but not within a caller's buffer, so graph inputs + // are always planned. + const auto unplanned = [&](uint32_t id) { + if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) != 0) + return true; + if (!options_.external_io || + (tensors_[id].flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != + TENSOR_FLAG_GRAPH_OUTPUT) + return false; + for (size_t o = 0; o < output_count_; ++o) { + if (outputs_[o] == id && + (options_.conversion_slots & (1U << (input_count_ + o))) != 0) + return false; + } + return true; + }; for (uint32_t i = 0; i < tensor_count; ++i) { first[i] = - (tensors_[i].flags & TENSOR_FLAG_GRAPH_INPUT) != 0 ? -1 : INT32_MAX; + (tensors_[i].flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE)) != 0 + ? -1 + : INT32_MAX; last[i] = (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); - tensor_offsets_[i] = SIZE_MAX; + planned[i] = unplanned(i); + offsets[i] = SIZE_MAX; + } + for (uint32_t e = tensor_count; e < entries; ++e) { + const int32_t op = static_cast(e - tensor_count); + const size_t scratch = op_states_[op].scratch_bytes; + first[e] = scratch != 0 ? op : INT32_MAX; + last[e] = op; + bytes[e] = align_up(scratch, ARENA_ALIGNMENT); + offsets[e] = SIZE_MAX; } for (int32_t op = 0; op < op_count; ++op) { - const OperationRecord& record = operations_[op]; + OperationRecord record{}; + read_operation(graph_, static_cast(op), record); for (uint8_t j = 0; j < record.num_inputs; ++j) { const uint32_t id = record.inputs[j]; if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) == 0 && last[id] < op) @@ -376,13 +545,29 @@ bool Int8Interpreter::plan_memory() { } } + // A state keeps its slot across invokes, so it is live for the whole graph. + // Its update is written straight into that slot when no operation reads the + // state at or after the update's first write; otherwise the update gets its + // own slot and is copied over the state after the last operation. + updates_in_place_ = 0; + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if (last[state] < first[update]) { + updates_in_place_ |= 1U << k; + planned[update] = true; + } else { + last[update] = op_count; + } + last[state] = op_count; + } + size_t arena_size = 0; - for (uint32_t step = 0; step < tensor_count; ++step) { + for (uint32_t step = 0; step < entries; ++step) { int32_t best = -1; - for (uint32_t i = 0; i < tensor_count; ++i) { - if (planned[i] || (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0) - continue; - if (first[i] == INT32_MAX) + for (uint32_t i = 0; i < entries; ++i) { + if (planned[i] || first[i] == INT32_MAX) continue; if (best < 0 || bytes[i] > bytes[best]) best = static_cast(i); @@ -394,14 +579,14 @@ bool Int8Interpreter::plan_memory() { bool moved = true; while (moved) { moved = false; - for (uint32_t j = 0; j < tensor_count; ++j) { - if (j == static_cast(best) || tensor_offsets_[j] == SIZE_MAX) + for (uint32_t j = 0; j < entries; ++j) { + if (j == static_cast(best) || offsets[j] == SIZE_MAX) continue; const bool lifetime_overlap = first[j] <= last[best] && first[best] <= last[j]; if (!lifetime_overlap) continue; - const size_t other_begin = tensor_offsets_[j]; + const size_t other_begin = offsets[j]; const size_t other_end = other_begin + bytes[j]; if (candidate < other_end && other_begin < candidate + bytes[best]) { candidate = other_end; @@ -409,63 +594,100 @@ bool Int8Interpreter::plan_memory() { } } } - tensor_offsets_[best] = candidate; + offsets[best] = candidate; if (candidate + bytes[best] > arena_size) arena_size = candidate + bytes[best]; } + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if ((updates_in_place_ & (1U << k)) != 0) + offsets[update] = offsets[state]; + } for (uint32_t i = 0; i < tensor_count; ++i) { - if ((tensors_[i].flags & TENSOR_FLAG_CONSTANT) == 0 && - tensor_offsets_[i] == SIZE_MAX) { + if (!unplanned(i) && offsets[i] == SIZE_MAX) { std::printf( "ERROR: tensor %u is never produced or consumed\n", static_cast(i)); return false; } } + for (uint32_t e = tensor_count; e < entries; ++e) { + op_states_[e - tensor_count].scratch_offset = + offsets[e] == SIZE_MAX ? 0 : static_cast(offsets[e]); + } + // Loading then needs at most the tensor offsets alongside the arena. + plan_buffers.release(); + // With external_io, a graph whose every activation is an output needs no + // arena. arena_size_ = arena_size; - arena_ = static_cast(allocate_preferring_internal( - arena_size_, - options_.prefer_internal_arena, - options_.internal_reserve_bytes, - &arena_internal_)); + if (arena_size_ == 0) + return true; + // Readable bytes after the last slot, for kernels that load a whole vector + // past the end of a tensor. + const size_t allocated = arena_size_ + ARENA_ALIGNMENT; + arena_ = allocated < arena_size_ + ? nullptr + : static_cast(allocate_preferring_internal( + allocated, + options_.prefer_internal_arena, + options_.internal_reserve_bytes, + &arena_internal_)); if (arena_ == nullptr) { std::printf( "ERROR: unable to allocate %u-byte activation arena\n", static_cast(arena_size_)); return false; } - std::memset(arena_, 0, arena_size_); + std::memset(arena_, 0, allocated); + for (uint32_t i = 0; i < tensor_count; ++i) { + if (offsets[i] != SIZE_MAX) + tensor_data_[i] = reinterpret_cast(arena_ + offsets[i]); + } return true; } bool Int8Interpreter::prepare_operations() { const uint32_t op_count = graph_.header.num_operations; + // Requantization parameters are used where they sit in a version 7 graph, + // and copied into params_ otherwise. size_t param_words = 0; for (uint32_t i = 0; i < op_count; ++i) { - const OperationRecord& op = operations_[i]; - const OpCode code = static_cast(op.opcode); - if (code == OpCode::CONV2D || code == OpCode::DEPTHWISE_CONV2D || - code == OpCode::LINEAR || code == OpCode::TRANSPOSE_CONV2D) { - param_words += 2U * tensor_element_count(tensors_[op.inputs[2]]); - } else if (code == OpCode::SOFTMAX) { + OperationRecord op{}; + read_operation(graph_, i, op); + const uint8_t* multipliers = nullptr; + const uint8_t* shifts = nullptr; + uint32_t channels = 0; + if (requantization_arrays( + graph_, tensors_, op, multipliers, shifts, channels)) { + if (int32s_in_place(tables_in_place_, multipliers) == nullptr || + int32s_in_place(tables_in_place_, shifts) == nullptr) + param_words += 2U * channels; + } else if ( + static_cast(op.opcode) == OpCode::SOFTMAX && + int32s_in_place( + tables_in_place_, graph_.parameter_data + op.parameter_offset) == + nullptr) { param_words += 256U; } } params_size_ = param_words * sizeof(int32_t); - if (params_size_ != 0) { - params_ = static_cast(allocate_preferring_internal( - params_size_, true, options_.internal_reserve_bytes, nullptr)); - if (params_ == nullptr) { - std::printf("ERROR: unable to allocate requantization tables\n"); - return false; - } - } - int32_t* cursor = params_; size_t scratch_size = 0; + auto require_scratch = [&scratch_size](OpState& state, int32_t needed) { + if (needed <= 0) + return; + const uint32_t bytes = static_cast(needed); + if (bytes > state.scratch_bytes) + state.scratch_bytes = bytes; + if (bytes > scratch_size) + scratch_size = bytes; + }; for (uint32_t i = 0; i < op_count; ++i) { - const OperationRecord& op = operations_[i]; + OperationRecord op{}; + read_operation(graph_, i, op); OpState& state = op_states_[i]; state = {}; state.opcode = static_cast(op.opcode); @@ -492,15 +714,6 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, weight)); state.bias = reinterpret_cast( tensor_constant_data(graph_, bias)); - state.multipliers = cursor; - state.shifts = cursor + channels; - for (uint32_t c = 0; c < channels; ++c) { - state.multipliers[c] = read_le_i32(params.multipliers + c * 4); - state.shifts[c] = read_le_i32(params.shifts + c * 4); - } - cursor += 2U * channels; - state.quant.mult = state.multipliers; - state.quant.shift = state.shifts; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; state.output_dims = { output.shape[3], output.shape[2], output.shape[1], 1}; @@ -547,8 +760,7 @@ bool Int8Interpreter::prepare_operations() { params.activation_min, params.activation_max}; const int needed = esp_nn_get_conv_scratch_size( &tile_in, &state.filter_dims, &tile_out, &state.conv); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } else { state.filter_dims = { params.kernel_w, params.kernel_h, output.shape[1], 1}; @@ -562,8 +774,7 @@ bool Int8Interpreter::prepare_operations() { params.activation_min, params.activation_max}; const int needed = esp_nn_get_depthwise_conv_scratch_size( &tile_in, &state.filter_dims, &tile_out, &state.dw); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } break; } @@ -582,8 +793,7 @@ bool Int8Interpreter::prepare_operations() { &state.filter_dims, &state.output_dims, &state.conv); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } else { state.filter_dims = { params.kernel_w, params.kernel_h, output.shape[1], 1}; @@ -599,8 +809,7 @@ bool Int8Interpreter::prepare_operations() { &state.filter_dims, &state.output_dims, &state.dw); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); } break; } @@ -618,6 +827,12 @@ bool Int8Interpreter::prepare_operations() { for (uint8_t k = 0; k < op.num_inputs; ++k) state.inputs[k] = op.inputs[k]; state.table = p; + // Several COPY operations may write parts of one output (a long cat). + state.clear_output = options_.external_io && + (output.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && + (read_le_i32(p + 4) & COPY_FLAG_FILL) == 0; + for (uint32_t j = 0; state.clear_output && j < i; ++j) + state.clear_output = op_states_[j].output != state.output; break; case OpCode::UPSAMPLE: state.table = p; @@ -629,7 +844,6 @@ bool Int8Interpreter::prepare_operations() { case OpCode::TRANSPOSE_CONV2D: { const TensorRecord& weight = tensors_[op.inputs[1]]; const TensorRecord& bias = tensors_[op.inputs[2]]; - const uint32_t channels = static_cast(output.shape[1]); state.weights = reinterpret_cast( tensor_constant_data(graph_, weight)); state.bias = reinterpret_cast( @@ -640,15 +854,6 @@ bool Int8Interpreter::prepare_operations() { state.dilation_w = read_le_i32(p + 36); state.value0 = read_le_i32(p + 44); state.value1 = read_le_i32(p + 48); - state.multipliers = cursor; - state.shifts = cursor + channels; - const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; - const uint8_t* shifts = multipliers + channels * 4U; - for (uint32_t c = 0; c < channels; ++c) { - state.multipliers[c] = read_le_i32(multipliers + c * 4); - state.shifts[c] = read_le_i32(shifts + c * 4); - } - cursor += 2U * channels; state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; state.output_dims = { output.shape[3], output.shape[2], output.shape[1], 1}; @@ -660,8 +865,7 @@ bool Int8Interpreter::prepare_operations() { state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; const int32_t needed = esp_nn_get_mean_scratch_size( input.shape[2], input.shape[3], input.shape[1]); - if (needed > 0 && static_cast(needed) > scratch_size) - scratch_size = needed; + require_scratch(state, needed); break; } case OpCode::MAX_POOL2D: @@ -689,10 +893,6 @@ bool Int8Interpreter::prepare_operations() { state.rows = static_cast(state.elements) / state.cols; break; case OpCode::SOFTMAX: - state.multipliers = cursor; - for (int k = 0; k < 256; ++k) - state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); - cursor += 256; state.cols = input.shape[1]; state.rows = static_cast(state.elements) / state.cols; break; @@ -724,16 +924,6 @@ bool Int8Interpreter::prepare_operations() { tensor_constant_data(graph_, bias)); state.value0 = read_le_i32(p); state.value1 = read_le_i32(p + 4); - state.multipliers = cursor; - state.shifts = cursor + state.rows; - const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; - const uint8_t* shifts = - multipliers + static_cast(state.rows) * 4U; - for (int32_t r = 0; r < state.rows; ++r) { - state.multipliers[r] = read_le_i32(multipliers + r * 4); - state.shifts[r] = read_le_i32(shifts + r * 4); - } - cursor += 2 * state.rows; break; } default: @@ -741,6 +931,59 @@ bool Int8Interpreter::prepare_operations() { } } + scratch_size_ = options_.scratch_in_arena ? 0 : scratch_size; + return true; +} + +bool Int8Interpreter::allocate_operation_buffers() { + if (params_size_ != 0) { + params_ = static_cast(allocate_preferring_internal( + params_size_, true, options_.internal_reserve_bytes, nullptr)); + if (params_ == nullptr) { + std::printf("ERROR: unable to allocate requantization tables\n"); + return false; + } + } + int32_t* cursor = params_; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph_, i, op); + OpState& state = op_states_[i]; + const uint8_t* multipliers = nullptr; + const uint8_t* shifts = nullptr; + uint32_t channels = 0; + if (requantization_arrays( + graph_, tensors_, op, multipliers, shifts, channels)) { + const int32_t* multipliers_in_place = + int32s_in_place(tables_in_place_, multipliers); + const int32_t* shifts_in_place = + int32s_in_place(tables_in_place_, shifts); + if (multipliers_in_place != nullptr && shifts_in_place != nullptr) { + state.multipliers = const_cast(multipliers_in_place); + state.shifts = const_cast(shifts_in_place); + } else { + state.multipliers = cursor; + state.shifts = cursor + channels; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(multipliers + c * 4); + state.shifts[c] = read_le_i32(shifts + c * 4); + } + cursor += 2U * channels; + } + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; + } else if (state.opcode == OpCode::SOFTMAX) { + const uint8_t* p = graph_.parameter_data + op.parameter_offset; + if (const int32_t* table = int32s_in_place(tables_in_place_, p)) { + state.multipliers = const_cast(table); + } else { + state.multipliers = cursor; + for (int k = 0; k < 256; ++k) + state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); + cursor += 256; + } + } + } if (phase_bytes_ != 0) { phase_buffer_ = static_cast(allocate_preferring_internal( phase_bytes_, true, options_.internal_reserve_bytes, nullptr)); @@ -751,7 +994,6 @@ bool Int8Interpreter::prepare_operations() { return false; } } - scratch_size_ = scratch_size; if (scratch_size_ != 0) { scratch_ = allocate_preferring_internal( scratch_size_, true, options_.internal_reserve_bytes, nullptr); @@ -781,11 +1023,33 @@ void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { state.bias = reinterpret_cast(rebase(state.bias)); if (state.table != nullptr) state.table = rebase(state.table); + state.multipliers = const_cast( + reinterpret_cast(rebase(state.multipliers))); + state.shifts = const_cast( + reinterpret_cast(rebase(state.shifts))); + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; } } +bool Int8Interpreter::in_arena(const int8_t* data) const { + const uintptr_t address = reinterpret_cast(data); + const uintptr_t arena = reinterpret_cast(arena_); + return address >= arena && address - arena < arena_size_; +} + int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { - return reinterpret_cast(arena_ + tensor_offsets_[tensor_id]); + return tensor_data_[tensor_id]; +} + +bool Int8Interpreter::bind_output(size_t index, int8_t* data) { + if (!loaded_ || !options_.external_io || index >= output_count_ || + data == nullptr || + reinterpret_cast(data) % ARENA_ALIGNMENT != 0 || + in_arena(tensor_data_[outputs_[index]])) + return false; + tensor_data_[outputs_[index]] = data; + return true; } const int8_t* Int8Interpreter::operand(uint32_t tensor_id) const { @@ -925,9 +1189,37 @@ int32_t Int8Interpreter::output_argmax(size_t index) const { return best; } +size_t Int8Interpreter::states_updated_in_place() const { + size_t count = 0; + for (uint32_t k = 0; k < graph_.num_states; ++k) + count += (updates_in_place_ >> k) & 1U; + return count; +} + +void Int8Interpreter::reset_states() { + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + std::memcpy( + activation(state), + tensor_constant_data(graph_, tensors_[state]), + tensor_data_size(tensors_[state])); + } +} + +size_t Int8Interpreter::table_bytes() const { + if (!loaded_) + return 0; + const size_t records = tables_in_place_ ? 0 : sizeof(TensorRecord); + return graph_.header.num_operations * sizeof(OpState) + + graph_.header.num_tensors * (records + sizeof(int8_t*)); +} + int64_t Int8Interpreter::op_time_us(uint32_t op) const { - return loaded_ && op < graph_.header.num_operations ? op_states_[op].last_us - : 0; + return loaded_ && op_us_ != nullptr && op < graph_.header.num_operations + ? op_us_[op] + : 0; } bool Int8Interpreter::run_operation(uint32_t index) { @@ -941,7 +1233,7 @@ bool Int8Interpreter::run_operation(uint32_t index) { run_generic_conv(s, input, output); return true; } - if (reference) { + if (reference || pointwise_beyond_uint16(s.filter_dims, s.output_dims)) { esp_nn_conv_s8_ansi( &s.input_dims, input, @@ -1095,7 +1387,9 @@ bool Int8Interpreter::run_operation(uint32_t index) { sw = s.pool[3]; const uint16_t ph = s.pool[4], pw = s.pool[5]; if (s.opcode == OpCode::MAX_POOL2D) { - if (reference) { + // The ESP32-S3 kernel takes a multiple of four channels and writes + // nothing otherwise. + if (reference || channels % 4 != 0) { esp_nn_max_pool_s8_ansi( input, in_w, @@ -1389,7 +1683,7 @@ void Int8Interpreter::run_dilated_by_phases( data_dims_t tile_dims{tile_w, tile_h, in_c, 1}; data_dims_t result_dims{cols, rows, out_c, 1}; if (s.opcode == OpCode::CONV2D) { - if (reference) { + if (reference || pointwise_beyond_uint16(s.filter_dims, result_dims)) { esp_nn_conv_s8_ansi( &tile_dims, tile, @@ -1521,6 +1815,10 @@ void Int8Interpreter::run_copy(const OpState& s, int8_t* output) const { if ((flags & COPY_FLAG_FILL) != 0) { std::memset( output, read_le_i32(p + 8), tensor_data_size(tensors_[s.output])); + } else if (s.clear_output) { + // Segments that do not cover a caller's output buffer must not leave its + // previous contents there. + std::memset(output, 0, tensor_data_size(tensors_[s.output])); } for (int32_t k = 0; k < segments; ++k) { const uint8_t* q = @@ -1674,16 +1972,37 @@ void Int8Interpreter::run_transposed_conv( bool Int8Interpreter::invoke() { if (!loaded_) return false; + if (options_.external_io) { + for (size_t o = 0; o < output_count_; ++o) { + if (tensor_data_[outputs_[o]] == nullptr) + return false; + } + } InvokeLock lock; + // A kernel that leaves part of its output unwritten must not leave a + // caller's previous contents in a bound output. + for (size_t o = 0; options_.external_io && o < output_count_; ++o) { + int8_t* data = tensor_data_[outputs_[o]]; + if (!in_arena(data)) + std::memset(data, 0, tensor_data_size(tensors_[outputs_[o]])); + } if (scratch_ != nullptr) { esp_nn_set_conv_scratch_buf(scratch_); esp_nn_set_depthwise_conv_scratch_buf(scratch_); esp_nn_set_mean_scratch_buf(scratch_); } - int64_t compute_us = 0; - int64_t last_yield = esp_timer_get_time(); + const int64_t invoke_started = esp_timer_get_time(); + int64_t op_started = invoke_started; + int64_t last_yield = invoke_started; + uint32_t last_check = esp_cpu_get_cycle_count(); for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { - const int64_t started = esp_timer_get_time(); + const OpState& state = op_states_[i]; + if (options_.scratch_in_arena && state.scratch_bytes != 0) { + void* scratch = arena_ + state.scratch_offset; + esp_nn_set_conv_scratch_buf(scratch); + esp_nn_set_depthwise_conv_scratch_buf(scratch); + esp_nn_set_mean_scratch_buf(scratch); + } if (!run_operation(i)) { std::printf( "ERROR: operation %u (%s) failed\n", @@ -1691,18 +2010,37 @@ bool Int8Interpreter::invoke() { opcode_name(op_states_[i].opcode)); return false; } - const int64_t finished = esp_timer_get_time(); - OpState& state = op_states_[i]; - state.last_us = finished - started; - state.total_us += state.last_us; - ++state.calls; - compute_us += state.last_us; - if (finished - last_yield > YIELD_INTERVAL_US) { + if (op_us_ == nullptr) { + const uint32_t cycles = esp_cpu_get_cycle_count(); + if (cycles - last_check < YIELD_CHECK_CYCLES) + continue; + last_check = cycles; + } + int64_t now = esp_timer_get_time(); + if (op_us_ != nullptr) + op_us_[i] = now - op_started; + if (now - last_yield > YIELD_INTERVAL_US) { vTaskDelay(1); - last_yield = esp_timer_get_time(); + now = esp_timer_get_time(); + last_yield = now; + } + op_started = now; + } + for (uint32_t k = 0; k < graph_.num_states; ++k) { + uint32_t state = 0; + uint32_t update = 0; + graph_state_pair(graph_, k, state, update); + if ((updates_in_place_ & (1U << k)) == 0) { + std::memcpy( + activation(state), + activation(update), + tensor_data_size(tensors_[state])); } } - last_invoke_us_ = compute_us; + const int64_t finished = esp_timer_get_time(); + if (finished - last_yield > YIELD_INTERVAL_US) + vTaskDelay(1); + last_invoke_us_ = finished - invoke_started; return true; } @@ -1750,7 +2088,7 @@ void Int8Interpreter::print_summary() const { } void Int8Interpreter::print_profile() const { - if (!loaded_) + if (!loaded_ || op_us_ == nullptr) return; int64_t per_type[OPCODE_COUNT]{}; uint32_t count_per_type[OPCODE_COUNT]{}; @@ -1758,7 +2096,7 @@ void Int8Interpreter::print_profile() const { const OpState& s = op_states_[i]; const unsigned type = static_cast(s.opcode); if (type < OPCODE_COUNT) { - per_type[type] += s.last_us; + per_type[type] += op_us_[i]; ++count_per_type[type]; } std::printf( @@ -1772,7 +2110,7 @@ void Int8Interpreter::print_profile() const { static_cast( tensors_[s.output].rank == 4 ? tensors_[s.output].shape[1] : static_cast(s.elements)), - static_cast(s.last_us)); + static_cast(op_us_[i])); } for (unsigned t = 0; t < OPCODE_COUNT; ++t) { if (count_per_type[t] == 0) diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index af6b6c38453..1e6927b668c 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -42,7 +42,7 @@ bool valid_dtype(uint8_t dtype) { } bool valid_opcode(uint8_t opcode, uint32_t version) { - if (version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (version >= GRAPH_ESP_NN_INT8_IO_VERSION) { return opcode <= static_cast(OpCode::TRANSPOSE_CONV2D); } return opcode <= static_cast(OpCode::SOFTMAX); @@ -154,8 +154,10 @@ bool validate_int8_conv( 1; if (output.shape[2] != expected_h || output.shape[3] != expected_w) return false; + // ESP-NN's convolution kernels take these as uint16_t. if (input.shape[1] > 65535 || input.shape[2] > 65535 || - input.shape[3] > 65535 || output.shape[1] > 65535) + input.shape[3] > 65535 || output.shape[1] > 65535 || + output.shape[2] > 65535 || output.shape[3] > 65535) return false; for (uint32_t i = 0; i < channels; ++i) { if (!valid_requantization( @@ -407,7 +409,8 @@ bool validate_int8_operation( !is_int8_activation(out) || !same_quantization(a, out) || a.rank != 4 || out.rank != 4 || a.shape[0] != 1 || out.shape[0] != 1 || a.shape[1] != out.shape[1] || - a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535) + a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535 || + out.shape[2] > 65535 || out.shape[3] > 65535) return false; const int32_t kh = p[0]; const int32_t kw = p[1]; @@ -524,24 +527,41 @@ bool validate_int8_operation( return false; } +// Set of tensor IDs, so that validation keeps 128 bytes per set on the stack +// instead of a 1,024-byte bool array. +class TensorSet { + public: + bool operator[](uint32_t id) const { + return ((bits_[id / 32] >> (id % 32)) & 1U) != 0; + } + void set(uint32_t id) { + bits_[id / 32] |= 1U << (id % 32); + } + + private: + uint32_t bits_[MAX_GRAPH_TENSORS / 32]{}; +}; + bool validate_int8_dataflow(const GraphView& graph) { - bool produced[MAX_GRAPH_TENSORS]{}; - bool copy_produced[MAX_GRAPH_TENSORS]{}; + TensorSet produced; + TensorSet copy_produced; uint32_t input_count = 0; uint32_t output_count = 0; for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; read_tensor(graph, i, tensor); if ((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0) { - produced[i] = true; + produced.set(i); ++input_count; } + if ((tensor.flags & TENSOR_FLAG_STATE) != 0) + produced.set(i); if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) ++output_count; } - if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { - bool listed_input[MAX_GRAPH_TENSORS]{}; - bool listed_output[MAX_GRAPH_TENSORS]{}; + if (graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION) { + TensorSet listed_input; + TensorSet listed_output; for (uint32_t i = 0; i < graph.num_inputs; ++i) { const uint32_t id = graph_input_id(graph, i); TensorRecord tensor{}; @@ -552,7 +572,7 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - listed_input[id] = true; + listed_input.set(id); } for (uint32_t i = 0; i < graph.num_outputs; ++i) { const uint32_t id = graph_output_id(graph, i); @@ -564,7 +584,7 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - listed_output[id] = true; + listed_output.set(id); } for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; @@ -608,8 +628,9 @@ bool validate_int8_dataflow(const GraphView& graph) { static_cast(id)); return false; } - produced[id] = true; - copy_produced[id] = is_copy; + produced.set(id); + if (is_copy) + copy_produced.set(id); } } for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { @@ -622,6 +643,38 @@ bool validate_int8_dataflow(const GraphView& graph) { return false; } } + TensorSet paired_states; + TensorSet paired_updates; + for (uint32_t k = 0; k < graph.num_states; ++k) { + uint32_t state_id = 0; + uint32_t update_id = 0; + TensorRecord state{}; + TensorRecord update{}; + graph_state_pair(graph, k, state_id, update_id); + read_tensor(graph, state_id, state); + read_tensor(graph, update_id, update); + const uint8_t not_an_update = TENSOR_FLAG_STATE | TENSOR_FLAG_CONSTANT | + TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT; + if ((state.flags & TENSOR_FLAG_STATE) == 0 || + (update.flags & not_an_update) != 0 || !produced[update_id] || + paired_states[state_id] || paired_updates[update_id] || + !same_shape(state, update) || !same_quantization(state, update)) { + std::printf( + "ERROR: state pair %u is invalid\n", static_cast(k)); + return false; + } + paired_states.set(state_id); + paired_updates.set(update_id); + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_STATE) != 0 && !paired_states[i]) { + std::printf( + "ERROR: state tensor %u has no update\n", static_cast(i)); + return false; + } + } return true; } @@ -636,8 +689,8 @@ int32_t read_le_i32(const uint8_t* p) { } bool is_int8_graph_version(uint32_t version) { - return version == GRAPH_ESP_NN_INT8_VERSION || - version == GRAPH_ESP_NN_INT8_IO_VERSION; + return version >= GRAPH_ESP_NN_INT8_VERSION && + version <= GRAPH_ESP_NN_INT8_STATE_VERSION; } namespace { @@ -689,6 +742,18 @@ uint32_t graph_input_id(const GraphView& graph, uint32_t index) { return flagged_tensor(graph, TENSOR_FLAG_GRAPH_INPUT, index); } +bool graph_state_pair( + const GraphView& graph, + uint32_t index, + uint32_t& state_id, + uint32_t& update_id) { + if (graph.state_ids == nullptr || index >= graph.num_states) + return false; + state_id = read_u32(graph.state_ids + 8 * index); + update_id = read_u32(graph.state_ids + 8 * index + 4); + return true; +} + uint32_t graph_output_id(const GraphView& graph, uint32_t index) { if (graph.io_ids != nullptr) { return index < graph.num_outputs @@ -852,7 +917,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.data = data; graph.size = size; graph.tensor_table_offset = GRAPH_HEADER_SIZE; - if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION) { if (!range_valid(GRAPH_HEADER_SIZE, GRAPH_IO_TABLE_HEADER_SIZE, size)) return false; graph.num_inputs = read_u16(data + GRAPH_HEADER_SIZE); @@ -876,6 +941,40 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.tensor_table_offset = GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE + io_bytes; } + if (graph.header.version >= GRAPH_ESP_NN_INT8_STATE_VERSION) { + const size_t at = graph.tensor_table_offset; + if (!range_valid(at, GRAPH_STATE_TABLE_HEADER_SIZE, size)) + return false; + graph.num_states = read_u16(data + at); + const size_t state_bytes = 8U * static_cast(graph.num_states); + if (graph.num_states > MAX_GRAPH_STATES || read_u16(data + at + 2) != 0 || + !range_valid(at + GRAPH_STATE_TABLE_HEADER_SIZE, state_bytes, size)) { + std::printf("ERROR: graph state table is invalid\n"); + return false; + } + graph.state_ids = data + at + GRAPH_STATE_TABLE_HEADER_SIZE; + for (uint32_t i = 0; i < 2U * graph.num_states; ++i) { + if (read_u32(graph.state_ids + 4 * i) >= graph.header.num_tensors) + return false; + } + graph.tensor_table_offset = + at + GRAPH_STATE_TABLE_HEADER_SIZE + state_bytes; + } + if (graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION) { + const size_t padded = (graph.tensor_table_offset + 3U) & ~size_t{3}; + if (!range_valid( + graph.tensor_table_offset, + padded - graph.tensor_table_offset, + size)) + return false; + for (size_t i = graph.tensor_table_offset; i < padded; ++i) { + if (data[i] != 0) { + std::printf("ERROR: graph table padding is not zero\n"); + return false; + } + } + graph.tensor_table_offset = padded; + } graph.tensor_record_size = TENSOR_RECORD_SIZE; const size_t tensor_bytes = static_cast(graph.header.num_tensors) * graph.tensor_record_size; @@ -903,7 +1002,8 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { graph.parameter_data = data + graph.parameter_offset; graph.constant_data = data + graph.constant_offset; - const unsigned allowed_flags = 0x0FU; + const unsigned allowed_flags = + graph.header.version >= GRAPH_ESP_NN_INT8_STATE_VERSION ? 0x1FU : 0x0FU; for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { TensorRecord tensor{}; read_tensor(graph, i, tensor); @@ -937,7 +1037,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; } const bool quantized_constant_allowed = - graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION && + graph.header.version >= GRAPH_ESP_NN_INT8_IO_VERSION && tensor.dtype == static_cast(DataType::INT8); if ((is_constant ? ((tensor.dtype != static_cast(DataType::INT8) && @@ -953,6 +1053,17 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; } const size_t data_size = tensor_data_size(tensor); + const bool is_state = (tensor.flags & TENSOR_FLAG_STATE) != 0; + if (is_state && + (is_constant || + (tensor.flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != 0 || + !range_valid( + tensor.constant_offset, data_size, graph.header.constant_size))) { + std::printf( + "ERROR: state tensor %u is invalid\n", static_cast(i)); + return false; + } const bool misaligned_int32 = is_constant && tensor.dtype == static_cast(DataType::INT32) && (tensor.constant_offset % sizeof(int32_t)) != 0; @@ -962,7 +1073,7 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { (is_constant && !range_valid( tensor.constant_offset, data_size, graph.header.constant_size)) || - (!is_constant && tensor.constant_offset != 0)) { + (!is_constant && !is_state && tensor.constant_offset != 0)) { std::printf( "ERROR: tensor %u constant data range or shape is invalid\n", static_cast(i)); @@ -974,6 +1085,8 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { for (uint32_t i = 0; i < graph.header.num_operations; ++i) { OperationRecord op{}; read_operation(graph, i, op); + if (graph.header.version >= GRAPH_ESP_NN_INT8_ALIGNED_VERSION) + expected_parameter_offset = (expected_parameter_offset + 3U) & ~size_t{3}; if (op.op_id != (i & 0xFFU) || !valid_opcode(op.opcode, graph.header.version) || op.num_inputs == 0 || op.num_inputs > 6 || op.num_outputs == 0 || op.num_outputs > 2 || diff --git a/backends/espressif/lowering.py b/backends/espressif/lowering.py index 9998b993a6d..7e0c30b2015 100644 --- a/backends/espressif/lowering.py +++ b/backends/espressif/lowering.py @@ -51,7 +51,11 @@ def _operator_library_report(edge: EdgeProgramManager) -> dict: def to_espnn_edge( - program, allow_cpu_fallback: bool = True, *, use_operator_library: bool = False + program, + allow_cpu_fallback: bool = True, + *, + use_operator_library: bool = False, + delegate_mutable_buffers: bool = False, ): """ Lower a PT2E-quantized torch.export program for the ESP32 ExecuTorch @@ -63,6 +67,9 @@ def to_espnn_edge( operators and executable Q/DQ. Its report counts esp_nn_ops and lists remaining cpu_ops by method, excluding Q/DQ and structural getitem nodes. + + With delegate_mutable_buffers=True, the delegates own the quantized mutable + buffers (streaming state) they read and update; see EspNNPartitioner. """ if use_operator_library: edge = to_edge_transform_and_lower( @@ -81,7 +88,10 @@ def to_espnn_edge( ) return edge, report - partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback) + partitioner = EspNNPartitioner( + allow_cpu_fallback=allow_cpu_fallback, + delegate_mutable_buffers=delegate_mutable_buffers, + ) edge = to_edge_transform_and_lower( program, transform_passes=[DuplicateDequantNodePass()], diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index c1aa0de730e..657043608bf 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -8,6 +8,8 @@ from __future__ import annotations +import logging + from typing import Callable, Optional import torch @@ -31,6 +33,7 @@ INT8_PASSTHROUGH_MARKERS, lower_int8_node, LUT_KINDS, + MAX_GRAPH_STATES, TensorRegistry, ) from executorch.exir.backend.compile_spec_schema import CompileSpec @@ -39,7 +42,7 @@ Partitioner, PartitionResult, ) -from executorch.exir.backend.utils import tag_constant_data +from executorch.exir.backend.utils import tag_constant_data, tag_mutated_buffer from torch.export import ExportedProgram from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner from torch.fx.passes.operator_support import OperatorSupportBase @@ -276,6 +279,136 @@ def _has_compute(nodes: set) -> bool: return any(not _is_passthrough(node) for node in nodes) +def _qparams(node: torch.fx.Node, op: str) -> Optional[tuple]: + if node.op != "call_function" or not _operator_name(node.target).startswith( + f"quantized_decomposed.{op}." + ): + return None + return tuple(node.args[1:]) + + +def _int8_state_compatible(buffer: torch.fx.Node, writer: torch.fx.Node) -> bool: + """ + Whether the delegate can keep a mutable buffer as int8: a float32 buffer + read only through quantize ops whose new value is dequantized, or an int8 + buffer read only through dequantize ops whose new value is quantized, all + with the same parameters. + """ + reader, producer = { + torch.float32: ("quantize_per_tensor", "dequantize_per_tensor"), + torch.int8: ("dequantize_per_tensor", "quantize_per_tensor"), + }.get(buffer.meta["val"].dtype, (None, None)) + if reader is None or not buffer.users: + return False + reads = {_qparams(user, reader) for user in buffer.users} + return len(reads) == 1 and None not in reads and _qparams(writer, producer) in reads + + +def _keeps_runtime_io( + exported_program: ExportedProgram, kept, released, writers +) -> bool: + """ + Whether the partition that would own the kept buffers still has a runtime + input and a runtime output, which its XTS3 graph needs. Constants do not + count, since a shared one is duplicated into the partition, and the + released buffers become runtime inputs. + """ + signature = exported_program.graph_signature + constants = { + *signature.inputs_to_parameters, + *signature.inputs_to_lifted_tensor_constants, + *( + name + for name, buffer in signature.inputs_to_buffers.items() + if buffer not in writers + ), + } + tag = kept[0][0].meta["delegation_tag"] + released_nodes = {node for node, _ in released} + nodes = { + node + for node in exported_program.graph.nodes + if node.meta.get("delegation_tag") == tag and node not in released_nodes + } + updates = {writers[buffer] for _, buffer in kept} + + def runtime(node): + return node.op != "get_attr" and node.name not in constants + + has_input = any( + source not in nodes and runtime(source) + for node in nodes + for source in node.all_input_nodes + ) + has_output = any( + runtime(node) + and user not in nodes + and not (user.op == "output" and node in updates) + for node in nodes + for user in node.users + ) + return has_input and has_output + + +def _tag_quantized_mutable_buffers(exported_program: ExportedProgram) -> list[str]: + """ + Give the mutable buffers to the partition that holds all their accesses, + unless the delegate could not keep one as int8, they would end up in + several partitions or leave it without a runtime input or output, or there + are more than MAX_GRAPH_STATES of them. Returns the delegated buffers' + names. + """ + tag_mutated_buffer(exported_program) + signature = exported_program.graph_signature + nodes = {node.name: node for node in exported_program.graph.nodes} + writers = { + buffer: nodes[output] for output, buffer in signature.buffers_to_mutate.items() + } + delegated = [] + for node in exported_program.graph.find_nodes(op="placeholder"): + buffer = signature.inputs_to_buffers.get(node.name) + if buffer not in writers or "delegation_tag" not in node.meta: + continue + if _int8_state_compatible(node, writers[buffer]): + delegated.append((node, buffer)) + continue + del node.meta["delegation_tag"] + logging.warning( + f"Mutable buffer {buffer} stays in the program: the ESP-NN delegate " + "owns a buffer only when it is read and written with the same " + "int8 quantization." + ) + # A reset reaches each delegate at the start of its own execute(), so + # state split across partitions could be reset between them. + if len({node.meta["delegation_tag"] for node, _ in delegated}) > 1: + logging.warning( + "Mutable buffers stay in the program: the ESP-NN delegate owns a " + "method's buffers only when one partition holds all of them." + ) + left = delegated + elif delegated and not _keeps_runtime_io( + exported_program, + delegated[:MAX_GRAPH_STATES], + delegated[MAX_GRAPH_STATES:], + writers, + ): + logging.warning( + "Mutable buffers stay in the program: owning them would leave the " + "ESP-NN partition without a runtime input or output." + ) + left = delegated + else: + left = delegated[MAX_GRAPH_STATES:] + for _, buffer in left: + logging.warning( + f"Mutable buffer {buffer} stays in the program: an ESP-NN " + f"delegate owns at most {MAX_GRAPH_STATES} buffers." + ) + for node, _ in left: + del node.meta["delegation_tag"] + return [buffer for _, buffer in delegated[: len(delegated) - len(left)]] + + class EspNNPartitioner(Partitioner): """ Delegates PT2E-quantized regions of a graph to the ESP32 ESP-NN backend. @@ -286,18 +419,29 @@ class EspNNPartitioner(Partitioner): With allow_cpu_fallback=False any operator left outside a delegate is an error, whether it is unsupported or its region could not be delegated; the standalone XTS3 firmware path requires this. + + With delegate_mutable_buffers, the quantized mutable buffers (streaming + state) whose readers and writers all fall in one partition are owned by + that delegate: they leave the program's inputs and outputs, and the + delegate keeps them as int8 between executions. Each buffer must be + float32 and read only through quantize ops, or int8 and read only through + dequantize ops, with the same quantization as its new value. The delegate + takes a method's buffers only when they all fall in one partition, and at + most MAX_GRAPH_STATES of them. Any other buffer stays in the program. """ def __init__( self, compile_specs: Optional[list[CompileSpec]] = None, allow_cpu_fallback: bool = True, + delegate_mutable_buffers: bool = False, ): specs = list(compile_specs or []) if not any(spec.key == INT8_FORMAT_SPEC.key for spec in specs): specs.append(INT8_FORMAT_SPEC) self.delegation_spec = DelegationSpec(EspNNBackend.__name__, specs) self.allow_cpu_fallback = allow_cpu_fallback + self.delegate_mutable_buffers = delegate_mutable_buffers self.report: dict = {} def ops_to_not_decompose( @@ -347,7 +491,11 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: tag = f"espnn_{len(tags)}" for node in nodes: node.meta["delegation_tag"] = tag - tags[tag] = self.delegation_spec + # Preprocessing adds conversion metadata specific to this partition. + tags[tag] = DelegationSpec( + self.delegation_spec.backend_id, + list(self.delegation_spec.compile_specs), + ) delegated = { node for node in graph_module.graph.nodes @@ -374,7 +522,13 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: if not tags and not self.allow_cpu_fallback: raise RuntimeError("No ESP32 int8 partition could be formed") tag_constant_data(exported_program) + delegated_buffers = ( + _tag_quantized_mutable_buffers(exported_program) + if self.delegate_mutable_buffers + else [] + ) self.report = { + "delegated_buffers": delegated_buffers, "partitions": len(tags), "rejected_partitions": rejected, "delegated_ops": sum(1 for node in delegated if not _is_passthrough(node)), diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py index 291ccca80f7..cf35bbbaba6 100644 --- a/backends/espressif/reference/esp_nn_reference.py +++ b/backends/espressif/reference/esp_nn_reference.py @@ -31,6 +31,7 @@ SOFTMAX_OUTPUT_ZERO_POINT, TENSOR_FLAG_GRAPH_INPUT, TENSOR_FLAG_GRAPH_OUTPUT, + TENSOR_FLAG_STATE, TRANSPOSE_CONV_PARAMS, TRANSPOSE_CONV_PARAMS_SIZE, UPSAMPLE_HEADER, @@ -229,6 +230,23 @@ def __init__(self, graph: XTS3Graph): self.outputs = list(graph.outputs) or [ t.tensor_id for t in graph.tensors if t.flags & TENSOR_FLAG_GRAPH_OUTPUT ] + self.states: dict[int, np.ndarray] = {} + self.reset_states() + + def reset_states(self) -> None: + """Restores every delegate-owned state tensor to its initial value.""" + for state_id, _ in self.graph.states: + tensor = self.graph.tensors[state_id] + self.states[state_id] = ( + np.frombuffer( + self.graph.constant_data, + dtype=np.int8, + count=int(np.prod(tensor.shape)), + offset=tensor.data_offset, + ) + .copy() + .reshape(device_shape(tensor)) + ) def constant(self, tensor_id: int) -> np.ndarray: tensor = self.graph.tensors[tensor_id] @@ -252,7 +270,8 @@ def value(self, tensor_id: int, values) -> np.ndarray: raise KeyError(f"Tensor {tensor_id} has not been produced") def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: - values: dict[int, np.ndarray] = {} + """Runs one invoke; delegate-owned state carries over to the next.""" + values: dict[int, np.ndarray] = dict(self.states) for tensor_id in self.inputs: tensor = self.graph.tensors[tensor_id] data = np.asarray(inputs[tensor_id], dtype=np.int8) @@ -263,6 +282,8 @@ def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: values[tensor_id] = data for operation in self.graph.operations: values[operation.outputs[0]] = self._run_operation(operation, values) + for state_id, update_id in self.graph.states: + self.states[state_id] = values[update_id].copy() return values def run_outputs(self, inputs: list[np.ndarray]) -> list[np.ndarray]: @@ -577,19 +598,33 @@ def _run_operation(self, operation, values) -> np.ndarray: # noqa: C901 raise ValueError(f"Unsupported opcode {opcode}") -def plan_activation_arena( - graph: XTS3Graph, alignment: int = 16 +def plan_activation_arena( # noqa: C901 + graph: XTS3Graph, alignment: int = 16, external_io: bool = False ) -> tuple[int, dict[int, int]]: - """Mirror of Int8Interpreter::plan_memory in the firmware.""" + """ + Mirror of Int8Interpreter::plan_memory in the firmware. With external_io + (as EspNNBackend loads graphs), graph outputs that are neither graph inputs + nor read by an operation are not planned. + """ op_count = len(graph.operations) big = 1 << 30 first: dict[int, int] = {} last: dict[int, int] = {} sizes: dict[int, int] = {} + io_flags = TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT + read = { + tensor_id for operation in graph.operations for tensor_id in operation.inputs + } for tensor in graph.tensors: - if tensor.is_constant: + if tensor.is_constant or ( + external_io + and (tensor.flags & io_flags) == TENSOR_FLAG_GRAPH_OUTPUT + and tensor.tensor_id not in read + ): continue - first[tensor.tensor_id] = -1 if tensor.flags & TENSOR_FLAG_GRAPH_INPUT else big + first[tensor.tensor_id] = ( + -1 if tensor.flags & (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_STATE) else big + ) last[tensor.tensor_id] = ( op_count if tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT else -1 ) @@ -600,10 +635,24 @@ def plan_activation_arena( if tensor_id in last: last[tensor_id] = max(last[tensor_id], index) for tensor_id in operation.outputs: - first[tensor_id] = min(first[tensor_id], index) - last[tensor_id] = max(last[tensor_id], index) + if tensor_id in first: + first[tensor_id] = min(first[tensor_id], index) + last[tensor_id] = max(last[tensor_id], index) + # A state keeps its slot for the whole graph. Its update shares that slot + # when no operation reads the state at or after the update's first write. + in_place = {} + for state_id, update_id in graph.states: + if last[state_id] < first[update_id]: + in_place[update_id] = state_id + else: + last[update_id] = op_count + last[state_id] = op_count order = sorted( - (tensor_id for tensor_id in sizes if first[tensor_id] != big), + ( + tensor_id + for tensor_id in sizes + if first[tensor_id] != big and tensor_id not in in_place + ), key=lambda tensor_id: (-sizes[tensor_id], tensor_id), ) offsets: dict[int, int] = {} @@ -626,6 +675,8 @@ def plan_activation_arena( moved = True offsets[tensor_id] = candidate arena = max(arena, candidate + sizes[tensor_id]) + for update_id, state_id in in_place.items(): + offsets[update_id] = offsets[state_id] return arena, offsets diff --git a/backends/espressif/reference/program_emulator.py b/backends/espressif/reference/program_emulator.py index 98754ab76b2..8087db24442 100644 --- a/backends/espressif/reference/program_emulator.py +++ b/backends/espressif/reference/program_emulator.py @@ -26,7 +26,13 @@ def delegate_output_dtypes(lowered_module) -> list[torch.dtype]: values = output.args[0] if not isinstance(values, (list, tuple)): values = (values,) - return [torch.float32 if _is_dequantize(value) else torch.int8 for value in values] + # New values of delegate-owned buffers stay in the delegate. + state_updates = set(program.graph_signature.buffers_to_mutate) + return [ + torch.float32 if _is_dequantize(value) else torch.int8 + for value in values + if value.name not in state_updates + ] class DelegateEmulator: @@ -37,6 +43,9 @@ def __init__(self, lowered_module): self.interpreter = reference.ReferenceInterpreter(self.graph) self.output_dtypes = delegate_output_dtypes(lowered_module) + def reset_states(self) -> None: + self.interpreter.reset_states() + def __call__(self, *inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: device_inputs = [] for tensor_id, value in zip(self.interpreter.inputs, inputs): @@ -69,7 +78,8 @@ class ProgramEmulator(torch.fx.Interpreter): """ Host model of an ExecuTorch program lowered with EspNNPartitioner: delegates run on the bit-exact ESP-NN reference, every other operator - runs in PyTorch (the device uses the ExecuTorch portable kernels). + runs in PyTorch (the device uses the ExecuTorch portable kernels). State + that a delegate owns persists from one call to the next, as on the device. """ def __init__(self, exported_program): @@ -94,5 +104,10 @@ def call_function(self, target, args, kwargs): def __call__(self, *inputs): return self.run(*inputs) + def reset_states(self) -> None: + """Restores the state that delegates own, as the runtime's reset does.""" + for delegate in self.delegates.values(): + delegate.reset_states() + __all__ = ["DelegateEmulator", "ProgramEmulator", "delegate_output_dtypes"] diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index 7ec71b95a1d..99069320356 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -19,10 +19,17 @@ ) MAGIC = b"XTS3" -XTS3_VERSION = 6 -XTS3_VERSIONS = (5, 6) +# Version 7 is version 6 with the IO table padded to 4 bytes, so that the +# runtime can use the tensor records and parameter blocks in place. Version 8 +# adds a state table after the IO table, and is written only for graphs that +# own mutable buffers. +XTS3_VERSION = 7 +XTS3_STATE_VERSION = 8 +XTS3_VERSIONS = (5, 6, 7, 8) IO_TABLE_HEADER = " int: TENSOR_FLAG_QUANTIZED = 1 << 1 TENSOR_FLAG_GRAPH_INPUT = 1 << 2 TENSOR_FLAG_GRAPH_OUTPUT = 1 << 3 +# Delegate-owned int8 state (a mutable buffer of the partition). Its initial +# value is in the constant region at data_offset, but it is not a constant: the +# state table pairs it with the tensor that replaces it after every invoke. +TENSOR_FLAG_STATE = 1 << 4 @dataclass @@ -149,6 +164,20 @@ class XTS3Graph: tensor_names: dict[int, str] = field(default_factory=dict) inputs: list[int] = field(default_factory=list) outputs: list[int] = field(default_factory=list) + # (state tensor, update tensor) pairs; version 8 only + states: list[tuple[int, int]] = field(default_factory=list) + + +def _mutated_buffer_placeholders(exported_program) -> set[str]: + signature = getattr(exported_program, "graph_signature", None) + if signature is None: + return set() + mutated = set(signature.buffers_to_mutate.values()) + return { + name + for name, buffer in signature.inputs_to_buffers.items() + if buffer in mutated + } class ExportedTensorResolver: @@ -167,6 +196,7 @@ def __init__(self, exported_program): ) or {} ) + self.mutated_buffers = _mutated_buffer_placeholders(exported_program) def _lookup_mapping(self, mapping, name): if mapping is None: @@ -182,6 +212,8 @@ def _lookup_mapping(self, mapping, name): return None def resolve(self, node): # noqa: C901 + if node is not None and node.name in self.mutated_buffers: + return None if node is not None and node.op == "call_function": target = str(node.target) if "dequantize_per_channel" in target: @@ -1753,7 +1785,7 @@ def _user_input_placeholders(exported_program): return [node for node in placeholders if node.name in user_inputs] -def _mark_int8_graph_io( +def _mark_int8_graph_io( # noqa: C901 exported_program, registry: TensorRegistry ) -> tuple[list[int], list[int]]: graph_module = exported_program.graph_module @@ -1774,6 +1806,8 @@ def _mark_int8_graph_io( raise ValueError("Int8 graph has no quantized runtime input") outputs: list[int] = [] + signature = getattr(exported_program, "graph_signature", None) + state_updates = set(signature.buffers_to_mutate) if signature else set() for node in graph_module.graph.nodes: if node.op != "output": continue @@ -1781,6 +1815,8 @@ def _mark_int8_graph_io( if not isinstance(values, (list, tuple)): values = (values,) for value in values: + if getattr(value, "name", None) in state_updates: + continue source = _quantized_source(value) if source is None or source not in registry.node_to_id: raise ValueError( @@ -1795,6 +1831,124 @@ def _mark_int8_graph_io( return inputs, outputs +def _mark_int8_state( + exported_program, registry: TensorRegistry, graph_outputs: list[int] +) -> list[tuple[int, int]]: + """ + Pair each mutable buffer of the partition with the int8 tensor holding its + new value. The buffer's tensor becomes TENSOR_FLAG_STATE, with its + quantized initial value in the constant region. A state cannot be a graph + output, so an old value that the partition returns is returned as a copy. + """ + signature = exported_program.graph_signature + if not signature.buffers_to_mutate: + return [] + nodes = {node.name: node for node in exported_program.graph_module.graph.nodes} + writers = { + buffer: nodes[output] for output, buffer in signature.buffers_to_mutate.items() + } + states: list[tuple[int, int]] = [] + for node in exported_program.graph_module.graph.nodes: + buffer = signature.inputs_to_buffers.get(node.name) + if node.op != "placeholder" or buffer not in writers: + continue + if node not in registry.node_to_id: + raise ValueError(f"Mutable buffer {buffer} is not read as an int8 tensor") + state = registry.tensors[registry.node_to_id[node]] + source = _quantized_source(writers[buffer]) + if source is None or source not in registry.node_to_id: + raise ValueError( + f"The new value of mutable buffer {buffer} is not an int8 tensor" + ) + update = registry.tensors[registry.node_to_id[source]] + if ( + tuple(update.shape) != tuple(state.shape) + or update.quant_scale != state.quant_scale + or update.quant_zero_point != state.quant_zero_point + ): + raise ValueError( + f"Mutable buffer {buffer} and its new value have different " + "shapes or quantization parameters" + ) + initial = exported_program.state_dict.get( + buffer, exported_program.constants.get(buffer) + ) + if initial is None: + raise ValueError(f"Mutable buffer {buffer} has no initial value") + if initial.dtype == torch.float32: + initial = torch.ops.quantized_decomposed.quantize_per_tensor( + initial, + state.quant_scale, + state.quant_zero_point, + -128, + 127, + torch.int8, + ) + if initial.dtype != torch.int8: + raise ValueError(f"Mutable buffer {buffer} must be float32 or int8") + raw = _int8_device_bytes( + initial.detach().cpu().contiguous(), tuple(state.shape) + ) + registry._align_constant_data() + state.data_offset = len(registry.constant_data) + state.data_size = len(raw) + registry.constant_data.extend(raw) + state.flags |= TENSOR_FLAG_STATE + states.append((state.tensor_id, update.tensor_id)) + if state.flags & TENSOR_FLAG_GRAPH_OUTPUT: + old_value = _copy_of(state, registry) + state.flags &= ~TENSOR_FLAG_GRAPH_OUTPUT + old_value.flags |= TENSOR_FLAG_GRAPH_OUTPUT + graph_outputs[:] = [ + old_value.tensor_id if output == state.tensor_id else output + for output in graph_outputs + ] + return _separate_state_updates(states, registry) + + +def _copy_of(tensor: XTS3Tensor, registry: TensorRegistry) -> XTS3Tensor: + """A new int8 tensor that a COPY after every other operation fills.""" + from .int8_ops import copy_operation, copy_segment + + copy = XTS3Tensor( + tensor_id=len(registry.tensors), + shape=tuple(tensor.shape), + dtype=XTS3DType.INT8, + quant_scale=tensor.quant_scale, + quant_zero_point=tensor.quant_zero_point, + ) + registry.tensors.append(copy) + segment = copy_segment(0, [prod(tensor.shape)], [1], 0, [1], 0, tensor, copy) + registry.extra_operations.append( + copy_operation([tensor.tensor_id], copy.tensor_id, [segment]) + ) + return copy + + +def _separate_state_updates( + states: list[tuple[int, int]], registry: TensorRegistry +) -> list[tuple[int, int]]: + """ + Give a copy of its own to each update the runtime cannot write into a + state's slot: a graph input or output, a constant, a state, or an update + shared by several states. The copies run after every other operation. + """ + shared = ( + TENSOR_FLAG_GRAPH_INPUT + | TENSOR_FLAG_GRAPH_OUTPUT + | TENSOR_FLAG_CONSTANT + | TENSOR_FLAG_STATE + ) + used: set[int] = set() + for index, (state_id, update_id) in enumerate(states): + update = registry.tensors[update_id] + if update.flags & shared or update_id in used: + update_id = _copy_of(update, registry).tensor_id + states[index] = (state_id, update_id) + used.add(update_id) + return states + + def build_graph(exported_program) -> XTS3Graph: """Lower a PT2E-quantized ExportedProgram (or delegate partition) to an XTS3 int8 graph.""" graph_module = exported_program.graph_module @@ -1817,11 +1971,16 @@ def build_graph(exported_program) -> XTS3Graph: lowered.op_id = len(operations) operations.append(lowered) graph_inputs, graph_outputs = _mark_int8_graph_io(exported_program, registry) + states = _mark_int8_state(exported_program, registry, graph_outputs) + for extra in registry.extra_operations: + extra.op_id = len(operations) + operations.append(extra) return XTS3Graph( tensors=registry.tensors, operations=operations, constant_data=bytes(registry.constant_data), - version=XTS3_VERSION, + version=XTS3_STATE_VERSION if states else XTS3_VERSION, + states=states, tensor_names={ tensor_id: node.name for node, tensor_id in registry.node_to_id.items() }, @@ -1868,7 +2027,7 @@ def _serialize_operation(operation: XTS3Operation, params_offset: int) -> bytes: ) -def check_runtime_limits(graph: XTS3Graph) -> None: +def check_runtime_limits(graph: XTS3Graph) -> None: # noqa: C901 """ Raise ValueError for graphs the device runtime would reject at init(): the same limits espnn_runtime/src/xts3_graph.cpp enforces. @@ -1880,6 +2039,13 @@ def check_runtime_limits(graph: XTS3Graph) -> None: f"Partition has {len(graph.inputs)} inputs and {len(graph.outputs)} " f"outputs; the ESP-NN runtime supports at most {MAX_GRAPH_IO} of each" ) + if graph.states and graph.version < XTS3_STATE_VERSION: + raise ValueError("Delegate-owned state needs XTS3 version 8") + if len(graph.states) > MAX_GRAPH_STATES: + raise ValueError( + f"Partition owns {len(graph.states)} mutable buffers; the ESP-NN " + f"runtime supports at most {MAX_GRAPH_STATES}" + ) if len(graph.tensors) > MAX_GRAPH_TENSORS: raise ValueError( f"Partition has {len(graph.tensors)} tensors; the ESP-NN runtime " @@ -1906,6 +2072,24 @@ def check_runtime_limits(graph: XTS3Graph) -> None: f"Upsample output {out_h}x{out_w} exceeds " f"{UPSAMPLE_MAX_EXTENT} in one dimension" ) + if operation.opcode in ( + XTS3OpCode.CONV2D, + XTS3OpCode.DEPTHWISE_CONV2D, + XTS3OpCode.MAX_POOL2D, + XTS3OpCode.AVG_POOL2D, + ): + source = graph.tensors[operation.inputs[0]].shape + target = graph.tensors[operation.outputs[0]].shape + extents = [*source[1:], *target[1:]] + elif operation.opcode == XTS3OpCode.LINEAR: + extents = list(graph.tensors[operation.inputs[1]].shape) + else: + extents = [] + if extents and max(extents) > KERNEL_MAX_EXTENT: + raise ValueError( + f"{XTS3OpCode(operation.opcode).name} has a dimension above " + f"{KERNEL_MAX_EXTENT}, which ESP-NN takes as uint16_t" + ) for tensor in graph.tensors: if ( tensor.flags & TENSOR_FLAG_CONSTANT @@ -1925,7 +2109,7 @@ def check_dataflow(graph: XTS3Graph) -> None: ready.update( tensor.tensor_id for tensor in graph.tensors - if tensor.flags & TENSOR_FLAG_CONSTANT + if tensor.flags & (TENSOR_FLAG_CONSTANT | TENSOR_FLAG_STATE) ) for index, operation in enumerate(graph.operations): for tensor_id in operation.inputs: @@ -1938,6 +2122,9 @@ def check_dataflow(graph: XTS3Graph) -> None: for tensor_id in graph.outputs: if tensor_id not in ready: raise ValueError(f"Graph output {tensor_id} is never written") + for state_id, update_id in graph.states: + if update_id not in ready: + raise ValueError(f"The new value of state {state_id} is never written") def serialize_graph(graph: XTS3Graph) -> bytes: @@ -1948,6 +2135,8 @@ def serialize_graph(graph: XTS3Graph) -> bytes: parameter_data = bytearray() operation_data = bytearray() for operation in graph.operations: + if graph.version >= 7: + parameter_data += b"\x00" * (-len(parameter_data) % 4) operation_data += _serialize_operation(operation, len(parameter_data)) parameter_data += operation.params io_table = b"" @@ -1959,6 +2148,12 @@ def serialize_graph(graph: XTS3Graph) -> bytes: ) + struct.pack( f"<{len(graph.inputs) + len(graph.outputs)}I", *graph.inputs, *graph.outputs ) + if graph.version >= 8: + io_table += struct.pack(STATE_TABLE_HEADER, len(graph.states), 0) + for state_id, update_id in graph.states: + io_table += struct.pack("<2I", state_id, update_id) + if graph.version >= 7: + io_table += b"\x00" * ((-(HEADER_SIZE + len(io_table))) % 4) prefix = ( HEADER_SIZE + len(io_table) @@ -1987,7 +2182,7 @@ def serialize_graph(graph: XTS3Graph) -> bytes: def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 - """Parse and bounds-check an XTS3 v5 or v6 int8 graph.""" + """Parse and bounds-check an XTS3 v5 to v8 int8 graph.""" if len(data) < HEADER_SIZE: raise ValueError("Graph blob is smaller than the header") magic, version, tensor_count, operation_count, parameter_size, constant_size = ( @@ -2012,6 +2207,26 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 ids = struct.unpack_from(f"<{input_count + output_count}I", data, offset) graph_inputs, graph_outputs = list(ids[:input_count]), list(ids[input_count:]) offset += 4 * (input_count + output_count) + states: list[tuple[int, int]] = [] + if version >= 8: + if offset + STATE_TABLE_HEADER_SIZE > len(data): + raise ValueError("Graph state table exceeds blob") + state_count, reserved = struct.unpack_from(STATE_TABLE_HEADER, data, offset) + if reserved: + raise ValueError("Graph state table reserved field is not zero") + offset += STATE_TABLE_HEADER_SIZE + if offset + 8 * state_count > len(data): + raise ValueError("Graph state table exceeds blob") + pairs = struct.unpack_from(f"<{2 * state_count}I", data, offset) + if any(tensor_id >= tensor_count for tensor_id in pairs): + raise ValueError("Graph state table names an unknown tensor") + states = list(zip(pairs[0::2], pairs[1::2])) + offset += 8 * state_count + if version >= 7: + padding = (-offset) % 4 + if data[offset : offset + padding] != b"\x00" * padding: + raise ValueError("Graph table padding is not zero") + offset += padding if offset + tensor_count * TENSOR_SIZE > len(data): raise ValueError("Graph tensor region exceeds blob") @@ -2103,11 +2318,14 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 version=version, inputs=graph_inputs, outputs=graph_outputs, + states=states, ) __all__ = [ "MAGIC", + "TENSOR_FLAG_STATE", + "XTS3_STATE_VERSION", "check_runtime_limits", "XTS3_VERSION", "XTS3_VERSIONS", diff --git a/backends/espressif/test/host/CMakeLists.txt b/backends/espressif/test/host/CMakeLists.txt new file mode 100644 index 00000000000..3f95c38708b --- /dev/null +++ b/backends/espressif/test/host/CMakeLists.txt @@ -0,0 +1,69 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Host build of the ESP-NN delegate, espnn_host_runner and +# espnn_host_interpreter, for backends/espressif/test/test_host_runtime.py and +# test_host_interpreter.py (see "On the host" in backends/espressif/README.md). +# -DESPNN_HOST_SANITIZE=ON builds everything with AddressSanitizer and +# UndefinedBehaviorSanitizer. + +cmake_minimum_required(VERSION 3.24) +project(espnn_host_runner CXX C) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +get_filename_component( + EXECUTORCH_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../.." ABSOLUTE +) + +option(ESPNN_HOST_SANITIZE "Build with ASan and UBSan" OFF) +if(ESPNN_HOST_SANITIZE) + add_compile_options( + -fsanitize=address,undefined -fno-sanitize-recover=all + -fno-omit-frame-pointer + ) + add_link_options(-fsanitize=address,undefined) +endif() + +set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON) +set(EXECUTORCH_BUILD_EXECUTOR_RUNNER OFF) +set(EXECUTORCH_BUILD_PYBIND OFF) +set(EXECUTORCH_BUILD_DEVTOOLS OFF) +set(EXECUTORCH_BUILD_XNNPACK OFF) +set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF) +set(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON) +set(EXECUTORCH_BUILD_PTHREADPOOL OFF) +set(EXECUTORCH_BUILD_CPUINFO OFF) +set(EXECUTORCH_BUILD_TESTS OFF) +add_subdirectory(${EXECUTORCH_ROOT} executorch EXCLUDE_FROM_ALL) +add_subdirectory( + ${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime espnn_runtime +) +# ESP-IDF's default warnings, as errors. +target_compile_options( + espnn_backend PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-sign-compare + -Werror +) +if(ESPNN_HOST_SANITIZE) + # ESP-NN's C kernels shift negative values left, as TFLite's reference kernels + # do; GCC and Clang define the result. + target_compile_options(esp_nn_host PRIVATE -fno-sanitize=shift-base) +endif() + +# The portable and quantized kernels run what a program leaves on the CPU. +executorch_target_link_options_shared_lib(portable_ops_lib) +executorch_target_link_options_shared_lib(quantized_ops_lib) +add_executable(espnn_host_runner espnn_host_runner.cpp) +target_include_directories(espnn_host_runner PRIVATE "${EXECUTORCH_ROOT}/..") +target_link_libraries( + espnn_host_runner PRIVATE executorch_core extension_data_loader espnn_backend + portable_ops_lib quantized_ops_lib +) + +add_executable(espnn_host_interpreter espnn_host_interpreter.cpp) +target_link_libraries(espnn_host_interpreter PRIVATE espnn_backend) diff --git a/backends/espressif/test/host/espnn_host_interpreter.cpp b/backends/espressif/test/host/espnn_host_interpreter.cpp new file mode 100644 index 00000000000..d0a2851b7bd --- /dev/null +++ b/backends/espressif/test/host/espnn_host_interpreter.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Runs one XTS3 graph on Int8Interpreter directly, for + * backends/espressif/test/test_host_interpreter.py: + * + * espnn_host_interpreter GRAPH OFFSET [--constants-to-psram] [--external-io] + * INPUT... -- OUTPUT... + * + * The graph is placed OFFSET bytes past a 16-byte boundary, so that a test can + * load it from addresses with any alignment. Inputs and outputs are raw + * device-layout (NHWC) int8 files. --constants-to-psram sets + * Int8RuntimeOptions::copy_constants_to_psram. When the interpreter copies the + * graph, the runner frees its own copy, as EspNNBackend does. With + * --external-io, outputs are bound to 16-byte aligned runner buffers prefilled + * with 0x5a, after checking that a misaligned buffer is refused. + */ + +#include +#include +#include +#include +#include +#include + +#include + +using executorch::backends::espressif::Int8Interpreter; +using executorch::backends::espressif::Int8RuntimeOptions; +using executorch::backends::espressif::Int8TensorInfo; + +namespace { + +std::vector read_file(const char* path) { + std::vector data; + std::FILE* file = std::fopen(path, "rb"); + if (file == nullptr) + return data; + uint8_t chunk[4096]; + size_t read = 0; + while ((read = std::fread(chunk, 1, sizeof(chunk), file)) > 0) + data.insert(data.end(), chunk, chunk + read); + std::fclose(file); + return data; +} + +int fail(const std::string& what) { + std::fprintf(stderr, "espnn_host_interpreter: %s\n", what.c_str()); + return 1; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 4) + return fail("usage: GRAPH OFFSET INPUT... -- OUTPUT..."); + const std::vector graph = read_file(argv[1]); + const size_t offset = std::strtoul(argv[2], nullptr, 10); + std::vector inputs; + std::vector outputs; + bool past_separator = false; + Int8RuntimeOptions options{}; + for (int a = 3; a < argc; ++a) { + if (std::strcmp(argv[a], "--") == 0) + past_separator = true; + else if (std::strcmp(argv[a], "--constants-to-psram") == 0) + options.copy_constants_to_psram = true; + else if (std::strcmp(argv[a], "--external-io") == 0) + options.external_io = true; + else + (past_separator ? outputs : inputs).push_back(argv[a]); + } + + // A 16-byte aligned buffer, holding the graph at the requested offset. + std::vector storage((graph.size() + offset) / 8 + 3); + uint8_t* base = reinterpret_cast(storage.data()); + base += (16 - reinterpret_cast(base) % 16) % 16; + std::memcpy(base + offset, graph.data(), graph.size()); + + Int8Interpreter interpreter; + if (!interpreter.load(base + offset, graph.size(), options)) + return fail("load failed"); + if (interpreter.constants_copied()) { + storage.clear(); + storage.shrink_to_fit(); + } + if (inputs.size() != interpreter.input_count() || + outputs.size() != interpreter.output_count()) + return fail("wrong number of inputs or outputs"); + std::vector> bound(outputs.size()); + for (size_t o = 0; options.external_io && o < bound.size(); ++o) { + Int8TensorInfo info{}; + interpreter.output_info(o, info); + bound[o].resize(info.bytes / 8 + 3); + int8_t* buffer = reinterpret_cast(bound[o].data()); + buffer += (16 - reinterpret_cast(buffer) % 16) % 16; + std::memset(buffer, 0x5a, info.bytes); + if (interpreter.bind_output(o, buffer + 1) || + !interpreter.bind_output(o, buffer)) + return fail("bound a misaligned output or refused an aligned one"); + } + for (size_t i = 0; i < inputs.size(); ++i) { + Int8TensorInfo info{}; + interpreter.input_info(i, info); + const std::vector data = read_file(inputs[i]); + if (data.size() != info.bytes) + return fail(std::string("wrong input size: ") + inputs[i]); + std::memcpy(interpreter.input_data(i), data.data(), data.size()); + } + if (!interpreter.invoke()) + return fail("invoke failed"); + for (size_t o = 0; o < outputs.size(); ++o) { + Int8TensorInfo info{}; + interpreter.output_info(o, info); + std::FILE* file = std::fopen(outputs[o], "wb"); + if (file == nullptr || + std::fwrite(interpreter.output_data(o), 1, info.bytes, file) != + info.bytes) + return fail(std::string("cannot write ") + outputs[o]); + std::fclose(file); + } +#ifdef ESPNN_HOST_PSRAM + const int psram = 1; +#else + const int psram = 0; +#endif + std::printf( + "STATS arena=%zu arena_internal=%d parameters=%zu blob_copied=%d " + "psram=%d\n", + interpreter.arena_bytes(), + interpreter.arena_in_internal_ram() ? 1 : 0, + interpreter.parameter_bytes(), + interpreter.constants_copied() ? 1 : 0, + psram); + return 0; +} diff --git a/backends/espressif/test/host/espnn_host_runner.cpp b/backends/espressif/test/host/espnn_host_runner.cpp new file mode 100644 index 00000000000..7aa851bcc96 --- /dev/null +++ b/backends/espressif/test/host/espnn_host_runner.cpp @@ -0,0 +1,363 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Runs an ExecuTorch program on the host build of the ESP-NN delegate, for + * backends/espressif/test/test_host_runtime.py. + * + * espnn_host_runner PROGRAM IO_DIR STEPS [--instances N] [--share-io] + * [--alias INPUT:OUTPUT]... + * [--share-output FIRST:SECOND]... + * [--misalign] [--reset-before STEP[:GROUP]]... + * [--reset-groups] + * [--unbound-output OUTPUT] + * + * Step s of instance k reads input i from IO_DIR/in__.bin and writes + * output o to IO_DIR/out___.bin: the raw bytes of the tensor in its + * dim order. With --instances, the method is loaded N times and each step runs + * on every instance in turn. Unplanned outputs are bound to runner buffers. + * With --share-io, inputs must be unplanned and are bound once through + * set_input. --alias makes an unplanned input and output share one buffer: + * that input is read from its file at step 0 only and afterwards holds the + * output of the previous step. --share-output writes unplanned output SECOND + * into output FIRST's buffer. + * --misalign places every runner buffer one byte past a 16-byte boundary. + * --unbound-output never gives that unplanned output memory. + * --reset-before asks EspNNBackend, through its reset_states backend option, + * to restore the state of its delegates, or of one reset group, before that + * step. --reset-groups loads instance k in reset group k. At the end, the + * runner prints the delegates' memory as a STATS line. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using executorch::aten::ScalarType; +using executorch::aten::TensorImpl; +using executorch::extension::FileDataLoader; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; +using executorch::runtime::TensorInfo; + +namespace { + +[[noreturn]] void fail(const char* what, int code = 0) { + std::fprintf(stderr, "espnn_host_runner: %s (%d)\n", what, code); + std::exit(1); +} + +std::vector read_file(const std::string& path, size_t expected) { + std::FILE* file = std::fopen(path.c_str(), "rb"); + if (file == nullptr) + fail(("cannot open " + path).c_str()); + std::vector data(expected + 1); + const size_t read = std::fread(data.data(), 1, data.size(), file); + std::fclose(file); + if (read != expected) + fail(("wrong size: " + path).c_str(), static_cast(read)); + data.resize(expected); + return data; +} + +void write_file(const std::string& path, const void* data, size_t size) { + std::FILE* file = std::fopen(path.c_str(), "wb"); + if (file == nullptr || std::fwrite(data, 1, size, file) != size) + fail(("cannot write " + path).c_str()); + std::fclose(file); +} + +uint8_t* aligned_buffer(uint8_t* base, size_t misalign) { + return base + (16 - reinterpret_cast(base) % 16) % 16 + misalign; +} + +// A tensor over a runner-owned buffer with the program's shape and dim order. +struct HostTensor { + HostTensor(const TensorInfo& info, size_t misalign) + : sizes(info.sizes().begin(), info.sizes().end()), + dim_order(info.dim_order().begin(), info.dim_order().end()), + strides(sizes.size()), + storage(info.nbytes() + 32), + data(aligned_buffer(storage.data(), misalign)), + nbytes(info.nbytes()), + impl( + info.scalar_type(), + static_cast(sizes.size()), + sizes.data(), + data, + dim_order.data(), + strides.data()) { + if (executorch::runtime::dim_order_to_stride( + sizes.data(), dim_order.data(), sizes.size(), strides.data()) != + Error::Ok) + fail("invalid dim order"); + } + std::vector sizes; + std::vector dim_order; + std::vector strides; + std::vector storage; + uint8_t* data; + size_t nbytes; + TensorImpl impl; +}; + +struct Instance { + executorch::runtime::BackendOptions<1> reset_group; + executorch::runtime::LoadBackendOptionsMap load_options; + std::vector> planned; + std::vector> spans; + std::unique_ptr planned_allocator; + std::vector method_pool; + std::unique_ptr method_allocator; + std::unique_ptr memory; + std::unique_ptr method; + std::vector> inputs; + std::vector> outputs; +}; + +} // namespace + +int main(int argc, char** argv) { + if (argc < 4) + fail("usage: espnn_host_runner PROGRAM IO_DIR STEPS [options]"); + const std::string io_dir = argv[2]; + const int steps = std::atoi(argv[3]); + int instance_count = 1; + bool share_io = false; + size_t misalign = 0; + size_t unbound_output = SIZE_MAX; + std::vector> aliases; + std::vector> shared_outputs; + // (step, reset group or -1 for every delegate) + std::vector> resets; + bool reset_groups = false; + for (int a = 4; a < argc; ++a) { + const std::string arg = argv[a]; + if (arg == "--instances" && a + 1 < argc) { + instance_count = std::atoi(argv[++a]); + } else if (arg == "--reset-before" && a + 1 < argc) { + int step = 0, group = -1; + if (std::sscanf(argv[++a], "%d:%d", &step, &group) < 1) + fail("--reset-before takes STEP[:GROUP]"); + resets.emplace_back(step, group); + } else if (arg == "--reset-groups") { + reset_groups = true; + } else if (arg == "--share-io") { + share_io = true; + } else if (arg == "--misalign") { + misalign = 1; + } else if (arg == "--unbound-output" && a + 1 < argc) { + unbound_output = std::strtoul(argv[++a], nullptr, 10); + } else if (arg == "--alias" && a + 1 < argc) { + unsigned input = 0, output = 0; + if (std::sscanf(argv[++a], "%u:%u", &input, &output) != 2) + fail("--alias takes INPUT:OUTPUT"); + aliases.emplace_back(input, output); + } else if (arg == "--share-output" && a + 1 < argc) { + unsigned first = 0, second = 0; + if (std::sscanf(argv[++a], "%u:%u", &first, &second) != 2) + fail("--share-output takes FIRST:SECOND"); + shared_outputs.emplace_back(first, second); + } else { + fail(("unknown argument " + arg).c_str()); + } + } + if (!aliases.empty() && !share_io) + fail("--alias needs --share-io"); + + executorch::runtime::runtime_init(); + Result loader = FileDataLoader::from(argv[1]); + if (!loader.ok()) + fail("cannot load program", static_cast(loader.error())); + Result program = Program::load(&loader.get()); + if (!program.ok()) + fail("invalid program", static_cast(program.error())); + Result meta = program->method_meta("forward"); + if (!meta.ok()) + fail("no forward method", static_cast(meta.error())); + + std::vector instances(instance_count); + for (int k = 0; k < instance_count; ++k) { + Instance& instance = instances[k]; + if (reset_groups && + (instance.reset_group.set_option( + executorch::backends::espressif::kResetGroupOption, k) != + Error::Ok || + instance.load_options.set_options( + executorch::backends::espressif::kEspNNBackendId, + instance.reset_group.view()) != Error::Ok)) + fail("reset group options"); + for (size_t b = 0; b < meta->num_memory_planned_buffers(); ++b) { + instance.planned.emplace_back(meta->memory_planned_buffer_size(b).get()); + } + for (auto& buffer : instance.planned) + instance.spans.emplace_back(buffer.data(), buffer.size()); + instance.planned_allocator = std::make_unique( + Span>(instance.spans.data(), instance.spans.size())); + instance.method_pool.resize(1 << 20); + instance.method_allocator = std::make_unique( + instance.method_pool.size(), instance.method_pool.data()); + instance.memory = std::make_unique( + instance.method_allocator.get(), instance.planned_allocator.get()); + Result method = program->load_method( + "forward", + instance.memory.get(), + nullptr, + nullptr, + &instance.load_options); + if (!method.ok()) + fail("cannot load method", static_cast(method.error())); + instance.method = std::make_unique(std::move(method.get())); + + for (size_t i = 0; i < meta->num_inputs(); ++i) { + instance.inputs.push_back(std::make_unique( + meta->input_tensor_meta(i).get(), misalign)); + } + for (size_t o = 0; o < meta->num_outputs(); ++o) { + instance.outputs.push_back(std::make_unique( + meta->output_tensor_meta(o).get(), misalign)); + } + for (const auto& [input, output] : aliases) { + if (input >= instance.inputs.size() || + output >= instance.outputs.size() || + meta->input_tensor_meta(input).get().is_memory_planned() || + meta->output_tensor_meta(output).get().is_memory_planned() || + instance.inputs[input]->nbytes != instance.outputs[output]->nbytes) + fail("invalid --alias"); + } + for (const auto& [first, second] : shared_outputs) { + if (first >= instance.outputs.size() || + second >= instance.outputs.size() || + meta->output_tensor_meta(first).get().is_memory_planned() || + meta->output_tensor_meta(second).get().is_memory_planned() || + instance.outputs[first]->nbytes != instance.outputs[second]->nbytes) + fail("invalid --share-output"); + } + if (share_io) { + for (size_t i = 0; i < instance.inputs.size(); ++i) { + if (meta->input_tensor_meta(i).get().is_memory_planned()) + fail("--share-io requires unplanned inputs"); + Error error = instance.method->set_input( + EValue(executorch::aten::Tensor(&instance.inputs[i]->impl)), i); + if (error != Error::Ok) + fail("set_input", static_cast(error)); + } + } + for (size_t o = 0; o < instance.outputs.size(); ++o) { + if (meta->output_tensor_meta(o).get().is_memory_planned() || + o == unbound_output) + continue; + void* buffer = instance.outputs[o]->data; + for (const auto& [input, output] : aliases) { + if (output == o) + buffer = instance.inputs[input]->data; + } + for (const auto& [first, second] : shared_outputs) { + if (second == o) + buffer = instance.outputs[first]->data; + } + Error error = instance.method->set_output_data_ptr( + buffer, instance.outputs[o]->nbytes, o); + if (error != Error::Ok) + fail("set_output_data_ptr", static_cast(error)); + } + } + + for (int step = 0; step < steps; ++step) { + for (const auto& [reset, group] : resets) { + if (reset != step) + continue; + executorch::runtime::BackendOptions<1> options; + const Error set = group < 0 + ? options.set_option( + executorch::backends::espressif::kResetStatesOption, true) + : options.set_option( + executorch::backends::espressif::kResetStatesOption, group); + if (set != Error::Ok) + fail("BackendOptions::set_option"); + Error error = executorch::runtime::set_option( + executorch::backends::espressif::kEspNNBackendId, options.view()); + if (error != Error::Ok) + fail("set_option", static_cast(error)); + } + for (int k = 0; k < instance_count; ++k) { + Instance& instance = instances[k]; + for (size_t i = 0; i < instance.inputs.size(); ++i) { + bool carried = false; + for (const auto& alias : aliases) + carried = carried || (alias.first == i && step > 0); + if (carried) + continue; + HostTensor& tensor = *instance.inputs[i]; + const std::vector bytes = read_file( + io_dir + "/in_" + std::to_string(step) + "_" + std::to_string(i) + + ".bin", + tensor.nbytes); + std::memcpy(tensor.data, bytes.data(), bytes.size()); + if (!share_io) { + Error error = instance.method->set_input( + EValue(executorch::aten::Tensor(&tensor.impl)), i); + if (error != Error::Ok) + fail("set_input", static_cast(error)); + } + } + Error error = instance.method->execute(); + if (error != Error::Ok) + fail("execute", static_cast(error)); + for (size_t o = 0; o < instance.outputs.size(); ++o) { + const executorch::aten::Tensor output = + instance.method->get_output(o).toTensor(); + write_file( + io_dir + "/out_" + std::to_string(k) + "_" + std::to_string(step) + + "_" + std::to_string(o) + ".bin", + output.const_data_ptr(), + output.nbytes()); + } + } + } + const auto stats = executorch::backends::espressif::delegate_stats(); + std::printf( + "STATS delegates=%u arena=%zu scratch=%zu parameters=%zu tables=%zu " + "staging=%zu staged=%u states=%u states_in_place=%u\n", + static_cast(stats.delegates), + stats.arena_bytes, + stats.scratch_bytes, + stats.parameter_bytes, + stats.table_bytes, + stats.staging_bytes, + static_cast(stats.staged_outputs), + static_cast(stats.states), + static_cast(stats.states_updated_in_place)); + + return 0; +} diff --git a/backends/espressif/test/test_contracts.py b/backends/espressif/test/test_contracts.py index 4d68ae301d2..729b14d4c32 100644 --- a/backends/espressif/test/test_contracts.py +++ b/backends/espressif/test/test_contracts.py @@ -43,7 +43,10 @@ MAX_GRAPH_OPERATIONS, TENSOR_FLAG_CONSTANT, XTS3DType, + XTS3Graph, XTS3OpCode, + XTS3Operation, + XTS3Tensor, ) from executorch.backends.espressif.test.test_int8_ops import quantize # noqa: E402 from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( # noqa: E402 @@ -175,6 +178,34 @@ def test_runtime_limits_reject_misaligned_int32_constants(): check_runtime_limits(dataclasses.replace(graph, tensors=tensors)) +@pytest.mark.parametrize( + "opcode, shapes", + [ + (XTS3OpCode.CONV2D, lambda n: [(1, n, 1, 1), (1, 1, 1, 1)]), + (XTS3OpCode.DEPTHWISE_CONV2D, lambda n: [(1, 1, 1, 1), (1, 1, n, 1)]), + (XTS3OpCode.MAX_POOL2D, lambda n: [(1, 1, 1, n - 1), (1, 1, 1, n)]), + (XTS3OpCode.AVG_POOL2D, lambda n: [(1, 1, n, 1), (1, 1, 1, 1)]), + (XTS3OpCode.LINEAR, lambda n: [(1, n), (1, n), (1, 1)]), + ], + ids=["conv", "depthwise", "max-pool", "avg-pool", "linear"], +) +def test_runtime_limits_follow_esp_nn_extents(opcode, shapes): + def graph(extent): + tensors = [ + XTS3Tensor(k, shape, XTS3DType.INT8) + for k, shape in enumerate(shapes(extent)) + ] + output = len(tensors) - 1 + operation = XTS3Operation(0, opcode, list(range(output)), [output], b"") + return XTS3Graph( + tensors=tensors, operations=[operation], inputs=[0], outputs=[output] + ) + + check_runtime_limits(graph(65535)) + with pytest.raises(ValueError, match="uint16_t"): + check_runtime_limits(graph(65536)) + + class DropoutClassifier(nn.Module): """MobileNetV3's classifier: Linear, Hardswish, in-place Dropout, Linear.""" diff --git a/backends/espressif/test/test_host_interpreter.py b/backends/espressif/test/test_host_interpreter.py new file mode 100644 index 00000000000..d8dce393916 --- /dev/null +++ b/backends/espressif/test/test_host_interpreter.py @@ -0,0 +1,194 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Runs XTS3 graphs on the host build of Int8Interpreter from addresses of every +alignment and checks the outputs against the bit-exact reference interpreter. +Skipped unless ESPNN_HOST_INTERPRETER points to espnn_host_interpreter; see +backends/espressif/test/host/CMakeLists.txt. + + ESPNN_HOST_INTERPRETER=cmake-out-espnn-host/espnn_host_interpreter \ + pytest backends/espressif/test/test_host_interpreter.py +""" + +from __future__ import annotations + +import os +import struct +import subprocess + +import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization import xts3 +from executorch.backends.espressif.test import ( + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) + +INTERPRETER = os.environ.get("ESPNN_HOST_INTERPRETER") +pytestmark = pytest.mark.skipif( + not INTERPRETER, reason="Set ESPNN_HOST_INTERPRETER to espnn_host_interpreter" +) + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def run( + tmp_path, graph: xts3.XTS3Graph, offset: int, inputs, *options, env=None +) -> tuple[list, dict]: + blob = tmp_path / "graph.xts3" + blob.write_bytes(xts3.serialize_graph(graph)) + input_paths, output_paths = [], [] + for i, array in enumerate(inputs): + path = tmp_path / f"in_{i}.bin" + path.write_bytes(array.astype(np.int8).tobytes()) + input_paths.append(str(path)) + for o in range(len(graph.outputs)): + output_paths.append(str(tmp_path / f"out_{offset}_{o}.bin")) + result = subprocess.run( + [ + INTERPRETER, + str(blob), + str(offset), + *options, + *input_paths, + "--", + *output_paths, + ], + capture_output=True, + text=True, + env=None if env is None else {**os.environ, **env}, + ) + assert result.returncode == 0, result.stdout + result.stderr + (stats,) = [line for line in result.stdout.splitlines() if line.startswith("STATS")] + outputs = [ + np.frombuffer(open(path, "rb").read(), dtype=np.int8) for path in output_paths + ] + return outputs, dict(field.split("=") for field in stats.split()[1:]) + + +def se_block(version: int): + """An XTS3 graph of the given version, random inputs and its outputs.""" + _, program = op_cases.quantize(lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]) + graph = xts3.build_graph(program) + graph.version = version + graph = xts3.parse_graph(xts3.serialize_graph(graph)) + interpreter = reference.ReferenceInterpreter(graph) + generator = np.random.default_rng(0) + inputs = [ + generator.integers(-128, 128, reference.device_shape(graph.tensors[i])) + for i in interpreter.inputs + ] + expected = [ + value.astype(np.int8).reshape(-1) for value in interpreter.run_outputs(inputs) + ] + return graph, inputs, expected + + +@pytest.mark.parametrize("version", [6, 7]) +def test_graph_at_any_alignment(tmp_path, version): + graph, inputs, expected = se_block(version) + for offset in (0, 1, 2, 3, 4, 8): + outputs, stats = run(tmp_path, graph, offset, inputs) + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + # A misaligned graph is copied to aligned memory first. + assert stats["blob_copied"] == ("0" if offset == 0 else "1") + # Version 7 graphs keep their requantization parameters in place. + assert (stats["parameters"] == "0") == (version == 7) + + +@pytest.mark.parametrize("version", [6, 7]) +def test_constants_copied_to_psram(tmp_path, version): + # In a build with a PSRAM stand-in the graph is copied there and the + # runner frees the original, so every table pointer must follow the copy. + graph, inputs, expected = se_block(version) + outputs, stats = run(tmp_path, graph, 0, inputs, "--constants-to-psram") + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + assert stats["blob_copied"] == stats["psram"] + + +def test_arena_gets_internal_ram_first(tmp_path): + # Internal RAM holds the arena, its 16 readable bytes past the end and the + # default 48 KiB reserve, but not the version 6 graph's requantization + # tables too. In a build with a PSRAM stand-in, allocating the tables + # first would push the arena out to PSRAM. + graph, inputs, expected = se_block(6) + _, stats = run(tmp_path, graph, 0, inputs) + limit = int(stats["arena"]) + 16 + 48 * 1024 + 1 + outputs, stats = run( + tmp_path, graph, 0, inputs, env={"ESPNN_HOST_INTERNAL_BYTES": str(limit)} + ) + for actual, wanted in zip(outputs, expected): + np.testing.assert_array_equal(actual, wanted) + assert int(stats["parameters"]) > 0 + assert stats["arena_internal"] == "1" + + +def test_partial_copy_does_not_leak_into_a_bound_output(tmp_path): + # One COPY segment writes 4 of the output's 16 elements; the rest of a + # caller's output buffer must not keep its previous contents. + def tensor(tensor_id, size, flags): + return xts3.XTS3Tensor( + tensor_id, + (1, size), + xts3.XTS3DType.INT8, + flags=flags | xts3.TENSOR_FLAG_QUANTIZED, + quant_scale=1.0, + ) + + segment = struct.pack( + xts3.COPY_SEGMENT, + 0, + 1, + 0, + 0, + 4, + 1, + 1, + 1, + 1, + 1, + *([1] + [0] * 5) * 2, + 0, + 0, + 0, + 0, + 0, + ) + graph = xts3.XTS3Graph( + tensors=[ + tensor(0, 4, xts3.TENSOR_FLAG_GRAPH_INPUT), + tensor(1, 16, xts3.TENSOR_FLAG_GRAPH_OUTPUT), + ], + operations=[ + xts3.XTS3Operation( + 0, + xts3.XTS3OpCode.COPY, + [0], + [1], + struct.pack(xts3.COPY_HEADER, 1, 0, 0) + segment, + ) + ], + inputs=[0], + outputs=[1], + ) + values = np.arange(1, 5, dtype=np.int8) + (output,), _ = run(tmp_path, graph, 0, [values], "--external-io") + np.testing.assert_array_equal(output, np.concatenate([values, np.zeros(12)])) diff --git a/backends/espressif/test/test_host_runtime.py b/backends/espressif/test/test_host_runtime.py new file mode 100644 index 00000000000..abb56d3daef --- /dev/null +++ b/backends/espressif/test/test_host_runtime.py @@ -0,0 +1,815 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Runs fully delegated programs on the host build of the ESP-NN delegate +(espnn_runtime built outside ESP-IDF, over esp-nn's portable C kernels) and +checks every output against ProgramEmulator, the bit-exact Python reference. +Skipped unless ESPNN_HOST_RUNNER points to espnn_host_runner; see +backends/espressif/test/host/CMakeLists.txt. + + ESPNN_HOST_RUNNER=cmake-out-espnn-host/espnn_host_runner \ + pytest backends/espressif/test/test_host_runtime.py +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif import to_espnn_edge +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer +from executorch.backends.espressif.reference import ( + esp_nn_reference as reference, + ProgramEmulator, +) +from executorch.backends.espressif.serialization import xts3 +from executorch.backends.espressif.test import ( + test_cpu_fallback as fallback_cases, + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) +from executorch.backends.espressif.test.test_mutable_buffers import ( + InputAsState, + lower_input_as_state, + lower_stateful, + ReturnsClonedOldState, + ReturnsOldState, + ShiftRegister, + split_streams, + StateReadLate, + TwoStreams, +) +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.backend.compile_spec_schema import CompileSpec +from executorch.exir.lowered_backend_module import LoweredBackendModule +from executorch.exir.passes import MemoryPlanningPass +from executorch.exir.passes.quantize_io_pass import quantize_input, quantize_output +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +RUNNER = os.environ.get("ESPNN_HOST_RUNNER") +pytestmark = pytest.mark.skipif( + not RUNNER, reason="Set ESPNN_HOST_RUNNER to espnn_host_runner" +) + + +class RepeatedOutput(torch.nn.Module): + """Returns one tensor twice, so two delegate outputs share a graph tensor.""" + + def __init__(self): + super().__init__() + self.conv = torch.nn.Conv2d(3, 4, 1) + + def forward(self, x): + y = self.conv(x) + return y, y + + +class LongCat(torch.nn.Module): + """A cat of seven tensors, which two COPY operations write into one output.""" + + def __init__(self): + super().__init__() + self.convs = torch.nn.ModuleList(torch.nn.Conv2d(3, 2, 1) for _ in range(7)) + + def forward(self, x): + return torch.cat([conv(x) for conv in self.convs], dim=2) + + +class AddChain(torch.nn.Module): + def __init__(self, count): + super().__init__() + self.count = count + + def forward(self, x): + x = InputAsState.qdq(x, 0.1) + for _ in range(self.count): + x = InputAsState.qdq(x + x, 0.1) + return x + + +class CpuBetweenAdds(torch.nn.Module): + def forward(self, x): + x = InputAsState.qdq(x, 0.1) + x = InputAsState.qdq(x + x, 0.1) + x = InputAsState.qdq(torch.cumsum(x, dim=1), 0.1) + return InputAsState.qdq(x + x, 0.1) + + +# Fully delegated programs, checked bit-exactly; transpose-select-stack leaves a +# cat on the CPU and runs with the CPU-fallback programs below. single-conv has +# no activation besides its input and output. +CASES = [ + ("single-conv", torch.nn.Conv2d(3, 4, 1), [(1, 3, 8, 8)]), + ("repeated-output", RepeatedOutput(), [(1, 3, 8, 8)]), + ("seven-way-cat", LongCat(), [(1, 3, 4, 4)]), + ("conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)]), + ("concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)]), + ("se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)]), +] + [ + (name, model, shapes) + for name, model, shapes, _ in op_cases.CASES + if name != "transpose-select-stack" +] + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def lower( + model, + shapes, + alloc_graph_input: bool = True, + alloc_graph_output: bool = True, + xts3_version: int | None = None, +): + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert not report["cpu_ops"] + if xts3_version is not None: + for _, module in edge.exported_program().graph_module.named_children(): + if isinstance(module, LoweredBackendModule): + graph = xts3.parse_graph(module.processed_bytes) + graph.version = xts3_version + module._processed_bytes = xts3.serialize_graph(graph) + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=alloc_graph_input, alloc_graph_output=alloc_graph_output + ) + ) + return edge.exported_program(), edge.to_executorch(config).buffer + + +def as_list(value): + return list(value) if isinstance(value, (tuple, list)) else [value] + + +def to_bytes(tensor: torch.Tensor) -> bytes: + """The tensor's bytes in memory order, which may be channels-last.""" + if tensor.dim() == 4 and tensor.is_contiguous(memory_format=torch.channels_last): + tensor = tensor.permute(0, 2, 3, 1) + return tensor.contiguous().numpy().tobytes() + + +def from_bytes(data: bytes, like: torch.Tensor, channels_last: bool) -> torch.Tensor: + channels_last = channels_last and like.dim() == 4 + shape = tuple(like.permute(0, 2, 3, 1).shape) if channels_last else like.shape + tensor = torch.frombuffer(bytearray(data), dtype=like.dtype).reshape(shape) + return tensor.permute(0, 3, 1, 2) if channels_last else tensor + + +def run_runner(tmp_path: Path, program: bytes, steps, *args): + (tmp_path / "program.pte").write_bytes(program) + for step, inputs in enumerate(steps): + for i, tensor in enumerate(inputs): + (tmp_path / f"in_{step}_{i}.bin").write_bytes(to_bytes(tensor)) + return subprocess.run( + [RUNNER, str(tmp_path / "program.pte"), str(tmp_path), str(len(steps))] + + [str(arg) for arg in args], + capture_output=True, + text=True, + ) + + +def run_host(tmp_path: Path, program: bytes, steps, *args): + """ + Runs one list of input tensors per step. Returns the output files per + instance and step, and the delegates' memory statistics. + """ + result = run_runner(tmp_path, program, steps, *args) + assert result.returncode == 0, result.stdout + result.stderr + (stats_line,) = [ + line for line in result.stdout.splitlines() if line.startswith("STATS ") + ] + stats = { + key: int(value) + for key, value in (field.split("=") for field in stats_line.split()[1:]) + } + instances = 1 + if "--instances" in args: + instances = int(args[args.index("--instances") + 1]) + outputs = [ + [sorted(tmp_path.glob(f"out_{k}_{step}_*.bin")) for step in range(len(steps))] + for k in range(instances) + ] + return outputs, stats + + +def assert_matches( + paths: list[Path], expected: list[torch.Tensor], channels_last: bool = False +): + """Compares output files, in the program's dim order, with the reference.""" + assert len(paths) == len(expected) + for path, tensor in zip(paths, expected): + actual = from_bytes(path.read_bytes(), tensor, channels_last) + if tensor.dtype == torch.float32: + torch.testing.assert_close(actual, tensor, atol=1e-5, rtol=1e-5) + else: + assert torch.equal(actual, tensor) + + +def arena_holds_plan(stats, planned: int) -> bool: + """Whether the arena is the planned size, plus any ESP-NN scratch in it.""" + return stats["arena"] == planned if stats["scratch"] else stats["arena"] >= planned + + +def random_steps(shapes, count: int, seed: int = 11): + generator = torch.Generator().manual_seed(seed) + return [ + [torch.randn(shape, generator=generator) for shape in shapes] + for _ in range(count) + ] + + +@pytest.mark.parametrize("name, model, shapes", CASES, ids=[c[0] for c in CASES]) +def test_host_delegate_matches_reference(tmp_path, name, model, shapes): + exported, program = lower(model, shapes) + steps = random_steps(shapes, 2) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + # With ESP-NN scratch planned in the arena, the arena also holds scratch. + planned = sum( + reference.plan_activation_arena(delegate.graph)[0] + for delegate in emulator.delegates.values() + ) + assert stats["delegates"] == len(emulator.delegates) + # Version 7 graphs keep every requantization table in place. + assert stats["parameters"] == 0 + if stats["scratch"]: + assert stats["arena"] == planned + else: + assert stats["arena"] >= planned + + +@pytest.mark.parametrize("alloc_graph_input", [False, True]) +@pytest.mark.parametrize("alloc_graph_output", [False, True]) +@pytest.mark.parametrize( + "input_int8, output_int8, channels_last, legacy, adds, arena_slots", + [ + pytest.param(False, False, False, False, 2, 2, id="float-nchw"), + pytest.param(False, False, True, False, 2, 2, id="float-nhwc"), + pytest.param(True, True, False, False, 2, 2, id="int8-nchw"), + pytest.param(True, True, True, False, 2, 2, id="int8-nhwc"), + pytest.param(False, True, True, False, 1, 1, id="float-input-only"), + pytest.param(True, False, True, False, 1, 2, id="float-output-only"), + pytest.param(False, False, False, True, 2, 2, id="legacy-float"), + pytest.param(True, True, True, True, 2, 2, id="legacy-int8"), + ], +) +def test_conversion_buffers_share_the_arena( + tmp_path, + input_int8, + output_int8, + channels_last, + legacy, + adds, + arena_slots, + alloc_graph_input, + alloc_graph_output, +): + shape = (1, 16, 64, 64) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + sample = torch.zeros(shape).contiguous(memory_format=memory_format) + program = torch.export.export(AddChain(adds), (sample,), strict=True) + if input_int8: + quantize_input(program, 0) + if output_int8: + quantize_output(program, 0) + edge, _ = to_espnn_edge(program, allow_cpu_fallback=False) + if legacy: + for module in edge.exported_program().graph_module.children(): + if isinstance(module, LoweredBackendModule): + module.compile_specs[:] = [ + spec + for spec in module.compile_specs + if spec.key != "espnn_io_conversion_slots" + ] + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=alloc_graph_input, alloc_graph_output=alloc_graph_output + ) + ) + buffer = edge.to_executorch(config).buffer + _, loaded = run_host(tmp_path, buffer, []) + assert loaded["arena"] == arena_slots * sample.numel() + # The staging buffer, allocated at init, covers outputs outside the arena. + bound = output_int8 and channels_last and not legacy + assert loaded["staging"] == ((sample.numel() + 15) // 16 * 16 if bound else 0) + steps = random_steps([shape], 2) + steps = [ + [ + ((x * 20).round().to(torch.int8) if input_int8 else x).contiguous( + memory_format=memory_format + ) + ] + for (x,) in steps + ] + (outputs,), stats = run_host(tmp_path, buffer, steps) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) + assert stats["arena"] == loaded["arena"] + assert stats["staged"] == 0 + + +def test_partitions_keep_independent_conversion_plans(tmp_path): + program = torch.export.export(CpuBetweenAdds(), (torch.zeros(1, 4),)) + quantize_input(program, 0) + edge, _ = to_espnn_edge(program) + steps = [[torch.full((1, 4), value, dtype=torch.int8)] for value in (1, 3)] + (outputs,), stats = run_host(tmp_path, edge.to_executorch().buffer, steps) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["delegates"] == 2 + # Inputs are always copied; both partitions' outputs need conversion. + assert stats["arena"] == 32 + 32 + assert stats["staged"] == 0 + + +def test_inputs_are_copied_for_exports_that_bound_them(tmp_path): + # Exports that recorded no conversion slots for compatible inputs. + sample = torch.zeros(1, 16, 8, 8).contiguous(memory_format=torch.channels_last) + program = torch.export.export(AddChain(2), (sample,), strict=True) + quantize_input(program, 0) + quantize_output(program, 0) + edge, _ = to_espnn_edge(program, allow_cpu_fallback=False) + for module in edge.exported_program().graph_module.children(): + if isinstance(module, LoweredBackendModule): + module.compile_specs[:] = [ + ( + CompileSpec(spec.key, bytes(4)) + if spec.key == "espnn_io_conversion_slots" + else spec + ) + for spec in module.compile_specs + ] + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) + ) + generator = torch.Generator().manual_seed(2) + steps = [ + [ + torch.randint( + -128, 128, sample.shape, generator=generator, dtype=torch.int8 + ).contiguous(memory_format=torch.channels_last) + ] + for _ in range(2) + ] + (outputs,), stats = run_host( + tmp_path, edge.to_executorch(config).buffer, steps, "--share-io" + ) + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), True) + assert stats["arena"] == 2 * sample.numel() + assert stats["staged"] == 0 + + +def test_instances_run_independently(tmp_path): + model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + exported, program = lower(model, shapes) + steps = random_steps(shapes, 3) + (first, second), stats = run_host(tmp_path, program, steps, "--instances", 2) + assert stats["delegates"] == 2 + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert_matches(first[step], expected) + assert_matches(second[step], expected) + + +def test_unplanned_io_shared_with_the_caller(tmp_path): + model, shapes = op_cases.TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)] + exported, program = lower( + model, shapes, alloc_graph_input=False, alloc_graph_output=False + ) + steps = random_steps(shapes, 2) + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io") + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + + +def test_output_without_memory_is_rejected(tmp_path): + model, shapes = lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + _, program = lower(model, shapes, alloc_graph_output=False) + result = run_runner( + tmp_path, program, random_steps(shapes, 1), "--unbound-output", 0 + ) + assert result.returncode != 0 + assert "output 0 has no data" in result.stdout + assert "execute (18)" in result.stderr # Error::InvalidArgument + + +def test_version_6_graphs_still_run(tmp_path): + model, shapes = lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] + exported, program = lower(model, shapes, xts3_version=6) + steps = random_steps(shapes, 2) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert all(d.graph.version == 6 for d in emulator.delegates.values()) + assert stats["parameters"] > 0 + + +FALLBACK_CASES = [ + ("transpose-select-stack", op_cases.TransposeSelect(), [(1, 3, 4, 5)]) +] + [(name, model, shapes) for name, model, shapes, _ in fallback_cases.CASES] + + +@pytest.mark.parametrize( + "name, model, shapes", FALLBACK_CASES, ids=[c[0] for c in FALLBACK_CASES] +) +def test_programs_with_cpu_operators(tmp_path, name, model, shapes): + # Delegates sit between portable kernels, which receive and return their + # tensors (some delegates return one tensor as several outputs); the + # kernels' float rounding gets the emulator suite's tolerance. + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program) + assert report["cpu_ops"] + exported = edge.exported_program() + steps = random_steps(shapes, 2) + (outputs,), _ = run_host(tmp_path, edge.to_executorch().buffer, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + expected = [t.to(torch.float32) for t in as_list(emulator(*inputs))] + peak = max(float(t.abs().max()) for t in expected) + for path, tensor in zip(outputs[step], expected): + actual = from_bytes(path.read_bytes(), tensor, False) + torch.testing.assert_close( + actual, tensor, atol=0.01 * max(peak, 1.0), rtol=0.01 + ) + + +class StreamingConv(torch.nn.Module): + """One step of a streaming convolution: the state holds the last frames.""" + + def __init__(self, channels_last: bool): + super().__init__() + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + self.memory_format = ( + torch.channels_last if channels_last else torch.contiguous_format + ) + + def forward(self, x, state): + window = torch.cat([state, x], dim=2) + new_state = window[:, :, 1:].clone(memory_format=self.memory_format) + return self.conv(window).clone(memory_format=self.memory_format), new_state + + +class LateStateRead(StreamingConv): + """Writes its new state before it reads the old one.""" + + def __init__(self, channels_last: bool): + super().__init__(channels_last) + self.conv = torch.nn.Conv2d(4, 8, (2, 1)) + self.frame = torch.nn.Conv2d(4, 4, 1) + + def forward(self, x, state): + new_state = ( + self.frame(x).expand(-1, -1, 2, -1).clone(memory_format=self.memory_format) + ) + return self.conv(state).clone(memory_format=self.memory_format), new_state + + +class EarlyOutput(LateStateRead): + """LateStateRead with its output computed before its new state.""" + + def forward(self, x, state): + output = self.conv(state).clone(memory_format=self.memory_format) + new_state = ( + self.frame(x).expand(-1, -1, 2, -1).clone(memory_format=self.memory_format) + ) + return output, new_state + + +class ChainedOutputs(StreamingConv): + """Returns a tensor and two functions of it, computed one after the other.""" + + def forward(self, x, state): + a = self.conv(torch.cat([state, x], dim=2)).clone( + memory_format=self.memory_format + ) + b = torch.relu(a).clone(memory_format=self.memory_format) + c = torch.sigmoid(a).clone(memory_format=self.memory_format) + return a, b, c + + +def lower_streaming(channels_last: bool, model_class=StreamingConv, patch=None): + """ + int8 inputs and outputs, not memory planned, optionally channels-last. + patch, if given, edits each delegate's XTS3 graph before serialization. + """ + model = model_class(channels_last).eval() + memory_format = torch.channels_last if channels_last else torch.contiguous_format + shapes = [(1, 4, 1, 1), (1, 4, 2, 1)] + + def sample(): + return tuple( + torch.randn(shape).to(memory_format=memory_format) for shape in shapes + ) + + example = sample() + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(16): + prepared(*sample()) + program = torch.export.export(convert_pt2e(prepared), example, strict=True) + for index in range(len(program.graph_signature.user_inputs)): + quantize_input(program, index) + for index in range(len(program.graph_signature.user_outputs)): + quantize_output(program, index) + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert not report["cpu_ops"] + if patch is not None: + for _, module in edge.exported_program().graph_module.named_children(): + if isinstance(module, LoweredBackendModule): + graph = xts3.parse_graph(module.processed_bytes) + patch(graph) + module._processed_bytes = xts3.serialize_graph(graph) + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ) + ) + return edge.exported_program(), edge.to_executorch(config).buffer, sample + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_device_layout_io_is_used_in_place(tmp_path, channels_last): + exported, program, sample = lower_streaming(channels_last) + emulator = ProgramEmulator(exported) + steps = [] + for _ in range(2): + floats = sample() + steps.append( + [torch.clamp((t * 20).round(), -128, 127).to(torch.int8) for t in floats] + ) + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), channels_last) + # Inputs and NCHW outputs use the arena; channels-last outputs are bound + # directly. + assert stats["staged"] == 0 + if channels_last: + assert arena_holds_plan( + stats, + sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ), + ) + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_output_sharing_an_input_buffer(tmp_path, channels_last): + # The state output is written into the state input's buffer, as an + # application updating its state in place would bind them. + exported, program, _ = lower_streaming(channels_last, LateStateRead) + emulator = ProgramEmulator(exported) + generator = torch.Generator().manual_seed(3) + steps = [ + [ + torch.randint(-128, 128, shape, generator=generator, dtype=torch.int8) + for shape in [(1, 4, 1, 1), (1, 4, 2, 1)] + ] + for _ in range(4) + ] + if channels_last: + steps = [ + [t.contiguous(memory_format=torch.channels_last) for t in inputs] + for inputs in steps + ] + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io", "--alias", "1:1") + state = steps[0][1] + for step, inputs in enumerate(steps): + expected = as_list(emulator(inputs[0], state)) + assert_matches(outputs[step], expected, channels_last) + state = expected[1] + + +def int8_steps(count: int, channels_last: bool, seed: int = 5): + generator = torch.Generator().manual_seed(seed) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + return [ + [ + torch.randint( + -128, 128, shape, generator=generator, dtype=torch.int8 + ).contiguous(memory_format=memory_format) + for shape in [(1, 4, 1, 1), (1, 4, 2, 1)] + ] + for _ in range(count) + ] + + +def test_misaligned_buffers_are_staged(tmp_path): + exported, program, _ = lower_streaming(channels_last=True) + emulator = ProgramEmulator(exported) + steps = int8_steps(2, channels_last=True) + _, loaded = run_host(tmp_path, program, [], "--share-io", "--misalign") + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io", "--misalign") + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs)), True) + # Staged through the buffer allocated at init; execute() allocates nothing. + assert stats["staged"] > 0 + assert stats["staging"] == loaded["staging"] > 0 + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_outputs_read_by_operators_keep_their_slots(tmp_path, channels_last): + # Output 1 is written over output 0's buffer before an operator reads + # output 0 for output 2, so output 0 must stay in its arena slot. + exported, program, _ = lower_streaming(channels_last, ChainedOutputs) + emulator = ProgramEmulator(exported) + steps = int8_steps(2, channels_last) + (outputs,), stats = run_host( + tmp_path, program, steps, "--share-io", "--share-output", "0:1" + ) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert_matches(outputs[step][2:], expected[2:], channels_last) + assert stats["staged"] == 0 + if channels_last: + assert arena_holds_plan( + stats, + sum( + reference.plan_activation_arena(delegate.graph, external_io=True)[0] + for delegate in emulator.delegates.values() + ), + ) + + +def return_state_input(graph): + """Makes the graph return its state input instead of its new state.""" + state_in = graph.inputs[1] + shape = graph.tensors[state_in].shape + (index,) = [ + k for k, t in enumerate(graph.outputs) if graph.tensors[t].shape == shape + ] + graph.tensors[graph.outputs[index]].flags &= ~xts3.TENSOR_FLAG_GRAPH_OUTPUT + graph.tensors[state_in].flags |= xts3.TENSOR_FLAG_GRAPH_OUTPUT + graph.outputs[index] = state_in + + +@pytest.mark.parametrize("channels_last", [False, True]) +@pytest.mark.parametrize("alias", [[], ["--alias", "1:0"]], ids=["", "aliased"]) +def test_input_returned_as_output(tmp_path, channels_last, alias): + # With the alias, the first output is written over the returned input's + # buffer, which must not change the second output. + exported, program, _ = lower_streaming( + channels_last, EarlyOutput, patch=return_state_input + ) + emulator = ProgramEmulator(exported) + steps = int8_steps(1 if alias else 2, channels_last) + (outputs,), _ = run_host(tmp_path, program, steps, "--share-io", *alias) + for step, inputs in enumerate(steps): + expected = as_list(emulator(*inputs)) + assert torch.equal(expected[1], inputs[1]) + assert_matches(outputs[step], expected, channels_last) + + +def stateful_steps(count: int): + return random_steps([(1, 4, 1, 1)], count, seed=7) + + +@pytest.mark.parametrize( + "model_class, in_place", [(ShiftRegister, 1), (StateReadLate, 0)] +) +def test_delegate_owned_state_persists(tmp_path, model_class, in_place): + exported, program, report = lower_stateful(model_class()) + assert report["delegated_buffers"] == ["state"] + assert len(exported.graph_signature.user_inputs) == 1 + assert not exported.graph_signature.buffers_to_mutate + steps = stateful_steps(5) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + assert stats["states_in_place"] == in_place + # The state changes the outputs, so the check above depends on it. + replay = ProgramEmulator(exported) + stateful = [as_list(replay(*inputs))[0] for inputs in steps] + fresh = [as_list(ProgramEmulator(exported)(*inputs))[0] for inputs in steps] + assert not all(torch.equal(a, b) for a, b in zip(stateful, fresh)) + + +def test_state_updated_from_a_graph_input(tmp_path): + exported, program, _ = lower_input_as_state() + steps = stateful_steps(4) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + + +@pytest.mark.parametrize("model_class", [ReturnsOldState, ReturnsClonedOldState]) +def test_returned_old_state(tmp_path, model_class): + exported, program, _ = lower_input_as_state(model_class) + steps = stateful_steps(4) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 1 + + +def test_state_reset_and_instances(tmp_path): + exported, program, _ = lower_stateful(ShiftRegister()) + steps = stateful_steps(6) + first, second = run_host( + tmp_path, program, steps, "--instances", 2, "--reset-before", 4 + )[0] + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + if step == 4: + emulator.reset_states() + expected = as_list(emulator(*inputs)) + assert_matches(first[step], expected) + assert_matches(second[step], expected) + + +def test_state_reset_by_group(tmp_path): + # Instance k is in reset group k, and only group 1 is reset. + exported, program, _ = lower_stateful(ShiftRegister()) + steps = stateful_steps(6) + kept, reset = run_host( + tmp_path, + program, + steps, + "--instances", + 2, + "--reset-groups", + "--reset-before", + "3:1", + )[0] + continuous, restarted = ProgramEmulator(exported), ProgramEmulator(exported) + for step, inputs in enumerate(steps): + if step == 3: + restarted.reset_states() + assert_matches(kept[step], as_list(continuous(*inputs))) + assert_matches(reset[step], as_list(restarted(*inputs))) + + +def test_state_in_two_partitions_stays_in_the_program(tmp_path, monkeypatch): + split_streams(monkeypatch) + exported, program, report = lower_stateful( + TwoStreams(), inputs=2, allow_cpu_fallback=True + ) + assert report["partitions"] == 2 and not report["delegated_buffers"] + steps = random_steps([(1, 4, 1, 1)] * 2, 5) + (outputs,), stats = run_host(tmp_path, program, steps) + emulator = ProgramEmulator(exported) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["delegates"] == 2 and stats["states"] == 0 + + +def test_streaming_example_runs_without_copies(tmp_path): + sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "examples/espressif")) + import export_streaming_model as example + + edge = example.lower(calibration_frames=32) + program = edge.to_executorch(example.executorch_config()).buffer + generator = torch.Generator().manual_seed(1) + steps = [ + [ + torch.randint( + -128, + 128, + (1, example.FEATURES, 1, 1), + generator=generator, + dtype=torch.int8, + ) + ] + for _ in range(6) + ] + (outputs,), stats = run_host(tmp_path, program, steps, "--share-io") + emulator = ProgramEmulator(edge.exported_program()) + for step, inputs in enumerate(steps): + assert_matches(outputs[step], as_list(emulator(*inputs))) + assert stats["states"] == 2 and stats["staged"] == 0 diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py index ee33d41fd23..e70ef48cf90 100644 --- a/backends/espressif/test/test_int8_ops.py +++ b/backends/espressif/test/test_int8_ops.py @@ -210,6 +210,17 @@ def forward(self, image, features): return mask, logits +class OddChannelPool(nn.Module): + """Max pooling over a channel count that is not a multiple of four.""" + + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 6, 3, padding=1) + + def forward(self, x): + return F.max_pool2d(F.relu(self.conv(x)), 2) + + class TransposeSelect(nn.Module): def __init__(self): super().__init__() @@ -263,6 +274,7 @@ def forward(self, x): ("broadcast-pad-split-cat", BroadcastPadSplit(), [(1, 4, 6, 6)], 2), ("two-inputs-two-outputs", TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)], 2), ("transpose-select-stack", TransposeSelect(), [(1, 3, 4, 5)], 2), + ("max-pool-six-channels", OddChannelPool(), [(1, 3, 8, 8)], 2), ] diff --git a/backends/espressif/test/test_mutable_buffers.py b/backends/espressif/test/test_mutable_buffers.py new file mode 100644 index 00000000000..a97e1e5a332 --- /dev/null +++ b/backends/espressif/test/test_mutable_buffers.py @@ -0,0 +1,452 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Lowering of mutable buffers (streaming state) with delegate_mutable_buffers: +which buffers the ESP-NN delegate owns, and the stateful host reference. +test_host_runtime.py runs the same models on the delegate. + + pytest backends/espressif/test/test_mutable_buffers.py +""" + +from __future__ import annotations + +import copy + +import pytest + +pytest.importorskip("torchao") + +import torch + +from executorch.backends.espressif import to_espnn_edge +from executorch.backends.espressif.partition import espnn_partitioner as partitioner +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer +from executorch.backends.espressif.reference import ProgramEmulator +from executorch.backends.espressif.serialization import xts3 +from executorch.exir.lowered_backend_module import LoweredBackendModule +from torch.fx.passes.infra.partitioner import Partition +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +class ShiftRegister(torch.nn.Module): + """A streaming convolution over the last frames, kept in a buffer.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 2, 1)) + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + + def forward(self, x): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:]) + return self.conv(window) + + +class StateReadLate(ShiftRegister): + """Computes its new state before an operator reads the old one.""" + + def forward(self, x): + new_state = torch.cat([x, x], dim=2) + y = self.conv(torch.cat([self.state, x], dim=2)) + self.state.copy_(new_state) + return y + + +class InputAsState(torch.nn.Module): + """ + Explicit int8 Q/DQ (no PT2E): the new state is the quantized input itself, + a graph input of the delegate, which the delegate copies into the state. + """ + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + @staticmethod + def qdq(value, scale): + args = (scale, 0, -128, 127, torch.int8) + quantized = torch.ops.quantized_decomposed.quantize_per_tensor(value, *args) + return torch.ops.quantized_decomposed.dequantize_per_tensor(quantized, *args) + + def forward(self, x): + frame = self.qdq(x, 0.05) + y = self.qdq(self.qdq(self.state, 0.05) + frame, 0.1) + self.state.copy_(frame) + return y + + +class ReturnsOldState(InputAsState): + """InputAsState that also returns the state it read.""" + + def forward(self, x): + old = self.qdq(self.state, 0.05) + frame = self.qdq(x, 0.05) + y = self.qdq(old + frame, 0.1) + self.state.copy_(frame) + return y, old + + +class ReturnsClonedOldState(InputAsState): + """ReturnsOldState with the old state returned through a clone.""" + + def forward(self, x): + old = self.qdq(self.state, 0.05) + frame = self.qdq(x, 0.05) + y = self.qdq(old + frame, 0.1) + self.state.copy_(frame) + return y, self.qdq(old.clone(), 0.05) + + +class StateOnlyInput(torch.nn.Module): + """Explicit int8 Q/DQ recurrence whose only input is its state.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + def forward(self): + qdq = InputAsState.qdq + state = qdq(self.state, 0.1) + update = qdq(state + state, 0.1) + self.state.copy_(update) + return qdq(update + state, 0.1) + + +class StateOnlyOutput(torch.nn.Module): + """Explicit int8 Q/DQ accumulator whose only delegated output is its state.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 1, 1)) + + def forward(self, x): + qdq = InputAsState.qdq + self.state.copy_(qdq(qdq(self.state, 0.1) + qdq(x, 0.1), 0.1)) + return torch.cumsum(x, dim=1) + + +class StateOnlyOutputWithConstant(StateOnlyOutput): + """StateOnlyOutput whose CPU operator reads a constant.""" + + def __init__(self): + super().__init__() + self.register_buffer("offset", torch.full((1, 4, 1, 1), 0.2)) + + def forward(self, x): + qdq = InputAsState.qdq + self.state.copy_(qdq(qdq(self.state, 0.1) + qdq(x, 0.1), 0.1)) + return torch.cumsum(x + self.offset, dim=1) + + +def lower_input_as_state(model_class=InputAsState): + program = torch.export.export( + model_class().eval(), (torch.zeros(1, 4, 1, 1),), strict=True + ) + edge, report = to_espnn_edge( + program, allow_cpu_fallback=False, delegate_mutable_buffers=True + ) + # to_executorch() rewrites the mutations of buffers left in the program. + exported = copy.deepcopy(edge.exported_program()) + return exported, edge.to_executorch().buffer, report + + +class ScaledState(ShiftRegister): + """Its new state has other quantization parameters than the state.""" + + def forward(self, x): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:] * 0.5) + return self.conv(window) + + +class TwoStreams(torch.nn.Module): + """Two shift registers on separate inputs, which partitions may split.""" + + def __init__(self): + super().__init__() + self.register_buffer("state", torch.zeros(1, 4, 2, 1)) + self.register_buffer("other", torch.zeros(1, 4, 2, 1)) + self.conv = torch.nn.Conv2d(4, 8, (3, 1)) + self.other_conv = torch.nn.Conv2d(4, 8, (3, 1)) + + def forward(self, x, y): + window = torch.cat([self.state, x], dim=2) + self.state.copy_(window[:, :, 1:]) + other = torch.cat([self.other, y], dim=2) + self.other.copy_(other[:, :, 1:]) + return self.conv(window), self.other_conv(other) + + +def split_streams(monkeypatch): + """Makes EspNNPartitioner propose each connected part as a partition.""" + propose = partitioner.CapabilityBasedPartitioner.propose_partitions + + def propose_streams(self): + streams = [] + for partition in propose(self): + nodes = set(partition.nodes) + while nodes: + stream, frontier = set(), [nodes.pop()] + while frontier: + node = frontier.pop() + stream.add(node) + for other in [*node.all_input_nodes, *node.users]: + if other in nodes: + nodes.discard(other) + frontier.append(other) + streams.append(Partition(nodes=stream)) + return streams + + monkeypatch.setattr( + partitioner.CapabilityBasedPartitioner, "propose_partitions", propose_streams + ) + + +class ManyStates(torch.nn.Module): + """More shift registers than one delegate can own.""" + + def __init__(self, count: int): + super().__init__() + for k in range(count): + self.register_buffer(f"state_{k}", torch.zeros(1, 4, 2, 1)) + self.convs = torch.nn.ModuleList( + torch.nn.Conv2d(4, 2, (3, 1)) for _ in range(count) + ) + + def forward(self, x): + outputs = [] + for k, conv in enumerate(self.convs): + state = getattr(self, f"state_{k}") + window = torch.cat([state, x], dim=2) + state.copy_(window[:, :, 1:]) + outputs.append(conv(window)) + return torch.cat(outputs, dim=1) + + +def quantize_stateful(model, inputs: int = 1): + shape = (1, 4, 1, 1) + generator = torch.Generator().manual_seed(5) + example = tuple(torch.zeros(shape) for _ in range(inputs)) + prepared = prepare_pt2e( + torch.export.export(model.eval(), example, strict=True).module(), + EspNNQuantizer(), + ) + for _ in range(16): + prepared(*(torch.randn(shape, generator=generator) for _ in example)) + converted = convert_pt2e(prepared) + # Calibration ran the model, so start again from the initial state. + for name, _ in model.named_buffers(): + getattr(converted, name).zero_() + return converted + + +def lower_stateful( + model, + delegate_mutable_buffers: bool = True, + quantized=None, + allow_cpu_fallback: bool = False, + inputs: int = 1, +): + converted = ( + copy.deepcopy(quantized) if quantized else quantize_stateful(model, inputs) + ) + example = tuple(torch.zeros(1, 4, 1, 1) for _ in range(inputs)) + program = torch.export.export(converted, example, strict=True) + edge, report = to_espnn_edge( + program, + allow_cpu_fallback=allow_cpu_fallback, + delegate_mutable_buffers=delegate_mutable_buffers, + ) + # to_executorch() rewrites the mutations of buffers left in the program. + exported = copy.deepcopy(edge.exported_program()) + return exported, edge.to_executorch().buffer, report + + +def delegate_graphs(exported) -> list[xts3.XTS3Graph]: + return [ + xts3.parse_graph(module.processed_bytes) + for _, module in exported.graph_module.named_children() + if isinstance(module, LoweredBackendModule) + ] + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def test_delegate_owns_the_state(): + exported, _, report = lower_stateful(ShiftRegister()) + assert report["delegated_buffers"] == ["state"] + assert len(exported.graph_signature.user_inputs) == 1 + assert not exported.graph_signature.buffers_to_mutate + (graph,) = delegate_graphs(exported) + assert graph.version == xts3.XTS3_STATE_VERSION + ((state, update),) = graph.states + assert graph.tensors[state].flags & xts3.TENSOR_FLAG_STATE + assert graph.inputs == [graph.inputs[0]] and state not in graph.inputs + assert xts3.parse_graph(xts3.serialize_graph(graph)).states == graph.states + + +def test_reference_tracks_the_quantized_model(): + model = quantize_stateful(ShiftRegister()) + exported, _, _ = lower_stateful(None, quantized=model) + emulator = ProgramEmulator(exported) + generator = torch.Generator().manual_seed(9) + for step in range(5): + x = torch.randn(1, 4, 1, 1, generator=generator) + if step == 3: + model.state.zero_() + emulator.reset_states() + torch.testing.assert_close(emulator(x), model(x), atol=0.05, rtol=0) + + +def test_mutable_buffers_stay_in_the_program_by_default(): + exported, _, report = lower_stateful(ShiftRegister(), False) + assert report["delegated_buffers"] == [] + assert list(exported.graph_signature.buffers_to_mutate.values()) == ["state"] + (graph,) = delegate_graphs(exported) + assert graph.version == xts3.XTS3_VERSION and not graph.states + + +def test_state_with_other_quantization_stays_in_the_program(): + exported, _, report = lower_stateful(ScaledState()) + assert report["delegated_buffers"] == [] + assert list(exported.graph_signature.buffers_to_mutate.values()) == ["state"] + + +@pytest.mark.parametrize( + "dtype, reader, writer, writer_scale, owned", + [ + (torch.float32, "quantize", "dequantize", 0.1, True), + (torch.int8, "dequantize", "quantize", 0.1, True), + (torch.int8, "dequantize", "quantize", 0.2, False), + (torch.int8, "quantize", "dequantize", 0.1, False), + ], +) +def test_int8_state_compatibility(dtype, reader, writer, writer_scale, owned): + ops = torch.ops.quantized_decomposed + graph = torch.fx.Graph() + buffer = graph.placeholder("buffer") + buffer.meta["val"] = torch.zeros(2, dtype=dtype) + read = graph.call_function( + getattr(ops, f"{reader}_per_tensor").default, + (buffer, 0.1, 0, -128, 127, torch.int8), + ) + new_value = graph.call_function( + getattr(ops, f"{writer}_per_tensor").default, + (read, writer_scale, 0, -128, 127, torch.int8), + ) + assert partitioner._int8_state_compatible(buffer, new_value) == owned + + +@pytest.mark.parametrize("model_class", [ReturnsOldState, ReturnsClonedOldState]) +def test_returned_old_state_is_a_copy(model_class): + # A state cannot also be a graph output, so the graph returns a copy of it. + exported, _, report = lower_input_as_state(model_class) + assert report["delegated_buffers"] == ["state"] + (graph,) = delegate_graphs(exported) + ((state, _),) = graph.states + assert state not in graph.outputs + (copy,) = [op for op in graph.operations if op.inputs == [state]] + assert copy.opcode == xts3.XTS3OpCode.COPY and copy.outputs[0] in graph.outputs + + +def test_state_updated_from_a_graph_input(): + exported, _, report = lower_input_as_state() + assert report["delegated_buffers"] == ["state"] + (graph,) = delegate_graphs(exported) + ((state, update),) = graph.states + # The runtime cannot write a graph input into the state's slot, so the + # update is a copy of it, made after every other operation. + assert update not in graph.inputs + copy = graph.operations[-1] + assert copy.opcode == xts3.XTS3OpCode.COPY and copy.outputs == [update] + assert copy.inputs == graph.inputs + emulator = ProgramEmulator(exported) + frames = [torch.full((1, 4, 1, 1), 0.1 * k) for k in (1, 2, 3)] + previous = torch.zeros(1, 4, 1, 1) + for frame in frames: + expected = InputAsState.qdq( + InputAsState.qdq(previous, 0.05) + InputAsState.qdq(frame, 0.05), 0.1 + ) + torch.testing.assert_close(emulator(frame), expected) + previous = frame + + +def test_state_in_two_partitions_stays_in_the_program(monkeypatch): + exported, _, report = lower_stateful(TwoStreams(), inputs=2) + assert sorted(report["delegated_buffers"]) == ["other", "state"] + # A reset could reach one partition's state and not yet the other's, so + # neither delegate owns its buffer. + split_streams(monkeypatch) + exported, _, report = lower_stateful( + TwoStreams(), inputs=2, allow_cpu_fallback=True + ) + assert report["partitions"] == 2 + assert report["delegated_buffers"] == [] + assert sorted(exported.graph_signature.buffers_to_mutate.values()) == [ + "other", + "state", + ] + assert not any(graph.states for graph in delegate_graphs(exported)) + + +def test_a_delegate_owns_at_most_16_buffers(): + model = ManyStates(xts3.MAX_GRAPH_STATES + 1) + exported, _, report = lower_stateful(model) + assert len(report["delegated_buffers"]) == xts3.MAX_GRAPH_STATES + assert len(exported.graph_signature.buffers_to_mutate) == 1 + (graph,) = delegate_graphs(exported) + assert len(graph.states) == xts3.MAX_GRAPH_STATES + + +@pytest.mark.parametrize( + "model, example", + [ + (StateOnlyInput(), ()), + (StateOnlyOutput(), (torch.zeros(1, 4, 1, 1),)), + ], + ids=["only-input", "only-output"], +) +def test_state_that_is_a_partition_boundary_stays_in_the_program(model, example): + # Owning the buffer would leave the partition without a runtime input or + # output, which an XTS3 graph needs. + program = torch.export.export(model.eval(), example, strict=True) + edge, report = to_espnn_edge(program, delegate_mutable_buffers=True) + assert report["partitions"] == 1 and report["delegated_buffers"] == [] + edge.to_executorch() + + +def test_constant_shared_with_the_cpu_is_not_a_runtime_output(monkeypatch): + # tag_constant_data gives a constant that the CPU also reads to the + # partition, as it would one that the partition reads too, and lowering + # then duplicates it for the CPU, so it is not a runtime output. + tag_constants = partitioner.tag_constant_data + + def tag_with_offset(program): + tag_constants(program) + buffers = program.graph_signature.inputs_to_buffers + nodes = list(program.graph.nodes) + (tag,) = { + node.meta["delegation_tag"] + for node in nodes + if "delegation_tag" in node.meta + } + (offset,) = [node for node in nodes if buffers.get(node.name) == "offset"] + offset.meta["delegation_tag"] = tag + + monkeypatch.setattr(partitioner, "tag_constant_data", tag_with_offset) + program = torch.export.export( + StateOnlyOutputWithConstant().eval(), (torch.zeros(1, 4, 1, 1),), strict=True + ) + edge, report = to_espnn_edge(program, delegate_mutable_buffers=True) + assert report["delegated_buffers"] == [] + edge.to_executorch() diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py index 00077357b9a..4ef71fd7ddd 100644 --- a/backends/espressif/test/test_xts3_parser_host.py +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -88,7 +88,11 @@ def _layout(graph: xts3.XTS3Graph): io_table = xts3.IO_TABLE_HEADER_SIZE + 4 * ( len(graph.inputs) + len(graph.outputs) ) + if graph.version >= 8: + io_table += xts3.STATE_TABLE_HEADER_SIZE + 8 * len(graph.states) tensors = xts3.HEADER_SIZE + io_table + if graph.version >= 7: + tensors += -tensors % 4 operations = tensors + xts3.TENSOR_SIZE * len(graph.tensors) parameters = operations + xts3.OP_SIZE * len(graph.operations) offsets, cursor = [], parameters @@ -152,6 +156,62 @@ def test_parser_rejects_oversized_upsample_tables(parser): assert parser({"upsample": corrupted}) == {"upsample": "reject"} +def _wide_graph(opcode, width, params, weight=False): + """One operation from a [1, 1, 1, width - 1] input to a [1, 1, 1, width] + output, with a [1, 1, 1, 2] weight and a zero bias when weight is set.""" + activation = xts3.TENSOR_FLAG_QUANTIZED + tensors = [ + xts3.XTS3Tensor(0, (1, 1, 1, width - 1), xts3.XTS3DType.INT8), + xts3.XTS3Tensor(1, (1, 1, 1, width), xts3.XTS3DType.INT8), + ] + tensors[0].flags = activation | xts3.TENSOR_FLAG_GRAPH_INPUT + tensors[1].flags = activation | xts3.TENSOR_FLAG_GRAPH_OUTPUT + for tensor in tensors: + tensor.quant_scale = 1.0 + inputs = [0] + constants = b"" + if weight: + constant = xts3.TENSOR_FLAG_CONSTANT + tensors += [ + xts3.XTS3Tensor(2, (1, 1, 1, 2), xts3.XTS3DType.INT8, 0, 2, constant), + xts3.XTS3Tensor(3, (1,), xts3.XTS3DType.INT32, 16, 4, constant), + ] + inputs += [2, 3] + constants = bytes(20) + graph = xts3.XTS3Graph( + tensors=tensors, + operations=[xts3.XTS3Operation(0, opcode, inputs, [1], params)], + constant_data=constants, + inputs=[0], + outputs=[1], + ) + return xts3.serialize_graph(graph) + + +def test_parser_rejects_outputs_wider_than_esp_nn_supports(parser, monkeypatch): + # ESP-NN's pooling and convolution kernels take uint16_t dimensions: a + # (1, 2) window with padding (0, 1) widens 65535 to 65536. The serializer + # refuses such graphs too, so its check is skipped to build them. + monkeypatch.setattr(xts3, "check_runtime_limits", lambda graph: None) + pool = struct.pack(xts3.INT8_POOL_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, -128, 127) + conv = struct.pack( + xts3.INT8_CONV_BASE_PARAMS, 1, 2, 1, 1, 0, 1, 1, 1, 1, -128, 127 + ) + struct.pack("<2i", 1 << 30, 0) + blobs = {} + for name, opcode, params, weight in [ + ("pool", xts3.XTS3OpCode.MAX_POOL2D, pool, False), + ("conv", xts3.XTS3OpCode.CONV2D, conv, True), + ]: + blobs[f"{name}_widest"] = _wide_graph(opcode, 65535, params, weight) + blobs[f"{name}_too_wide"] = _wide_graph(opcode, 65536, params, weight) + assert parser(blobs) == { + "pool_widest": "accept", + "pool_too_wide": "reject", + "conv_widest": "accept", + "conv_too_wide": "reject", + } + + def test_parser_rejects_misaligned_int32_constants(parser): graph = _lowered(ops.DetectionHead(), [(1, 3, 16, 16)]) blob = xts3.serialize_graph(graph) @@ -168,3 +228,39 @@ def test_parser_rejects_misaligned_int32_constants(parser): "aligned": "accept", "misaligned": "reject", } + + +def test_parser_validates_state_tables(parser): + from executorch.backends.espressif.test import test_mutable_buffers as stateful + + exported, _, _ = stateful.lower_stateful(stateful.ShiftRegister()) + (graph,) = stateful.delegate_graphs(exported) + blob = xts3.serialize_graph(graph) + assert graph.version == xts3.XTS3_STATE_VERSION + ((state, update),) = graph.states + table = xts3.HEADER_SIZE + xts3.IO_TABLE_HEADER_SIZE + table += 4 * (len(graph.inputs) + len(graph.outputs)) + tensors, _ = _layout(graph) + flags = tensors + xts3.TENSOR_SIZE * state + 9 + blobs = { + "valid": blob, + "unpaired": _patch(blob, table, "