Skip to content

Commit 87d6190

Browse files
authored
Arm backend: Restore inputs between profiling iterations (#24283)
Restore input tensors between subsequent inferences so that an output from a previous execution cannot become an input in the next iteration. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Emma Kujala <Emma.Kujala@arm.com>
1 parent 7586c16 commit 87d6190

3 files changed

Lines changed: 91 additions & 29 deletions

File tree

‎backends/arm/runtime/tests/ethos-u/CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,16 @@ set_tests_properties(
8888
RESOURCE_LOCK arm_ethos_u_runtime
8989
)
9090

91+
add_test(
92+
NAME runner_input_restoration
93+
COMMAND "${_ethos_u_runtime_tests_dir}/test_runner_input_restoration.sh"
94+
"${EXECUTORCH_ROOT}" "${_ethos_u_runtime_test_output_dir}"
95+
)
96+
set_tests_properties(
97+
runner_input_restoration PROPERTIES LABELS "arm;ethos-u;runtime;runner;fvp"
98+
RESOURCE_LOCK arm_ethos_u_runtime
99+
)
100+
91101
add_test(
92102
NAME memory_allocation_planned_fast
93103
COMMAND "${_ethos_u_runtime_tests_dir}/test_memory_allocation_planned_fast.sh"
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Copyright 2026 Arm Limited and/or its affiliates.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
set -eo pipefail
8+
9+
source "$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)/setup_env.sh"
10+
11+
test_dir="${test_output_dir}/runner_input_restoration"
12+
mkdir -p "${test_dir}"
13+
14+
for target in ethos-u55-128 ethos-u85-128; do
15+
log_file="${test_dir}/${target}.log"
16+
17+
# add.py reuses the input arena for its output. Without restoration,
18+
# x + x + 2 produces 4, 10, 22 instead of 4 on every iteration.
19+
"${et_root_dir}/examples/arm/run.sh" \
20+
--et_build_root="${test_dir}" \
21+
--output="${test_dir}/${target}" \
22+
--target="${target}" \
23+
--model_name="${et_root_dir}/examples/arm/example_modules/add.py" \
24+
--no_delegate \
25+
--no_quantize \
26+
--select_ops_list="aten::add.out" \
27+
--bundleio \
28+
--extra_build_flags="-DET_NUM_INFERENCES=3" \
29+
2>&1 | tee "${log_file}"
30+
31+
grep -q "3 inferences finished" "${log_file}"
32+
grep -q "Test_result: PASS" "${log_file}"
33+
done

‎examples/arm/executor_runner/arm_executor_runner.cpp‎

Lines changed: 48 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,16 @@ template <typename ValueType>
464464

465465
Error prepare_input_tensors(
466466
Method& method,
467-
MemoryAllocator& allocator,
468-
const std::vector<std::pair<char*, size_t>>& input_buffers) {
467+
EValue* input_evalues,
468+
const std::vector<std::pair<char*, size_t>>& input_buffers,
469+
ET_UNUSED const void* bundled_model_data = nullptr) {
470+
#if defined(ET_BUNDLE_IO)
471+
if (bundled_model_data != nullptr) {
472+
return executorch::bundled_program::load_bundled_input(
473+
method, bundled_model_data, testset_idx);
474+
}
475+
#endif
476+
469477
MethodMeta method_meta = method.method_meta();
470478
size_t num_inputs = method_meta.num_inputs();
471479

@@ -476,12 +484,6 @@ Error prepare_input_tensors(
476484
"Wrong number of inputs allocated compared to method");
477485
#endif
478486

479-
EValue* input_evalues = allocator.allocateList<EValue>(num_inputs);
480-
ET_CHECK_OR_RETURN_ERROR(
481-
input_evalues != nullptr,
482-
MemoryAllocationFailed,
483-
"Could not allocate memory for input evalues.");
484-
485487
Error err = method.get_inputs(input_evalues, num_inputs);
486488
ET_CHECK_OK_OR_RETURN_ERROR(err);
487489

@@ -618,6 +620,7 @@ struct RunnerContext {
618620
Box<HierarchicalAllocator> planned_memory;
619621
Box<MemoryManager> memory_manager;
620622
Box<Result<Method>> method;
623+
EValue* input_evalues = nullptr;
621624
#if defined(ET_EVENT_TRACER_ENABLED)
622625
Box<ETDumpGen> etdump_gen;
623626
#if defined(ET_DUMP_INTERMEDIATE_OUTPUTS) || defined(ET_DUMP_OUTPUTS)
@@ -890,24 +893,28 @@ void runner_init(
890893
size_t input_membase = ctx.method_allocator->used_size();
891894

892895
#if defined(ET_BUNDLE_IO)
893-
if (ctx.bundle_io) {
894-
// Get inputs from bundled IO ".bpte" data
895-
// Useful for testing
896-
ET_LOG(Info, "Input testset[%d] from bundled bpte", testset_idx);
897-
Error status = executorch::bundled_program::load_bundled_input(
898-
*ctx.method.value(), model_data, testset_idx);
899-
ET_CHECK_MSG(
900-
status == Error::Ok,
901-
"load_bundled_input failed with status 0x%" PRIx32,
902-
status);
903-
} else
896+
if (!ctx.bundle_io
897+
#if defined(SEMIHOSTING)
898+
|| ctx.server_mode
899+
#endif
900+
)
904901
#endif
905902
{
906-
Error status = ::prepare_input_tensors(
907-
*ctx.method.value(), ctx.method_allocator.value(), input_buffers);
903+
const size_t num_inputs = ctx.method.value()->inputs_size();
904+
ctx.input_evalues = ctx.method_allocator->allocateList<EValue>(num_inputs);
908905
ET_CHECK_MSG(
909-
status == Error::Ok, "Failed to prepare inputs 0x%" PRIx32, status);
906+
ctx.input_evalues != nullptr || num_inputs == 0,
907+
"Could not allocate memory for input evalues.");
910908
}
909+
Error input_status = ::prepare_input_tensors(
910+
*ctx.method.value(),
911+
ctx.input_evalues,
912+
input_buffers,
913+
ctx.bundle_io ? model_data : nullptr);
914+
ET_CHECK_MSG(
915+
input_status == Error::Ok,
916+
"Failed to prepare inputs 0x%" PRIx32,
917+
input_status);
911918
#if defined(ET_LOG_DUMP_INPUT)
912919
{
913920
std::vector<EValue> inputs((*ctx.method.value())->inputs_size());
@@ -1333,7 +1340,10 @@ bool verify_result(RunnerContext& ctx, const void* model_data) {
13331340
return model_ok;
13341341
}
13351342

1336-
bool run_model(RunnerContext& ctx, const void* model_data) {
1343+
bool run_model(
1344+
RunnerContext& ctx,
1345+
const void* model_data,
1346+
const std::vector<std::pair<char*, size_t>>& input_buffers) {
13371347
Error status;
13381348
ET_LOG(
13391349
Info,
@@ -1344,7 +1354,16 @@ bool run_model(RunnerContext& ctx, const void* model_data) {
13441354
for (n = 0; n < num_inferences; n++) {
13451355
ET_LOG(
13461356
Debug, "Running inference number %lu", static_cast<printf_size_t>(n));
1347-
// Run the model.
1357+
if (n > 0) {
1358+
status = ::prepare_input_tensors(
1359+
*ctx.method.value(),
1360+
ctx.input_evalues,
1361+
input_buffers,
1362+
ctx.bundle_io ? model_data : nullptr);
1363+
if (status != Error::Ok) {
1364+
break;
1365+
}
1366+
}
13481367
status = ctx.method.value()->execute();
13491368
if (status != Error::Ok) {
13501369
break;
@@ -1408,7 +1427,7 @@ bool run_model_server(
14081427
status == Error::Ok, "Could not reload inputs: 0x%" PRIx32, status);
14091428

14101429
status = ::prepare_input_tensors(
1411-
*ctx.method.value(), ctx.temp_allocator.value(), input_buffers);
1430+
*ctx.method.value(), ctx.input_evalues, input_buffers);
14121431
ET_CHECK_MSG(
14131432
status == Error::Ok, "Failed to prepare inputs 0x%" PRIx32, status);
14141433

@@ -1544,8 +1563,8 @@ int main(int argc, const char* argv[]) {
15441563
// Byte 4-7 is usually a nice magic number that could be good to print to make
15451564
// sure it's OK ETxx for PTE and BPxx for bundled pte where xx is a number.
15461565
// cppcheck-suppress knownConditionTrueFalse
1547-
if (model_data == nullptr || model_size == 0) {
1548-
ET_LOG(Fatal, "Model data is not initialized");
1566+
if (model_data == nullptr || model_size < 8) {
1567+
ET_LOG(Fatal, "Model data is missing or too short");
15491568
return 1;
15501569
}
15511570
#if defined(SEMIHOSTING)
@@ -1578,15 +1597,15 @@ int main(int argc, const char* argv[]) {
15781597
if (ctx.server_mode) {
15791598
model_ok = run_model_server(ctx, model_data, input_buffers);
15801599
} else {
1581-
model_ok = run_model(ctx, model_data);
1600+
model_ok = run_model(ctx, model_data, input_buffers);
15821601
}
15831602
#if defined(ETHOSU55)
15841603
if (keep_powered) {
15851604
set_runner_power(false);
15861605
}
15871606
#endif
15881607
#else
1589-
model_ok = run_model(ctx, model_data);
1608+
model_ok = run_model(ctx, model_data, input_buffers);
15901609
#endif
15911610
ET_LOG(Info, "Model run: %d", model_ok);
15921611

0 commit comments

Comments
 (0)