Skip to content

Commit 63f9724

Browse files
authored
3/x: Wire LoadBackendOptionsMap through Program and Method
Differential Revision: D92461088 Pull Request resolved: pytorch#17531
1 parent 6e59981 commit 63f9724

7 files changed

Lines changed: 208 additions & 14 deletions

File tree

.github/workflows/trunk.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -400,12 +400,12 @@ jobs:
400400
setup_script_args=""
401401
if [[ ${{ matrix.os}} == "bare_metal" ]]; then
402402
toolchain_prefix=arm-none-eabi-
403-
threshold="110592" # 108 KiB
403+
threshold="110642" # 108 KiB
404404
toolchain_cmake=examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
405405
elif [[ ${{ matrix.os}} == "zephyr-preset" ]]; then
406406
setup_script_args="--target-toolchain zephyr"
407407
toolchain_prefix=arm-zephyr-eabi-
408-
threshold="136000" # 136 KiB
408+
threshold="136020" # 136 KiB
409409
toolchain_cmake=examples/zephyr/x86_64-linux-arm-zephyr-eabi-gcc.cmake
410410
else
411411
echo "Fail unsupport OS selection ${{ matrix.os }}"
@@ -875,7 +875,7 @@ jobs:
875875
qwen3-0.6b|xnnpack|--quantize,
876876
qwen3-1.7b|xnnpack|--quantize,
877877
gemma3-1b|xnnpack|--quantize,
878-
# phi4-mini|xnnpack|--quantize, transformers v5.0.0rc0 introduces a data-dependent branching in transformers/modeling_rope_utils.py:61
878+
# phi4-mini|xnnpack|--quantize, transformers v5.0.0rc0 introduces a data-dependent branching in transformers/modeling_rope_utils.py:61
879879
smollm2-135m|xnnpack|--quantize,
880880
smollm3-3b|xnnpack|--quantize
881881
]

CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,12 @@ install(
578578
FILES_MATCHING
579579
PATTERN "*.h"
580580
)
581+
install(
582+
DIRECTORY runtime/backend/
583+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/backend
584+
FILES_MATCHING
585+
PATTERN "*.h"
586+
)
581587
install(
582588
DIRECTORY runtime/platform/
583589
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/runtime/platform

runtime/executor/method.cpp

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,8 @@ Result<Method> Method::load(
806806
const Program* program,
807807
MemoryManager* memory_manager,
808808
EventTracer* event_tracer,
809-
const NamedDataMap* external_data_map) {
809+
const NamedDataMap* external_data_map,
810+
const LoadBackendOptionsMap* backend_options) {
810811
MemoryAllocator* temp_allocator = memory_manager->temp_allocator();
811812
if (temp_allocator == nullptr) {
812813
PlatformMemoryAllocator* platform_allocator =
@@ -820,7 +821,7 @@ Result<Method> Method::load(
820821
}
821822
Method method(program, memory_manager, event_tracer, temp_allocator);
822823
ET_LOG(Debug, "Loading method: %s.", s_plan->name()->c_str());
823-
Error err = method.init(s_plan, external_data_map);
824+
Error err = method.init(s_plan, external_data_map, backend_options);
824825
if (err != Error::Ok) {
825826
return err;
826827
} else {
@@ -831,9 +832,9 @@ Result<Method> Method::load(
831832

832833
Error Method::init(
833834
executorch_flatbuffer::ExecutionPlan* s_plan,
834-
const NamedDataMap* external_data_map) {
835-
EXECUTORCH_SCOPE_PROF("Method::init");
836-
internal::EventTracerProfileMethodScope event_tracer_profile_scope =
835+
const NamedDataMap* external_data_map,
836+
const LoadBackendOptionsMap* backend_options) {
837+
internal::EventTracerProfileMethodScope event_tracer_scope =
837838
internal::EventTracerProfileMethodScope(event_tracer_, "Method::init");
838839
ET_CHECK_OR_RETURN_ERROR(
839840
// Don't use !initialized() here because we also want to fail on the
@@ -902,11 +903,21 @@ Error Method::init(
902903

903904
for (size_t i = 0; i < n_delegate; ++i) {
904905
const auto& delegate = *delegates->Get(i);
906+
907+
// Get per-delegate runtime specs from the LoadBackendOptionsMap if
908+
// provided
909+
Span<const BackendOption> delegate_runtime_specs;
910+
if (backend_options != nullptr && delegate.id() != nullptr) {
911+
delegate_runtime_specs =
912+
backend_options->get_options(delegate.id()->c_str());
913+
}
914+
905915
BackendInitContext backend_init_context(
906916
method_allocator,
907917
/*event_tracer=*/event_tracer_,
908918
/*method_name=*/serialization_plan_->name()->c_str(),
909-
/*named_data_map=*/named_data_map);
919+
/*named_data_map=*/named_data_map,
920+
/*runtime_specs=*/delegate_runtime_specs);
910921
Error err = BackendDelegate::Init(
911922
delegate, program_, backend_init_context, &delegates_[i]);
912923
if (err != Error::Ok) {

runtime/executor/method.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
1515
#endif
1616

17+
#include <executorch/runtime/backend/backend_options_map.h>
1718
#include <executorch/runtime/core/evalue.h>
1819
#include <executorch/runtime/core/event_tracer.h>
1920
#include <executorch/runtime/core/exec_aten/exec_aten.h>
@@ -355,7 +356,8 @@ class Method final {
355356
const Program* program,
356357
MemoryManager* memory_manager,
357358
EventTracer* event_tracer,
358-
const NamedDataMap* named_data_map);
359+
const NamedDataMap* named_data_map,
360+
const LoadBackendOptionsMap* backend_options = nullptr);
359361

360362
/**
361363
* Initialize the method from its serialized representation.
@@ -364,7 +366,8 @@ class Method final {
364366
*/
365367
ET_NODISCARD Error init(
366368
executorch_flatbuffer::ExecutionPlan* s_plan,
367-
const NamedDataMap* named_data_map);
369+
const NamedDataMap* named_data_map,
370+
const LoadBackendOptionsMap* backend_options);
368371

369372
/// Returns true if the Method was successfully initialized.
370373
inline bool initialized() const {

runtime/executor/program.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,8 @@ Result<Method> Program::load_method(
307307
const char* method_name,
308308
MemoryManager* memory_manager,
309309
EventTracer* event_tracer,
310-
const NamedDataMap* named_data_map) const {
310+
const NamedDataMap* named_data_map,
311+
const LoadBackendOptionsMap* backend_options) const {
311312
EXECUTORCH_SCOPE_PROF("Program::load_method");
312313
internal::event_tracer_create_event_block(event_tracer, "Default");
313314
internal::EventTracerProfileMethodScope event_tracer_scope =
@@ -325,7 +326,12 @@ Result<Method> Program::load_method(
325326
return plan.error();
326327
}
327328
return Method::load(
328-
plan.get(), this, memory_manager, event_tracer, named_data_map);
329+
plan.get(),
330+
this,
331+
memory_manager,
332+
event_tracer,
333+
named_data_map,
334+
backend_options);
329335
}
330336

331337
Result<MethodMeta> Program::method_meta(const char* method_name) const {

runtime/executor/program.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <cstdint>
1212
#include <optional>
1313

14+
#include <executorch/runtime/backend/backend_options_map.h>
1415
#include <executorch/runtime/core/data_loader.h>
1516
#include <executorch/runtime/core/error.h>
1617
#include <executorch/runtime/core/event_tracer.h>
@@ -139,14 +140,18 @@ class Program final {
139140
* @param[in] event_tracer The event tracer to use for this method run.
140141
* @param[in] named_data_map An optional map of {name, blob} used to resolve
141142
* data that is external to the PTE, if any.
143+
* @param[in] backend_options An optional map of per-backend load-time options
144+
* (RuntimeSpecs). Each backend will receive its corresponding options
145+
* during initialization.
142146
*
143147
* @returns The loaded method on success, or an error on failure.
144148
*/
145149
Result<Method> load_method(
146150
const char* method_name,
147151
MemoryManager* memory_manager,
148152
EventTracer* event_tracer = nullptr,
149-
const NamedDataMap* named_data_map = nullptr) const;
153+
const NamedDataMap* named_data_map = nullptr,
154+
const LoadBackendOptionsMap* backend_options = nullptr) const;
150155

151156
/**
152157
* Gathers metadata for the named method.

runtime/executor/test/backend_integration_test.cpp

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,169 @@ TEST_P(BackendIntegrationTest, GetMethodNameDuringExecuteSuccess) {
627627
ASSERT_EQ(err, Error::Ok);
628628
}
629629

630+
TEST_P(BackendIntegrationTest, RuntimeSpecsPassedToBackendInitContext) {
631+
using executorch::runtime::BackendOptions;
632+
using executorch::runtime::LoadBackendOptionsMap;
633+
634+
Result<FileDataLoader> loader = FileDataLoader::from(program_path());
635+
ASSERT_EQ(loader.error(), Error::Ok);
636+
637+
// Track whether init was called and what runtime_specs it received
638+
bool init_called = false;
639+
size_t received_specs_size = 0;
640+
int received_num_threads = 0;
641+
bool received_enable_profiling = false;
642+
const char* received_compute_unit = nullptr;
643+
644+
StubBackend::singleton().install_init(
645+
[&](FreeableBuffer* processed,
646+
ET_UNUSED ArrayRef<CompileSpec> compile_specs,
647+
BackendInitContext& backend_init_context) -> Result<DelegateHandle*> {
648+
init_called = true;
649+
650+
// Get runtime_specs from context
651+
received_specs_size = backend_init_context.runtime_specs().size();
652+
653+
// Use convenience methods to extract values
654+
auto num_threads_result =
655+
backend_init_context.get_runtime_spec<int>("num_threads");
656+
if (num_threads_result.ok()) {
657+
received_num_threads = num_threads_result.get();
658+
}
659+
660+
auto enable_profiling_result =
661+
backend_init_context.get_runtime_spec<bool>("enable_profiling");
662+
if (enable_profiling_result.ok()) {
663+
received_enable_profiling = enable_profiling_result.get();
664+
}
665+
666+
auto compute_unit_result =
667+
backend_init_context.get_runtime_spec<const char*>("compute_unit");
668+
if (compute_unit_result.ok()) {
669+
received_compute_unit = compute_unit_result.get();
670+
}
671+
672+
processed->Free();
673+
return nullptr;
674+
});
675+
676+
Result<Program> program = Program::load(&loader.get());
677+
ASSERT_EQ(program.error(), Error::Ok);
678+
679+
// Create backend options for StubBackend
680+
BackendOptions<4> stub_opts;
681+
stub_opts.set_option("num_threads", 8);
682+
stub_opts.set_option("enable_profiling", true);
683+
stub_opts.set_option("compute_unit", "cpu_and_gpu");
684+
685+
// Create the map and set options for StubBackend
686+
LoadBackendOptionsMap backend_options;
687+
ASSERT_EQ(
688+
backend_options.set_options(StubBackend::kName, stub_opts.view()),
689+
Error::Ok);
690+
691+
ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes);
692+
693+
// Load method with backend_options
694+
Result<Method> method = program->load_method(
695+
"forward",
696+
&mmm.get(),
697+
/*event_tracer=*/nullptr,
698+
/*named_data_map=*/nullptr,
699+
&backend_options);
700+
EXPECT_TRUE(method.ok());
701+
702+
// Verify that init was called and received the correct runtime_specs
703+
EXPECT_TRUE(init_called);
704+
EXPECT_EQ(received_specs_size, 3);
705+
EXPECT_EQ(received_num_threads, 8);
706+
EXPECT_TRUE(received_enable_profiling);
707+
ASSERT_NE(received_compute_unit, nullptr);
708+
EXPECT_STREQ(received_compute_unit, "cpu_and_gpu");
709+
}
710+
711+
TEST_P(BackendIntegrationTest, NoRuntimeSpecsWhenBackendOptionsNull) {
712+
Result<FileDataLoader> loader = FileDataLoader::from(program_path());
713+
ASSERT_EQ(loader.error(), Error::Ok);
714+
715+
// Track whether init was called and what runtime_specs it received
716+
bool init_called = false;
717+
size_t received_specs_size = 0;
718+
719+
StubBackend::singleton().install_init(
720+
[&](FreeableBuffer* processed,
721+
ET_UNUSED ArrayRef<CompileSpec> compile_specs,
722+
BackendInitContext& backend_init_context) -> Result<DelegateHandle*> {
723+
init_called = true;
724+
received_specs_size = backend_init_context.runtime_specs().size();
725+
processed->Free();
726+
return nullptr;
727+
});
728+
729+
Result<Program> program = Program::load(&loader.get());
730+
ASSERT_EQ(program.error(), Error::Ok);
731+
732+
ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes);
733+
734+
// Load method WITHOUT backend_options (nullptr)
735+
Result<Method> method = program->load_method("forward", &mmm.get());
736+
EXPECT_TRUE(method.ok());
737+
738+
// Verify that init was called but received empty runtime_specs
739+
EXPECT_TRUE(init_called);
740+
EXPECT_EQ(received_specs_size, 0);
741+
}
742+
743+
TEST_P(BackendIntegrationTest, NoRuntimeSpecsWhenBackendNotInMap) {
744+
using executorch::runtime::BackendOptions;
745+
using executorch::runtime::LoadBackendOptionsMap;
746+
747+
Result<FileDataLoader> loader = FileDataLoader::from(program_path());
748+
ASSERT_EQ(loader.error(), Error::Ok);
749+
750+
// Track whether init was called and what runtime_specs it received
751+
bool init_called = false;
752+
size_t received_specs_size = 0;
753+
754+
StubBackend::singleton().install_init(
755+
[&](FreeableBuffer* processed,
756+
ET_UNUSED ArrayRef<CompileSpec> compile_specs,
757+
BackendInitContext& backend_init_context) -> Result<DelegateHandle*> {
758+
init_called = true;
759+
received_specs_size = backend_init_context.runtime_specs().size();
760+
processed->Free();
761+
return nullptr;
762+
});
763+
764+
Result<Program> program = Program::load(&loader.get());
765+
ASSERT_EQ(program.error(), Error::Ok);
766+
767+
// Create backend options for a DIFFERENT backend (not StubBackend)
768+
BackendOptions<2> other_opts;
769+
other_opts.set_option("key", "value");
770+
771+
LoadBackendOptionsMap backend_options;
772+
ASSERT_EQ(
773+
backend_options.set_options("OtherBackend", other_opts.view()),
774+
Error::Ok);
775+
776+
ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes);
777+
778+
// Load method with backend_options that don't include StubBackend
779+
Result<Method> method = program->load_method(
780+
"forward",
781+
&mmm.get(),
782+
/*event_tracer=*/nullptr,
783+
/*named_data_map=*/nullptr,
784+
&backend_options);
785+
EXPECT_TRUE(method.ok());
786+
787+
// Verify that init was called but received empty runtime_specs
788+
// (because StubBackend wasn't in the map)
789+
EXPECT_TRUE(init_called);
790+
EXPECT_EQ(received_specs_size, 0);
791+
}
792+
630793
// TODO: Add more tests for the runtime-to-backend interface. E.g.:
631794
// - Errors during init() or execute() result in runtime init/execution failures
632795
// - Correct values are passed to init()/execute()

0 commit comments

Comments
 (0)