Skip to content

Commit 493a634

Browse files
winskuo-quicwinskuo-quic
committed
Qualcomm AI Engine Direct - Minimal Inerence Runtime Core Requirment
1. Removed from_blob tensor creation 2. Compile and Linking Option optimization 3. Function visibility optimization 4. Expose Power Config to user
1 parent 94d70b9 commit 493a634

11 files changed

Lines changed: 128 additions & 26 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,17 @@ project(executorch)
5050

5151
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
5252

53+
# Hexagon toolchain with release build complains about code in third party
54+
# libraries.
55+
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon AND ${CMAKE_BUILD_TYPE} STREQUAL
56+
"Release"
57+
)
58+
add_compile_options(
59+
-Wno-error=format -Wno-error=implicit-int-conversion
60+
-Wno-error=unused-variable -Wno-error=unused-function
61+
)
62+
endif()
63+
5364
# --- ExecuTorch Version ---
5465
# Parse version from version.txt (single source of truth)
5566
file(READ "${EXECUTORCH_ROOT}/version.txt" ET_VERSION_STRING)

‎backends/qualcomm/CMakeLists.txt‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -77,22 +77,34 @@ if(${ANDROID})
7777
find_library(android_log log)
7878
endif()
7979

80-
add_compile_options("-Wall" "-Werror" "-Wno-sign-compare")
80+
add_compile_options("-Wall" "-Werror" "-fvisibility=hidden")
8181
add_compile_definitions(C10_USING_CUSTOM_GENERATED_MACROS)
8282

83-
# GNU emit wanring for ignored attributes Unfortunately, we use [[maybe_unused]]
84-
# which can be ignored by GNU. So we make it a warning, not an error in GNU.
83+
# GNU emits warning for ignored attributes Unfortunately, we use
84+
# [[maybe_unused]] which can be ignored by GNU. So we make it a warning, not an
85+
# error in GNU.
8586
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
8687
add_compile_options("-Wno-error=attributes")
8788
add_link_options("-flto=auto")
8889
endif()
8990

9091
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
9192
# strip symbols
92-
add_link_options("-s")
93+
add_link_options(LINKER:-s,--gc-sections)
94+
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon)
95+
add_compile_options(
96+
"-Os"
97+
"-ffunction-sections"
98+
"-fdata-sections"
99+
"-frtti"
100+
"-fno-exceptions"
101+
"-fomit-frame-pointer"
102+
"-fno-asynchronous-unwind-tables"
103+
)
104+
else()
93105

94-
# --gc-sections is added by torch.
95-
add_compile_options("-O3" "-ffunction-sections" "-fdata-sections" "-frtti")
106+
add_compile_options("-O3" "-ffunction-sections" "-fdata-sections" "-frtti")
107+
endif()
96108
endif()
97109

98110
include_directories(
@@ -230,9 +242,8 @@ target_link_libraries(
230242
qnn_schema shared_buffer qnn_dlc_manager
231243
)
232244
target_link_libraries(
233-
qnn_executorch_backend
234-
PRIVATE qnn_executorch_header qnn_schema qnn_manager executorch_core
235-
extension_tensor qnn_backend_options
245+
qnn_executorch_backend PRIVATE qnn_executorch_header qnn_schema qnn_manager
246+
executorch_core qnn_backend_options
236247
)
237248

238249
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon)

‎backends/qualcomm/aot/python/PyQnnManagerAdaptor.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ class PyQnnManager {
250250
std::vector<std::vector<std::shared_ptr<OpWrapper>>>& op_wrappers) {
251251
QnnExecuTorchContextBinary binary_info;
252252

253-
for (int i = 0; i < graph_names.size(); ++i) {
253+
for (uint32_t i = 0; i < graph_names.size(); ++i) {
254254
if (qnn_manager_->Compile(graph_names[i], op_wrappers[i]) !=
255255
executorch::runtime::Error::Ok) {
256256
QNN_EXECUTORCH_LOG_ERROR("Fail to compile QNN graph");

‎backends/qualcomm/aot/wrappers/TensorWrapper.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ class TensorWrapper {
7676
rank);
7777
return;
7878
}
79-
for (int i = 0; i < rank; ++i) {
79+
for (size_t i = 0; i < rank; ++i) {
8080
QNN_TENSOR_VER_PTR(tensor_)->dimensions[i] = dims[i];
8181
}
8282
}

‎backends/qualcomm/runtime/QnnExecuTorch.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,14 +63,18 @@ struct CustomMemTensorInfo {
6363
/// alignment as MemoryAllocator::kDefaultAlignment.
6464
/// See runtime/core/memory_allocator.h. The function returns a valid pointer
6565
/// if allocation is successful.
66-
void* QnnExecuTorchAllocCustomMem(size_t bytes, size_t alignment);
66+
__attribute__((__visibility__("default"))) void* QnnExecuTorchAllocCustomMem(
67+
size_t bytes,
68+
size_t alignment);
6769

6870
/// Add tensor to custom memory with custom type descriptor. Create memory
6971
/// handle to tensor wrapper during execution
70-
void QnnExecuTorchAddCustomMemTensorAddr(void* tensor_addr, void* custom_mem);
72+
__attribute__((__visibility__("default"))) void
73+
QnnExecuTorchAddCustomMemTensorAddr(void* tensor_addr, void* custom_mem);
7174

7275
/// Free the allocated shared memory.
73-
void QnnExecuTorchFreeCustomMem(void* buffer_ptr);
76+
__attribute__((__visibility__("default"))) void QnnExecuTorchFreeCustomMem(
77+
void* buffer_ptr);
7478

7579
#ifdef __cplusplus
7680
}

‎backends/qualcomm/runtime/QnnManager.cpp‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
#include <executorch/backends/qualcomm/runtime/backends/QnnBackendCommon.h>
1313
#include <executorch/backends/qualcomm/runtime/backends/QnnCustomProtocol.h>
1414
#include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>
15-
#include <executorch/extension/tensor/tensor.h>
1615
#include <algorithm>
1716
#include <cstdlib>
1817
#include <cstring>
@@ -427,19 +426,25 @@ Error QnnManager::Execute(
427426
QNN_TENSOR_VER_PTR(output_tensor)->dimensions +
428427
QNN_TENSOR_VER_PTR(output_tensor)->rank);
429428

430-
auto dump_tensor = executorch::extension::from_blob(
431-
QNN_TENSOR_VER_PTR(output_tensor)->clientBuf.data,
432-
sizes,
429+
std::vector<executorch::aten::StridesType> stride_size(sizes.size(), 0);
430+
// Avoid using from_blob as it significantly increases shared library
431+
// size.
432+
executorch::aten::TensorImpl tensor_impl(
433433
qnn_dtype_to_scalar_type_[QNN_TENSOR_VER_PTR(output_tensor)
434-
->dataType]);
434+
->dataType],
435+
sizes.size(),
436+
sizes.data(),
437+
QNN_TENSOR_VER_PTR(output_tensor)->clientBuf.data,
438+
nullptr,
439+
stride_size.data());
435440

436441
executorch::runtime::event_tracer_log_output_delegate<
437442
executorch::aten::Tensor>(
438443
event_tracer,
439444
QNN_TENSOR_VER_PTR(output_tensor)->name,
440445
/*delegate_debug_id=*/
441446
static_cast<executorch::runtime::DebugHandle>(-1),
442-
*dump_tensor);
447+
executorch::aten::Tensor(&tensor_impl));
443448
}
444449
}
445450

@@ -547,7 +552,7 @@ Error QnnManager::CompileDlc() {
547552

548553
// Mapping memory address for the input and output of mutable buffer
549554
std::unordered_map<int, const void*> mutable_buffer_id_to_memory_map;
550-
for (int i = 0; i < graphInfo.numInputTensors; ++i) {
555+
for (uint32_t i = 0; i < graphInfo.numInputTensors; ++i) {
551556
auto tw = CreateTensorWrapper(graphInfo.inputTensors[i]);
552557
tw->UpdateQnnTensorMeta(graphInfo.inputTensors[i]);
553558

@@ -560,7 +565,7 @@ Error QnnManager::CompileDlc() {
560565
}
561566
graph_inputs.push_back(tw);
562567
}
563-
for (int i = 0; i < graphInfo.numOutputTensors; ++i) {
568+
for (uint32_t i = 0; i < graphInfo.numOutputTensors; ++i) {
564569
auto tw = CreateTensorWrapper(graphInfo.outputTensors[i]);
565570
tw->UpdateQnnTensorMeta(graphInfo.outputTensors[i]);
566571
int mutable_buffer_id = ExtractMutableBufferNumber(tw->GetName());

‎backends/qualcomm/runtime/SharedBuffer.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ std::size_t std::hash<CustomMemTensorInfo>::operator()(
2121
hash_val ^= std::hash<void*>()(info.custom_mem);
2222
hash_val ^= std::hash<size_t>()(info.pos);
2323
hash_val ^= std::hash<size_t>()(info.tensor_bytes);
24-
for (int i = 0; i < info.rank; ++i) {
24+
for (size_t i = 0; i < info.rank; ++i) {
2525
hash_val ^= std::hash<uint32_t>()(info.shape[i]);
2626
}
2727
hash_val ^= std::hash<uint32_t>()(info.rank);
@@ -36,7 +36,7 @@ bool operator==(
3636
(lhs.tensor_addr == rhs.tensor_addr && lhs.custom_mem == rhs.custom_mem &&
3737
lhs.pos == rhs.pos && lhs.tensor_bytes == rhs.tensor_bytes &&
3838
lhs.rank == rhs.rank && lhs.dtype == rhs.dtype);
39-
for (int i = 0; i < lhs.rank; ++i) {
39+
for (size_t i = 0; i < lhs.rank; ++i) {
4040
is_same &= lhs.shape[i] == rhs.shape[i];
4141
}
4242
return is_same;

‎backends/qualcomm/runtime/backends/direct_mode/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@ target_link_libraries(
4545
${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++.so.1
4646
${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++abi.so.1
4747
)
48+
target_compile_options(qnn_executorch_skel PRIVATE "-fvisibility=default")

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from executorch.backends.qualcomm.debugger.utils import generate_optrace
3030
from executorch.backends.qualcomm.serialization.qc_schema import (
3131
QnnExecuTorchBackendType,
32+
QnnExecuTorchHtpPerformanceMode,
3233
)
3334
from executorch.backends.qualcomm.tests.utils import (
3435
convert_pt2e,
@@ -4834,6 +4835,33 @@ def setUp(self):
48344835
saver=False,
48354836
)
48364837

4838+
def test_qnn_backend_compile_time_option_htp_performance(self):
4839+
backend_options = generate_htp_compiler_spec(
4840+
use_fp16=True,
4841+
htp_performance_mode=QnnExecuTorchHtpPerformanceMode.kHtpHighPowerSaver,
4842+
)
4843+
TestQNN.compiler_specs = generate_qnn_executorch_compiler_spec(
4844+
soc_model=self.chipset_table[TestQNN.model],
4845+
backend_options=backend_options,
4846+
)
4847+
module = SimpleModel() # noqa: F405
4848+
sample_input = (torch.ones(1, 32, 28, 28), torch.ones(1, 32, 28, 28))
4849+
4850+
def output_callback(log_msg):
4851+
msg = log_msg.stdout
4852+
# Refer to HtpDevice.cpp for the following values
4853+
min_voltage = "coreVoltageCornerMin 80"
4854+
self.assertTrue(min_voltage in msg, f"Expecting '{min_voltage} ' in log")
4855+
4856+
runtime_extra_commands = " --log_level 4"
4857+
self.lower_module_and_test_output(
4858+
module,
4859+
sample_input,
4860+
extra_cmds=runtime_extra_commands,
4861+
output_callback=partial(output_callback),
4862+
save_inference_speed=True,
4863+
)
4864+
48374865
def test_qnn_backend_dump_intermediate_outputs_topk(self):
48384866
TestQNN.dump_intermediate_outputs = True
48394867
backend_options = generate_htp_compiler_spec(use_fp16=True)
@@ -5423,6 +5451,34 @@ def setUp(self):
54235451
saver=False,
54245452
)
54255453

5454+
def test_qnn_backend_compile_time_option_htp_performance(self):
5455+
backend_options = generate_htp_compiler_spec(
5456+
use_fp16=False,
5457+
htp_performance_mode=QnnExecuTorchHtpPerformanceMode.kHtpHighPowerSaver,
5458+
)
5459+
TestQNN.compiler_specs = generate_qnn_executorch_compiler_spec(
5460+
soc_model=self.chipset_table[TestQNN.model],
5461+
backend_options=backend_options,
5462+
)
5463+
module = SimpleModel() # noqa: F405
5464+
sample_input = (torch.ones(1, 32, 28, 28), torch.ones(1, 32, 28, 28))
5465+
module = self.get_qdq_module(module, sample_input)
5466+
5467+
def output_callback(log_msg):
5468+
msg = log_msg.stdout
5469+
# Refer to HtpDevice.cpp for the following values
5470+
min_voltage = "coreVoltageCornerMin 80"
5471+
self.assertTrue(min_voltage in msg, f"Expecting '{min_voltage} ' in log")
5472+
5473+
runtime_extra_commands = " --log_level 4"
5474+
self.lower_module_and_test_output(
5475+
module,
5476+
sample_input,
5477+
extra_cmds=runtime_extra_commands,
5478+
output_callback=partial(output_callback),
5479+
save_inference_speed=True,
5480+
)
5481+
54265482
def test_qnn_backend_dump_intermediate_outputs_simple_model(self):
54275483
TestQNN.dump_intermediate_outputs = True
54285484
backend_options = generate_htp_compiler_spec(use_fp16=False)

‎backends/qualcomm/utils/utils.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -994,6 +994,7 @@ def generate_htp_compiler_spec(
994994
use_multi_contexts: bool = False,
995995
use_weight_sharing: bool = False,
996996
use_slc_allocator: bool = False,
997+
htp_performance_mode: QnnExecuTorchHtpPerformanceMode = QnnExecuTorchHtpPerformanceMode.kHtpBurst,
997998
) -> QnnExecuTorchBackendOptions:
998999
"""
9991000
Helper function generating backend options for QNN HTP
@@ -1025,7 +1026,7 @@ def generate_htp_compiler_spec(
10251026
# This actually is not an option which can affect the compiled blob.
10261027
# But we don't have other place to pass this option at execution stage.
10271028
# TODO: enable voting mechanism in runtime and make this as an option
1028-
htp_options.performance_mode = QnnExecuTorchHtpPerformanceMode.kHtpBurst
1029+
htp_options.performance_mode = htp_performance_mode
10291030
htp_options.use_multi_contexts = use_multi_contexts
10301031
htp_options.use_weight_sharing = use_weight_sharing
10311032
htp_options.use_dlbc = use_dlbc

0 commit comments

Comments
 (0)