Qualcomm AI Engine Direct - LPAI Direct Mode Support

Author: winskuo-quic

.claude/skills/qualcomm/SKILL.md

@@ -31,7 +31,7 @@ Use `backends/qualcomm/scripts/build.sh`. Linux only (macOS not supported).
 |---|---|---|
 | x86_64 (Python interface + host tools) | enabled | `build-x86/` |
 | Android arm64-v8a (device runner) | enabled | `build-android/` |
-| Hexagon DSP (direct mode) | disabled | `build-hexagon/` |
+| Direct mode (LPAI ADSP or Hexagon CDSP) | disabled | `build-direct/` |
 | OE Linux embedded | disabled | `build-oe-linux/` |
 
 **Common build commands:**

.gitignore

@@ -18,7 +18,7 @@ cmake-out*
 cmake-out-android/
 build-android/
 build-x86/
-build-hexagon/
+build-direct/
 dist/
 arm-scratch/
 executorch.egg-info

backends/qualcomm/CMakeLists.txt

@@ -248,21 +248,31 @@ target_link_libraries(
 
 if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon)
   # Add macro here so we can dlopen the correct .so library.
-  if(DSP_TYPE STREQUAL "3")
-    string(TOUPPER ${DSP_VERSION} CAPITAL_DSP_VERSION)
-    set(HEXAGON_LIB "libQnnHtp${CAPITAL_DSP_VERSION}.so")
+  if(DSP_TYPE STREQUAL "0")
+    message(
+      STATUS
+        "aDSP direct and non-direct mode uses shared libraries under different folders but have the same name, skipping HEXAGON_LIB override."
+    )
+    # Just random string here since QnnBackendUnifiedRegistry.h will ignore this
+    # macro for aDSP case.
+    add_compile_definitions(HEXAGON_LIB="")
+  elseif(DSP_TYPE STREQUAL "3")
+    string(TOUPPER ${CDSP_VERSION} CAPITAL_CDSP_VERSION)
+    set(HEXAGON_LIB "libQnnHtp${CAPITAL_CDSP_VERSION}.so")
     add_compile_definitions(HEXAGON_LIB="${HEXAGON_LIB}")
     message(STATUS "For hexagon build, using HTP Library: ${HEXAGON_LIB}")
   else()
     message(FATAL_ERROR "Unknown DSP_TYPE ${DSP_TYPE}")
   endif()
 
+  # aDSP will also use cDSP c/c++ library since aDSP does not have these
+  # libraries.
   target_link_libraries(
     qnn_executorch_backend
     PRIVATE
-      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc.so
-      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++.so.1
-      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++abi.so.1
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${CDSP_VERSION}/G0/pic/libc.so
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${CDSP_VERSION}/G0/pic/libc++.so.1
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${CDSP_VERSION}/G0/pic/libc++abi.so.1
   )
 endif()
 

backends/qualcomm/aot/wrappers/TensorWrapper.cpp

@@ -7,6 +7,7 @@
  */
 
 #include <executorch/backends/qualcomm/aot/wrappers/TensorWrapper.h>
+#include <executorch/backends/qualcomm/runtime/backends/QnnCustomProtocol.h>
 
 #include <atomic>
 #include <cstring>
@@ -115,19 +116,24 @@ TensorWrapper::TensorWrapper(
   }
 }
 
-Error TensorWrapper::FillDataBuffer(const void* data, bool copy_data) {
+Error TensorWrapper::FillDataBuffer(const void* data) {
   if (data != nullptr) {
     QNN_TENSOR_VER_PTR(tensor_)->memType = QNN_TENSORMEMTYPE_RAW;
+#ifdef __hexagon__
+    // data's address is already aligned in idl skel implementation. e.g.
+    // QnnExecuTorchIdlWrapper.cpp Here, we are ensuring we pass data size that
+    // is also multiple of 64. QnnExecuTorchIdlWrapper.cpp should have created
+    // sufficient space for tensor.
+    auto align_size = [](size_t alignment, size_t sz) {
+      return (sz + (alignment - 1)) & ~(alignment - 1);
+    };
+    QNN_TENSOR_VER_PTR(tensor_)->clientBuf.dataSize =
+        align_size(QNN_TENSOR_ALIGNMENT, bytes_);
+#else
     QNN_TENSOR_VER_PTR(tensor_)->clientBuf.dataSize = bytes_;
-    if (copy_data) {
-      owned_data_ = std::make_unique<char[]>(bytes_);
-      const char* src_data = static_cast<const char*>(data);
-      std::memcpy(owned_data_.get(), src_data, bytes_);
-      QNN_TENSOR_VER_PTR(tensor_)->clientBuf.data = owned_data_.get();
-    } else {
-      // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
-      QNN_TENSOR_VER_PTR(tensor_)->clientBuf.data = const_cast<void*>(data);
-    }
+#endif
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
+    QNN_TENSOR_VER_PTR(tensor_)->clientBuf.data = const_cast<void*>(data);
   } else {
     QNN_EXECUTORCH_LOG_WARN("Data pointer is nullptr");
   }

backends/qualcomm/aot/wrappers/TensorWrapper.h

@@ -36,9 +36,7 @@ class TensorWrapper {
       const void* data = nullptr,
       bool copy_data = false);
 
-  executorch::runtime::Error FillDataBuffer(
-      const void* data,
-      bool copy_data = false);
+  executorch::runtime::Error FillDataBuffer(const void* data);
 
   executorch::runtime::Error AllocateDataBuffer();
 

backends/qualcomm/export_utils.py

@@ -34,10 +34,10 @@
     QcomChipset,
     QnnExecuTorchBackendType,
     QnnExecuTorchHtpPerformanceMode,
+    QnnExecuTorchLpaiTargetEnv,
     QnnExecuTorchOpPackageOptions,
 )
 from executorch.backends.qualcomm.utils.constants import (
-    DSP_VERSION,
     HEXAGON_SDK_ROOT,
     HEXAGON_TOOLS_ROOT,
 )
@@ -76,7 +76,7 @@ class QnnConfig:
         backend (str): The target backend, such as htp, gpu, etc. QnnConfig will then parse this to type QnnExecuTorchBackendType.
         soc_model (QcomChipset): The target Qualcomm System on Chip (SoC) model.
         build_folder (str): Path to cmake binary directory for target platform, e.g., /path/to/build-android.
-        direct_build_folder (str): Path to cmake binary directory for direct_mode. E.g., path/to/build-hexagon.
+        direct_build_folder (str): Path to cmake binary directory for direct_mode. E.g., path/to/build-direct.
         target (str): Target platform for deployment.
         online_prepare (bool): Compose QNN graph on device if set to True.
         shared_buffer (bool): Enables usage of shared buffer(zero-copy mechanism) between application and backend for graph I/O during runtime.
@@ -235,16 +235,14 @@ def __init__(
             )
         self.runner = runner
         if qnn_config.direct_build_folder:
-            required_env = [HEXAGON_SDK_ROOT, HEXAGON_TOOLS_ROOT, DSP_VERSION]
+            required_env = [HEXAGON_SDK_ROOT, HEXAGON_TOOLS_ROOT]
             assert all(
                 var in os.environ for var in required_env
             ), f"Please ensure the following environment variables are set: {required_env}"
             self.hexagon_sdk_root = os.getenv(HEXAGON_SDK_ROOT)
             self.hexagon_tools_root = os.getenv(HEXAGON_TOOLS_ROOT)
-            self.dsp_arch = os.getenv(DSP_VERSION)
             logging.info(f"{HEXAGON_SDK_ROOT}={self.hexagon_sdk_root}")
             logging.info(f"{HEXAGON_TOOLS_ROOT}={self.hexagon_tools_root}")
-            logging.info(f"{DSP_VERSION}={self.dsp_arch}")
         self.qnn_config = qnn_config
         self.qnn_sdk = os.getenv("QNN_SDK_ROOT")
         self.build_path = qnn_config.build_folder
@@ -287,17 +285,25 @@ def __init__(
         if self.direct_build_folder:
             direct_general_artifacts = [
                 f"{self.build_path}/examples/qualcomm/direct_executor_runner/libqnn_executorch_stub.so",
-                f"{self.direct_build_folder}/backends/qualcomm/libqnn_executorch_backend.so",
-                f"{self.direct_build_folder}/backends/qualcomm/qnn_executorch/direct_mode/libqnn_executorch_skel.so",
             ]
             self.backend_library_paths.update(
                 {
                     QnnExecuTorchBackendType.kHtpBackend: [
+                        f"{self.direct_build_folder}/backends/qualcomm/libqnn_executorch_backend.so",
+                        f"{self.direct_build_folder}/backends/qualcomm/qnn_executorch/direct_mode/libqnn_executorch_skel.so",
                         f"{self.qnn_sdk}/lib/hexagon-v{self.htp_arch}/unsigned/libQnnHtpV{self.htp_arch}.so",
                         f"{self.qnn_sdk}/lib/hexagon-v{self.htp_arch}/unsigned/libQnnSystem.so",
                         f"{self.hexagon_tools_root}/Tools/target/hexagon/lib/v{self.htp_arch}/G0/pic/libc++abi.so.1",
                         f"{self.hexagon_tools_root}/Tools/target/hexagon/lib/v{self.htp_arch}/G0/pic/libc++.so.1",
-                    ]
+                    ],
+                    QnnExecuTorchBackendType.kLpaiBackend: [
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libqnn_executorch_backend.so",
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libqnn_executorch_skel.so",
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libQnnLpai.so",
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libQnnSystem.so",
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libc++abi.so.1",
+                        f"{self.qnn_sdk}/lib/lpai-v{self.lpai_hw_ver}/signed/libc++.so.1",
+                    ],
                 }
             )
             for _, library_paths in self.backend_library_paths.items():
@@ -378,6 +384,12 @@ def push(  # noqa: C901
             # backend libraries
             for backend in backends:
                 artifacts.extend(self.backend_library_paths[backend])
+
+            # Ensure that all necessary library artifacts exists.
+            missing = [path for path in artifacts if not os.path.exists(path)]
+            assert not missing, "Missing the following libraries:\n" + "\n".join(
+                f"  {p}" for p in missing
+            )
         with tempfile.TemporaryDirectory() as tmp_dir:
             input_list_file, input_files = generate_inputs(
                 tmp_dir, self.input_list_filename, inputs
@@ -440,6 +452,13 @@ def execute(
                 )
                 + self.extra_cmds
             )
+            if self.qnn_config.direct_build_folder:
+                qnn_executor_runner_args = " ".join(
+                    [
+                        qnn_executor_runner_args,
+                        f"--domain_id {get_dsp_id(self.qnn_config.backend)}",
+                    ]
+                )
             qnn_executor_runner_cmds = " ".join(
                 [
                     f"cd {self.workspace} &&",
@@ -526,7 +545,9 @@ def build_executorch_binary(
     ):
         raise RuntimeError("Currently LPAI backend only supports offline_prepare.")
     backend_options = {
-        QnnExecuTorchBackendType.kLpaiBackend: generate_lpai_compiler_spec(),
+        QnnExecuTorchBackendType.kLpaiBackend: generate_lpai_compiler_spec(
+            target_env=get_lpai_target_env(qnn_config)
+        ),
         QnnExecuTorchBackendType.kGpuBackend: generate_gpu_compiler_spec(),
         QnnExecuTorchBackendType.kHtpBackend: generate_htp_compiler_spec(
             use_fp16=False if quant_dtype is not None else True,
@@ -652,10 +673,31 @@ def make_quantizer(
     return quantizer
 
 
+def get_lpai_target_env(qnn_config: QnnConfig):
+    if qnn_config.enable_x86_64:
+        return QnnExecuTorchLpaiTargetEnv.kX86
+    elif qnn_config.direct_build_folder:
+        return QnnExecuTorchLpaiTargetEnv.kAdsp
+    return QnnExecuTorchLpaiTargetEnv.kArm
+
+
 def get_backend_type(backend: str):
     return getattr(QnnExecuTorchBackendType, f"k{backend.title()}Backend")
 
 
+def get_dsp_id(backend):
+    dsp_id_map = {
+        QnnExecuTorchBackendType.kLpaiBackend: 0,
+        QnnExecuTorchBackendType.kHtpBackend: 3,
+    }
+    if backend not in dsp_id_map:
+        raise ValueError(
+            f"Unsupported backend {backend} for direct mode. "
+            f"Supported: {list(dsp_id_map.keys())}"
+        )
+    return dsp_id_map[backend]
+
+
 def setup_common_args_and_variables():
     parser = argparse.ArgumentParser()
 
@@ -822,7 +864,7 @@ def setup_common_args_and_variables():
 
     parser.add_argument(
         "--direct_build_folder",
-        help="Path to cmake binary directory for direct_mode. E.g., path/to/build-hexagon."
+        help="Path to cmake binary directory for direct_mode. E.g., path/to/build-direct."
         "If enabled, run self-defined protocol to control fastrpc communication.",
         type=str,
     )

backends/qualcomm/requirements.txt

@@ -3,3 +3,4 @@ pydot
 py-cpuinfo
 requests
 tabulate
+openpyxl

backends/qualcomm/runtime/QnnExecuTorch.h

@@ -26,6 +26,9 @@
 #define QNN_RUNTIME_LPAI_AFFINITY "qnn_runtime_lpai_affinity"
 #define QNN_RUNTIME_LPAI_CORE_SELECTION "qnn_runtime_lpai_core_selection"
 
+// This is for direct mode, especially LPAI
+#define QNN_TENSOR_ALIGNMENT 64
+
 #ifdef __cplusplus
 extern "C" {
 #endif // __cplusplus

backends/qualcomm/runtime/QnnExecuTorchBackend.cpp

@@ -148,8 +148,7 @@ Error QnnExecuTorchBackend::execute(
           Error::Ok) {
         // update data ptr only should be fine
         input_tensor->FillDataBuffer(
-            args[args_index]->toTensor().const_data_ptr(),
-            false /* copy_data */);
+            args[args_index]->toTensor().const_data_ptr());
         // use the real input shape instead of nominal one to make sure
         // dynamic shape is functional
         auto dims = args[args_index]->toTensor().sizes();
@@ -167,7 +166,7 @@ Error QnnExecuTorchBackend::execute(
       void* mutable_data_ptr = args[args_index]->toTensor().mutable_data_ptr();
       if (qnn_manager->RegisterMem(mutable_data_ptr, output_tensor) !=
           Error::Ok) {
-        output_tensor->FillDataBuffer(mutable_data_ptr, false /* copy_data */);
+        output_tensor->FillDataBuffer(mutable_data_ptr);
       }
       args_index++;
     }

backends/qualcomm/runtime/QnnManager.cpp

@@ -376,8 +376,7 @@ Error QnnManager::AllocateTensor(const std::string& graph_name) {
             mutable_buffer_id_to_memory_map.end()) {
       // Fill the same memory for I/O of mutable buffer
       tensor_wrapper->FillDataBuffer(
-          mutable_buffer_id_to_memory_map[mutable_buffer_id],
-          false /* copy_data */);
+          mutable_buffer_id_to_memory_map[mutable_buffer_id]);
     }
     output_tensors_[graph_name].emplace_back(std::move(tensor_wrapper));
   }
@@ -581,9 +580,7 @@ Error QnnManager::CompileDlc() {
           mutable_buffer_id_to_memory_map.find(mutable_buffer_id) !=
               mutable_buffer_id_to_memory_map.end()) {
         // Fill the same memory for I/O of mutable buffer
-        tw->FillDataBuffer(
-            mutable_buffer_id_to_memory_map[mutable_buffer_id],
-            false /* copy_data */);
+        tw->FillDataBuffer(mutable_buffer_id_to_memory_map[mutable_buffer_id]);
       }
       graph_outputs.push_back(tw);
     }