Qualcomm AI Engine Direct - CDSP Direct Mode

.ci/scripts/build-qnn-sdk.sh

@@ -18,7 +18,7 @@ build_qnn_backend() {
   export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
 
   parallelism=$(( $(nproc) - 1 ))
-  bash backends/qualcomm/scripts/build.sh --skip_linux_android --skip_linux_embedded --job_number ${parallelism} --release
+  bash backends/qualcomm/scripts/build.sh --skip_linux_android --job_number ${parallelism} --release
 }
 
 set_up_aot() {

backends/qualcomm/CMakeLists.txt

@@ -174,7 +174,6 @@ add_library(qnn_sys_function_interface INTERFACE)
 add_library(qnn_sys_implementation STATIC)
 add_library(shared_buffer STATIC)
 add_library(wrappers STATIC)
-add_library(utils STATIC)
 
 #
 # declare dependency
@@ -240,24 +239,38 @@ target_link_libraries(
 )
 
 target_link_libraries(
-  qnn_manager
-  PRIVATE qnn_factory
-          qnn_backend_unified_registry
-          wrappers
-          qnn_schema
-          utils
-          shared_buffer
-          qnn_dlc_manager
+  qnn_manager PRIVATE qnn_factory qnn_backend_unified_registry wrappers
+                      qnn_schema shared_buffer qnn_dlc_manager
 )
 target_link_libraries(
   qnn_executorch_backend
   PRIVATE qnn_executorch_header qnn_schema qnn_manager executorch_core
           extension_tensor qnn_backend_options
 )
+
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon)
+  # Add macro here so we can dlopen the correct .so library.
+  if(DSP_TYPE STREQUAL "3")
+    string(TOUPPER ${DSP_VERSION} CAPITAL_DSP_VERSION)
+    set(HEXAGON_LIB "libQnnHtp${CAPITAL_DSP_VERSION}.so")
+    add_compile_definitions(HEXAGON_LIB="${HEXAGON_LIB}")
+    message(STATUS "For hexagon build, using HTP Library: ${HEXAGON_LIB}")
+  else()
+    message(FATAL_ERROR "Unknown DSP_TYPE ${DSP_TYPE}")
+  endif()
+
+  target_link_libraries(
+    qnn_executorch_backend
+    PRIVATE
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc.so
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++.so.1
+      ${HEXAGON_TOOLS_ROOT}/Tools/target/hexagon/lib/${DSP_VERSION}/G0/pic/libc++abi.so.1
+  )
+endif()
+
 set_target_properties(
   qnn_executorch_backend PROPERTIES LINK_FLAGS "-Wl,-rpath='$ORIGIN'"
 )
-target_link_libraries(utils PRIVATE qnn_executorch_logging)
 target_link_libraries(
   shared_buffer PRIVATE qnn_executorch_logging ${CMAKE_DL_LIBS}
 )
@@ -288,6 +301,13 @@ install(
   RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/executorch/backends/qualcomm
 )
 
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES Hexagon)
+  add_subdirectory(
+    ${QNN_EXECUTORCH_ROOT_DIR}/runtime/backends/direct_mode
+    ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/direct_mode
+  )
+endif()
+
 # QNN pybind
 if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
   add_subdirectory(

backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp

@@ -147,6 +147,10 @@ std::shared_ptr<TensorWrapper> CreateTensorWrapper(
       copy_data);
 }
 
+int GetQNNCtxBinAlignment() {
+  return QNN_CTX_BIN_ALIGNMENT;
+}
+
 std::string GetQnnSdkBuildId(std::string library_path) {
   QnnImplementation qnn_loaded_backend = QnnImplementation(library_path);
   ET_CHECK_MSG(
@@ -191,6 +195,7 @@ PYBIND11_MODULE(PyQnnManagerAdaptor, m) {
   using namespace qnn_delegate;
   PYBIND11_NUMPY_DTYPE(PyQnnTensorWrapper::EncodingData, scale, offset);
 
+  m.def("GetQNNCtxBinAlignment", &GetQNNCtxBinAlignment);
   m.def("GetQnnSdkBuildId", &GetQnnSdkBuildId);
   m.def("StripProtocol", &StripProtocol);
   py::class_<QnnExecuTorchContextBinary>(m, "QnnExecuTorchContextBinary")

backends/qualcomm/runtime/CMakeLists.txt

@@ -42,13 +42,6 @@ target_sources(
   PRIVATE ${CMAKE_CURRENT_LIST_DIR}/Logging.cpp
 )
 
-# utils
-target_sources(
-  utils
-  PUBLIC ${CMAKE_CURRENT_LIST_DIR}/Utils.h
-  PRIVATE ${CMAKE_CURRENT_LIST_DIR}/Utils.cpp
-)
-
 # shared_buffer
 target_sources(
   shared_buffer PRIVATE ${CMAKE_CURRENT_LIST_DIR}/SharedBuffer.h

backends/qualcomm/runtime/Logging.cpp

@@ -11,6 +11,9 @@
 #ifdef __ANDROID__
 #include <android/log.h>
 #endif
+#ifdef __hexagon__
+#include "HAP_farf.h"
+#endif
 namespace executorch {
 namespace backends {
 namespace qnn {
@@ -58,10 +61,18 @@ void Log(QnnExecuTorchLogLevel log_level, const char* format, ...) {
   }
   __android_log_vprint(android_severity, "[Qnn ExecuTorch]", format, args);
 #endif
+
+#ifndef __hexagon__
   fprintf(stderr, "[%s] [Qnn ExecuTorch]: ", serverity_name);
   vfprintf(stderr, format, args);
   va_end(args);
   fputc('\n', stderr);
+#else
+  char buf[QNN_EXECUTORCH_FARF_BUF_SIZE] = {0};
+  vsnprintf(buf, QNN_EXECUTORCH_FARF_BUF_SIZE, format, args);
+  va_end(args);
+  FARF(RUNTIME_HIGH, "[%s] [Qnn ExecuTorch]: %s\n", serverity_name, buf);
+#endif
 }
 } // namespace qnn
 } // namespace backends

backends/qualcomm/runtime/Logging.h

@@ -16,6 +16,8 @@ using namespace qnn_delegate;
 
 void Log(QnnExecuTorchLogLevel log_level, const char* format, ...);
 
+#define QNN_EXECUTORCH_FARF_BUF_SIZE 128
+
 #define QNN_EXECUTORCH_LOG(log_level, format, ...) \
   do {                                             \
     Log(log_level, format, ##__VA_ARGS__);         \

backends/qualcomm/runtime/QnnExecuTorchBackend.cpp

@@ -112,8 +112,12 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
     }
   }
   add_cached_delegate(signature, qnn_manager);
+
+#ifndef __hexagon__
   // This backend does not need its processed data after Init.
   processed->Free();
+#endif
+
   return qnn_manager;
 }
 

backends/qualcomm/runtime/QnnManager.cpp

@@ -9,7 +9,6 @@
 #include <executorch/backends/qualcomm/runtime/QnnBackendOptions.h>
 #include <executorch/backends/qualcomm/runtime/QnnManager.h>
 #include <executorch/backends/qualcomm/runtime/SharedBuffer.h>
-#include <executorch/backends/qualcomm/runtime/Utils.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnBackendCommon.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnCustomProtocol.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>

backends/qualcomm/runtime/backends/CMakeLists.txt

@@ -51,6 +51,15 @@ set(HOST_ARCHITECTURE_HTP
 )
 set(HOST_ARCHITECTURE_IR ${CMAKE_CURRENT_LIST_DIR}/ir/${CMAKE_SYSTEM_PROCESSOR})
 
+set(platform target)
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES x86_64)
+  set(platform host)
+endif()
+
+set(HOST_ARCHITECTURE_GPU ${CMAKE_CURRENT_LIST_DIR}/gpu/${platform})
+set(HOST_ARCHITECTURE_HTP ${CMAKE_CURRENT_LIST_DIR}/htp/${platform})
+set(HOST_ARCHITECTURE_IR ${CMAKE_CURRENT_LIST_DIR}/ir/${platform})
+
 # qnn_device
 target_sources(
   qnn_device

backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h

@@ -71,7 +71,12 @@ class QnnBackendUnifiedRegistry {
   QnnBackendUnifiedRegistry& operator=(const QnnBackendUnifiedRegistry&) =
       delete;
 
+#ifdef __hexagon__
+  // For macro, refer to executorch/backends/qualcomm/CMakeLists.txt
+  static constexpr const char* htp_library_name_ = HEXAGON_LIB;
+#else
   static constexpr const char* htp_library_name_ = "libQnnHtp.so";
+#endif
   static constexpr const char* gpu_library_name_ = "libQnnGpu.so";
   static constexpr const char* dsp_library_name_ = "libQnnDsp.so";
 

backends/qualcomm/runtime/backends/QnnCustomProtocol.cpp

@@ -20,8 +20,7 @@ void QnnContextCustomProtocol::BuildContextCustomBuffer() {
     uint8_t magic_number_proto_size = sizeof(magic_number_);
     uint8_t binary_proto_size = sizeof(binary_size_);
     uint8_t signature_proto_size = sizeof(signature_);
-    uint64_t buffer_size = magic_number_proto_size + signature_proto_size +
-        binary_proto_size + binary_size_;
+    uint64_t buffer_size = QNN_CTX_BIN_ALIGNMENT + binary_size_;
     qnn_custom_buffer_.resize(buffer_size, 0);
 
     size_t pos = 0;
@@ -62,6 +61,8 @@ QnnContextCustomProtocol::DeserializeContextCustomBuffer(void* processed_data) {
   uint8_t magic_number_proto_size = sizeof(magic_number_);
   uint8_t binary_proto_size = sizeof(binary_size_);
   uint8_t signature_proto_size = sizeof(signature_);
+  uint32_t padding_size = QNN_CTX_BIN_ALIGNMENT - magic_number_proto_size -
+      binary_proto_size - signature_proto_size;
 
   uint32_t magic_number;
   std::memcpy(&magic_number, ptr, magic_number_proto_size);
@@ -80,13 +81,13 @@ QnnContextCustomProtocol::DeserializeContextCustomBuffer(void* processed_data) {
 
   uint64_t binary_size;
   std::memcpy(&binary_size, ptr, binary_proto_size);
-  ptr += binary_proto_size;
+  ptr += binary_proto_size + padding_size;
 
   return {status, signature_, binary_size, static_cast<void*>(ptr)};
 }
 
 uint64_t QnnContextCustomProtocol::GetContextBinaryOffset() {
-  return sizeof(magic_number_) + sizeof(signature_) + sizeof(binary_size_);
+  return QNN_CTX_BIN_ALIGNMENT;
 }
 
 } // namespace qnn

backends/qualcomm/runtime/backends/QnnCustomProtocol.h

@@ -18,6 +18,8 @@
 #include <tuple>
 #include <vector>
 
+#define QNN_CTX_BIN_ALIGNMENT 256
+
 namespace executorch {
 namespace backends {
 namespace qnn {
@@ -51,6 +53,8 @@ class QnnCustomProtocol {
 // ---------------------------------
 // | context_binary_size (8 bytes) |
 // ---------------------------------
+// | extra padding (236 bytes)     |
+// ---------------------------------
 // | context_binary.data           |
 // ---------------------------------
 class QnnContextCustomProtocol : public QnnCustomProtocol {

backends/qualcomm/runtime/backends/QnnImplementation.cpp

@@ -8,6 +8,10 @@
 #include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>
 #include <memory>
 #include "QnnInterface.h"
+#ifdef __hexagon__
+#include "HAP_farf.h"
+#endif
+
 namespace executorch {
 namespace backends {
 namespace qnn {
@@ -25,6 +29,7 @@ struct DlCloser {
 Error QnnImplementation::InitBackend(
     void* const lib_handle,
     const QnnSaver_Config_t** saver_config) {
+#ifndef __hexagon__
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
   // saver_config must be set before backend initialization
   auto saver_initialize =
@@ -39,6 +44,7 @@ Error QnnImplementation::InitBackend(
       return Error::Internal;
     }
   }
+#endif
   return Error::Ok;
 }
 
@@ -50,6 +56,13 @@ const QnnInterface_t* QnnImplementation::StartBackend(
     const std::string& lib_path,
     const QnnSaver_Config_t** saver_config) {
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
+
+#ifdef __hexagon__
+  FARF(RUNTIME_HIGH, "Opening lib_path %s", lib_path.c_str());
+  std::unique_ptr<void, DlCloser> lib_handle(
+      dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL));
+  FARF(RUNTIME_HIGH, "Done loading lib_path %s", lib_path.c_str());
+#else
   // If the library is already loaded, return the handle.
   std::unique_ptr<void, DlCloser> lib_handle(
       dlopen(lib_path.c_str(), RTLD_NOW | RTLD_NOLOAD));
@@ -64,6 +77,7 @@ const QnnInterface_t* QnnImplementation::StartBackend(
         dlerror());
     return nullptr;
   }
+#endif
 
   // load get_provider function
   auto get_providers = loadQnnFunction<QnnInterfaceGetProvidersFn*>(

backends/qualcomm/runtime/backends/QnnSysImplementation.cpp

@@ -16,7 +16,6 @@ using executorch::runtime::Error;
 
 Error QnnSystemImplementation::Load() {
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
-
   void* lib_handle_ = dlopen(lib_path_.c_str(), RTLD_NOW | RTLD_LOCAL);
   if (lib_handle_ == nullptr) {
     QNN_EXECUTORCH_LOG_ERROR(
@@ -48,7 +47,6 @@ Error QnnSystemImplementation::Load() {
         QNN_GET_ERROR_CODE(error));
     return Error::Internal;
   }
-
   if (num_providers != required_num_providers_) {
     QNN_EXECUTORCH_LOG_ERROR(
         "QnnSystemInterface Num "