Qualcomm AI Engine Direct - Refactor QnnDlcManager (#19105)

### Summary
Refactor Dlc manager from experimental API to formal one.

#### Reference

https://docs.qualcomm.com/doc/80-63442-10/topic/function_QnnSystemDlc_8h_1ad09233e5a66c421e0e80f4cdbf4c1b7e.html

https://docs.qualcomm.com/doc/80-63442-10/topic/function_QnnSystemDlc_8h_1aa3fcdf5c15256a69d445fc2a8c7a0e60.html

### Test plan
There is a unit test for online prepare already.


cc @cccclai @cbilgin @abhinaykukkadapu

Author: chenweng-quic

backends/qualcomm/CMakeLists.txt

@@ -109,7 +109,6 @@ endif()
 
 include_directories(
   BEFORE ${_common_include_directories} ${QNN_SDK_ROOT}/include/QNN
-  ${QNN_SDK_ROOT}/share/QNN/converter/jni
   ${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10
 )
 

backends/qualcomm/export_utils.py

@@ -311,7 +311,6 @@ def __init__(
             traditional_general_artifacts = [
                 f"{self.qnn_sdk}/lib/{self.target}/libQnnSystem.so",
                 f"{self.build_path}/backends/qualcomm/libqnn_executorch_backend.so",
-                f"{self.qnn_sdk}/lib/{self.target}/libQnnModelDlc.so",
             ]
             self.backend_library_paths.update(
                 {

backends/qualcomm/runtime/QnnManager.cpp

@@ -246,6 +246,7 @@ Error QnnManager::InitContext(
         options_->backend_options()->backend_type());
     backend_params_ptr_ = QnnBackendFactory().Create(
         backend_bundle_ptr_->implementation.get(),
+        backend_bundle_ptr_->system_implementation.get(),
         backend_bundle_ptr_->qnn_backend_ptr.get(),
         backend_bundle_ptr_->qnn_device_ptr.get(),
         qnn_context_blob_,
@@ -279,7 +280,10 @@ Error QnnManager::InitContext(
         BackendInitializeState::INITIALIZED;
   }
 
-  if (IsOnlinePrepare()) {
+  if (IsOnlinePrepare() &&
+      backend_params_ptr_->qnn_backend_cache_ptr_->GetCacheState() ==
+          QnnBackendCache::SERIALIZE) {
+    // Set up DLC environment at AOT time
     // Check whether the QNN version supports the DLC format.
     Qnn_ApiVersion_t qnn_version = {QNN_VERSION_INIT};
     backend_bundle_ptr_->implementation->GetQnnInterface()
@@ -304,6 +308,7 @@ Error QnnManager::InitContextCache() {
         options_->backend_options()->backend_type());
     backend_params_ptr_ = QnnBackendFactory().Create(
         backend_bundle_ptr_->implementation.get(),
+        backend_bundle_ptr_->system_implementation.get(),
         backend_bundle_ptr_->qnn_backend_ptr.get(),
         backend_bundle_ptr_->qnn_device_ptr.get(),
         qnn_context_blob_,
@@ -476,9 +481,9 @@ Error QnnManager::ProfileExecuteData(
 }
 
 void QnnManager::Destroy() {
+  qnn_dlc_manager_->Destroy();
   backend_params_ptr_.reset(new BackendConfigParameters());
   backend_bundle_ptr_.reset(new QnnBackendBundle());
-  qnn_dlc_manager_->Destroy();
 }
 
 void QnnManager::DestroyContext() {
@@ -539,12 +544,25 @@ Error QnnManager::GetContextBinary(
 
 Error QnnManager::CompileDlc() {
   Qnn_ErrorHandle_t error;
-  auto qnn_dlc_graph_info = qnn_dlc_manager_->GetQnnDlcGraphInfoPtr();
-  uint32_t qnn_dlc_graph_info_num = qnn_dlc_manager_->GetQnnDlcGraphInfoNum();
-  for (uint32_t i = 0; i < qnn_dlc_graph_info_num; ++i) {
-    auto& graphInfo = (*qnn_dlc_graph_info)[i];
+  auto graphs = qnn_dlc_manager_->GetQnnDlcGraphInfoPtr();
+  uint32_t num_graphs = qnn_dlc_manager_->GetQnnDlcGraphInfoNum();
+  for (uint32_t i = 0; i < num_graphs; ++i) {
+    auto& graphInfo = graphs[i].graphInfoV1;
+    Qnn_GraphHandle_t graphHandle;
+    error = backend_bundle_ptr_->implementation->GetQnnInterface()
+                .qnn_graph_retrieve(
+                    backend_params_ptr_->qnn_context_ptr_->GetHandle(),
+                    graphInfo.graphName,
+                    &graphHandle);
+    if (error != QNN_SUCCESS) {
+      QNN_EXECUTORCH_LOG_ERROR(
+          "Failed to retrieve graph %s. Error %d.",
+          graphInfo.graphName,
+          QNN_GET_ERROR_CODE(error));
+      return Error::Internal;
+    }
     backend_params_ptr_->qnn_graph_ptr_->SetGraphHandle(
-        graphInfo.graphName, graphInfo.graph);
+        graphInfo.graphName, graphHandle);
     error =
         backend_params_ptr_->qnn_graph_ptr_->GraphFinalize(graphInfo.graphName);
     if (error != QNN_SUCCESS) {
@@ -559,9 +577,9 @@ Error QnnManager::CompileDlc() {
 
     // Mapping memory address for the input and output of mutable buffer
     std::unordered_map<int, const void*> mutable_buffer_id_to_memory_map;
-    for (uint32_t i = 0; i < graphInfo.numInputTensors; ++i) {
-      auto tw = CreateTensorWrapper(graphInfo.inputTensors[i]);
-      tw->UpdateQnnTensorMeta(graphInfo.inputTensors[i]);
+    for (uint32_t i = 0; i < graphInfo.numGraphInputs; ++i) {
+      auto tw = CreateTensorWrapper(graphInfo.graphInputs[i]);
+      tw->UpdateQnnTensorMeta(graphInfo.graphInputs[i]);
 
       int mutable_buffer_id = ExtractMutableBufferNumber(tw->GetName());
       if (mutable_buffer_id != -1) {
@@ -572,9 +590,9 @@ Error QnnManager::CompileDlc() {
       }
       graph_inputs.push_back(tw);
     }
-    for (uint32_t i = 0; i < graphInfo.numOutputTensors; ++i) {
-      auto tw = CreateTensorWrapper(graphInfo.outputTensors[i]);
-      tw->UpdateQnnTensorMeta(graphInfo.outputTensors[i]);
+    for (uint32_t i = 0; i < graphInfo.numGraphOutputs; ++i) {
+      auto tw = CreateTensorWrapper(graphInfo.graphOutputs[i]);
+      tw->UpdateQnnTensorMeta(graphInfo.graphOutputs[i]);
       int mutable_buffer_id = ExtractMutableBufferNumber(tw->GetName());
       if (mutable_buffer_id != -1 &&
           mutable_buffer_id_to_memory_map.find(mutable_buffer_id) !=

backends/qualcomm/runtime/backends/QnnBackendCache.cpp

@@ -19,7 +19,7 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary(
     void* buffer,
     uint32_t nbytes) {
   const QnnSystemInterface& qnn_sys_interface =
-      qnn_sys_impl_.GetQnnSystemInterface();
+      qnn_sys_impl_->GetQnnSystemInterface();
   std::uint32_t num_graphs;
   QnnSystemContext_GraphInfo_t* graphs = nullptr;
   const QnnSystemContext_BinaryInfo_t* binaryinfo{nullptr};
@@ -88,18 +88,11 @@ Error QnnBackendCache::Configure(const std::vector<std::string>& graph_names) {
     return Error::Ok;
   }
 
-  if (qnn_sys_impl_.Load() != Error::Ok) {
-    QNN_EXECUTORCH_LOG_ERROR(
-        "Failed to Load QnnSystem "
-        "APIs. Caching mechanism is being disabled.");
-    return Error::Internal;
-  }
-
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
 
   // create QNN SystemContext
   const QnnSystemInterface& qnn_sys_interface =
-      qnn_sys_impl_.GetQnnSystemInterface();
+      qnn_sys_impl_->GetQnnSystemInterface();
   error = qnn_sys_interface.qnn_system_context_create(&sys_context_handle_);
 
   if (error != QNN_SUCCESS) {
@@ -137,14 +130,13 @@ QnnBackendCache::~QnnBackendCache() {
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
   if (sys_context_handle_ != nullptr) {
     const QnnSystemInterface& qnn_sys_interface =
-        qnn_sys_impl_.GetQnnSystemInterface();
+        qnn_sys_impl_->GetQnnSystemInterface();
     error = qnn_sys_interface.qnn_system_context_free(sys_context_handle_);
     if (error != QNN_SUCCESS) {
       QNN_EXECUTORCH_LOG_WARN("Failed to free QNN system context.");
     }
     sys_context_handle_ = nullptr;
   }
-  qnn_sys_impl_.Unload();
 }
 
 std::vector<Qnn_Tensor_t> QnnBackendCache::GetGraphInputs(

backends/qualcomm/runtime/backends/QnnBackendCache.h

@@ -26,8 +26,10 @@ class QnnBackendCache {
     ONLINE_PREPARE = 3,
     MULTI_GRAPH = 4,
   };
-  explicit QnnBackendCache(const QnnExecuTorchContextBinary& qnn_context_blob)
-      : qnn_context_blob_(qnn_context_blob) {}
+  explicit QnnBackendCache(
+      const QnnExecuTorchContextBinary& qnn_context_blob,
+      QnnSystemImplementation* qnn_sys_impl)
+      : qnn_sys_impl_(qnn_sys_impl), qnn_context_blob_(qnn_context_blob) {}
   virtual ~QnnBackendCache();
   QnnBackendCache(const QnnBackendCache&) = delete;
   QnnBackendCache(QnnBackendCache&&) = delete;
@@ -66,6 +68,7 @@ class QnnBackendCache {
       __ET_UNUSED const QnnSystemContext_BinaryInfo_t* binaryinfo) {
     return executorch::runtime::Error::Ok;
   }
+  QnnSystemImplementation* qnn_sys_impl_;
 
  private:
   executorch::runtime::Error GetQnnGraphInfoFromBinary(
@@ -79,7 +82,6 @@ class QnnBackendCache {
 
   QnnExecuTorchContextBinary qnn_context_blob_;
   QnnSystemContext_Handle_t sys_context_handle_{nullptr};
-  QnnSystemImplementation qnn_sys_impl_{"libQnnSystem.so"};
   std::vector<std::string> graph_names_;
   std::unordered_map<std::string, std::vector<Qnn_Tensor_t>>
       input_tensor_structs_;

backends/qualcomm/runtime/backends/QnnBackendFactory.cpp

@@ -17,6 +17,7 @@ using executorch::runtime::Error;
 
 std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
     QnnImplementation* implementation_ptr,
+    QnnSystemImplementation* system_implementation_ptr,
     QnnBackend* qnn_backend_ptr,
     QnnDevice* qnn_device_ptr,
     const QnnExecuTorchContextBinary& qnn_context_blob,
@@ -63,10 +64,12 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
             htp_options->use_weight_sharing());
       }
       backend_params->qnn_backend_cache_ptr_ =
-          std::make_unique<HtpBackendCache>(qnn_context_blob);
+          std::make_unique<HtpBackendCache>(
+              qnn_context_blob, system_implementation_ptr);
 
       backend_params->qnn_context_ptr_ = std::make_unique<HtpContext>(
           implementation_ptr,
+          system_implementation_ptr,
           qnn_backend_ptr,
           qnn_device_ptr,
           backend_params->qnn_backend_cache_ptr_.get(),
@@ -107,10 +110,12 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
       }
 
       backend_params->qnn_backend_cache_ptr_ =
-          std::make_unique<QnnBackendCache>(qnn_context_blob);
+          std::make_unique<QnnBackendCache>(
+              qnn_context_blob, system_implementation_ptr);
 
       backend_params->qnn_context_ptr_ = std::make_unique<GpuContext>(
           implementation_ptr,
+          system_implementation_ptr,
           qnn_backend_ptr,
           qnn_device_ptr,
           backend_params->qnn_backend_cache_ptr_.get(),
@@ -151,10 +156,12 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
             "target_env in lpai_options: %d", lpai_options->target_env());
       }
       backend_params->qnn_backend_cache_ptr_ =
-          std::make_unique<QnnBackendCache>(qnn_context_blob);
+          std::make_unique<QnnBackendCache>(
+              qnn_context_blob, system_implementation_ptr);
 
       backend_params->qnn_context_ptr_ = std::make_unique<LpaiContext>(
           implementation_ptr,
+          system_implementation_ptr,
           qnn_backend_ptr,
           qnn_device_ptr,
           backend_params->qnn_backend_cache_ptr_.get(),

backends/qualcomm/runtime/backends/QnnBackendFactory.h

@@ -63,6 +63,7 @@ class QnnBackendFactory {
  public:
   std::unique_ptr<BackendConfigParameters> Create(
       QnnImplementation* implementation,
+      QnnSystemImplementation* system_implementation,
       QnnBackend* qnn_backend_ptr,
       QnnDevice* qnn_device_ptr,
       const QnnExecuTorchContextBinary& qnn_context_blob,

backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp

@@ -152,8 +152,15 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle(
   if (backend->VerifyQNNSDKVersion() != Error::Ok) {
     return Error::Internal;
   }
+  // 5. Create QnnSystemImplementation and load qnn library
+  std::unique_ptr<QnnSystemImplementation> system_implementation =
+      std::make_unique<QnnSystemImplementation>("libQnnSystem.so");
+  ret = system_implementation->Load();
+  ET_CHECK_OR_RETURN_ERROR(
+      ret == Error::Ok, Internal, "Fail to load Qnn system library");
 
   bundle->implementation = std::move(implementation);
+  bundle->system_implementation = std::move(system_implementation);
   bundle->qnn_logger_ptr = std::move(logger);
   bundle->qnn_backend_ptr = std::move(backend);
   bundle->qnn_device_ptr = std::move(device);

backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h

@@ -12,6 +12,7 @@
 #include <executorch/backends/qualcomm/runtime/backends/QnnDeviceCommon.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnLogger.h>
+#include <executorch/backends/qualcomm/runtime/backends/QnnSysImplementation.h>
 #include <executorch/runtime/core/error.h>
 
 #include <memory>
@@ -28,18 +29,21 @@ struct QnnBackendBundle {
   std::unique_ptr<QnnLogger> qnn_logger_ptr;
   std::unique_ptr<QnnBackend> qnn_backend_ptr;
   std::unique_ptr<QnnDevice> qnn_device_ptr;
+  std::unique_ptr<QnnSystemImplementation> system_implementation;
 
   // Default ctor
   QnnBackendBundle()
       : implementation(nullptr),
         qnn_logger_ptr(nullptr),
         qnn_backend_ptr(nullptr),
-        qnn_device_ptr(nullptr) {}
+        qnn_device_ptr(nullptr),
+        system_implementation{nullptr} {}
   // Default dtor
   ~QnnBackendBundle() {
     qnn_device_ptr.reset();
     qnn_backend_ptr.reset();
     qnn_logger_ptr.reset();
+    system_implementation.reset();
     implementation.reset();
   }
 };

backends/qualcomm/runtime/backends/QnnContextCommon.cpp

@@ -153,9 +153,9 @@ Error QnnContext::Configure() {
     return Error::Internal;
   }
   if (cache_->GetCacheState() == QnnBackendCache::ONLINE_PREPARE) {
-    // Register graphs from DLC during online prepare for HTP/GPU/DSP backends
+    // Register DLC graphs at runtime
     return qnn_dlc_manager_->RegisterGraphsFromDLC(
-        implementation_, backend_, this, cache_);
+        implementation_, system_implementation_, backend_, this, cache_);
   }
   return Error::Ok;
 }