[CANN] Fix deadlock during parallel execution

onnxruntime/core/providers/cann/cann_execution_provider.cc

@@ -8,6 +8,8 @@
 #include <iterator>
 #include <map>
 #include <unordered_set>
+#include <future>
+#include <thread>
 
 #define ORT_API_MANUAL_INIT
 #include "core/session/onnxruntime_cxx_api.h"
@@ -30,6 +32,13 @@ namespace onnxruntime {
 
 // Models can only be parsed and built serially in the same process
 std::mutex g_mutex;
+namespace cann {
+
+// See cann_graph.cc
+extern std::promise<void> g_ge_promise_final;
+extern std::thread g_ge_thread;
+
+}  // namespace cann
 
 class Memcpy final : public OpKernel {
  public:
@@ -1067,9 +1076,13 @@ void InitializeRegistry() {
 void DeleteRegistry() {
   s_kernel_registry.reset();
 
-  ge::aclgrphBuildFinalize();
-
   if (!cann::GetRepeatInitFlag()) {
+    // Calls ge::aclgrphBuildFinalize
+    cann::g_ge_promise_final.set_value();
+    if (cann::g_ge_thread.joinable()) {
+      cann::g_ge_thread.join();
+    }
+
     CANN_CALL_THROW(aclFinalize());
   }
 }

onnxruntime/core/providers/cann/cann_graph.cc

@@ -4,6 +4,9 @@
 
 #include <map>
 #include <set>
+#include <exception>
+#include <future>
+#include <thread>
 
 #include "core/providers/cann/cann_graph.h"
 
@@ -14,6 +17,12 @@ static int lower_bound = 8;  // Supported domain version lower bounds
 
 std::once_flag flag;
 
+static std::promise<void> ge_promise_init;
+std::promise<void> g_ge_promise_final;
+static std::future<void> ge_future_init = ge_promise_init.get_future();
+static std::future<void> ge_future_final = g_ge_promise_final.get_future();
+std::thread g_ge_thread;
+
 /**
  * This function will been changed with the evolution of ONNX and CANN
  * and will be replaced by the corresponding API provided by CANN in the future, probably.
@@ -96,20 +105,46 @@ Status ParserONNXModel(std::string string_model, ge::Graph& graph) {
 
 Status BuildONNXModel(ge::Graph& graph, std::string input_shape, const char* soc_name, std::string file_name,
                       CANNExecutionProviderInfo& info, ge::ModelBufferData& model) {
+  static std::exception_ptr call_once_ex_ptr = nullptr;
   std::call_once(flag, [&soc_name, &info]() {
-    std::map<ge::AscendString, ge::AscendString> options;
-    options.emplace(ge::ir_option::SOC_VERSION, soc_name);
-
-    if (!info.precision_mode.empty())
-      options.emplace(ge::ir_option::PRECISION_MODE, info.precision_mode.c_str());
-    if (!info.op_select_impl_mode.empty())
-      options.emplace(ge::ir_option::OP_SELECT_IMPL_MODE, info.op_select_impl_mode.c_str());
-    if (!info.optypelist_for_implmode.empty())
-      options.emplace(ge::ir_option::OPTYPELIST_FOR_IMPLMODE, info.optypelist_for_implmode.c_str());
-
-    CANN_CALL_THROW(ge::aclgrphBuildInitialize(options));
+    try {
+      // Both aclgrphBuildInitialize and aclgrphBuildFinalize
+      // need to be called from the same thread
+      g_ge_thread = std::thread([&]() {
+        try {
+          std::map<ge::AscendString, ge::AscendString> options;
+          options.emplace(ge::ir_option::SOC_VERSION, soc_name);
+
+          if (!info.precision_mode.empty())
+            options.emplace(ge::ir_option::PRECISION_MODE, info.precision_mode.c_str());
+          if (!info.op_select_impl_mode.empty())
+            options.emplace(ge::ir_option::OP_SELECT_IMPL_MODE, info.op_select_impl_mode.c_str());
+          if (!info.optypelist_for_implmode.empty())
+            options.emplace(ge::ir_option::OPTYPELIST_FOR_IMPLMODE, info.optypelist_for_implmode.c_str());
+
+          CANN_CALL_THROW(ge::aclgrphBuildInitialize(options));
+          ge_promise_init.set_value();
+
+          ge_future_final.wait();
+          ge::aclgrphBuildFinalize();
+        } catch (...) {
+          call_once_ex_ptr = std::current_exception();
+          ge_promise_init.set_value();
+        }
+      });
+
+      if (ge_future_init.valid()) {
+        ge_future_init.wait();
+      }
+    } catch (...) {
+      call_once_ex_ptr = std::current_exception();
+    }
   });
 
+  if (call_once_ex_ptr) {
+    std::rethrow_exception(call_once_ex_ptr);
+  }
+
   std::map<ge::AscendString, ge::AscendString> options;
   options.emplace(ge::ir_option::INPUT_SHAPE, input_shape.c_str());
   CANN_GRAPH_RETURN_IF_ERROR(ge::aclgrphBuildModel(graph, options, model));