Add support of out-of-order command buffers to L0 adapter v2 (#18570)

Author: Mikołaj Komar

source/adapters/level_zero/v2/command_buffer.cpp

@@ -32,14 +32,17 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object {
   ur_result_t
   registerExecutionEventUnlocked(ur_event_handle_t nextExecutionEvent);
 
-  lockable<ur_command_list_manager> commandListManager;
-
-  ur_result_t finalizeCommandBuffer();
   // Indicates if command-buffer commands can be updated after it is closed.
   const bool isUpdatable = false;
+  const bool isInOrder = true;
+
   // Command-buffer profiling is enabled.
   const bool isProfilingEnabled = false;
 
+  lockable<ur_command_list_manager> commandListManager;
+
+  ur_result_t finalizeCommandBuffer();
+
   ur_result_t
   createCommandHandle(locked<ur_command_list_manager> &commandListLocked,
                       ur_kernel_handle_t hKernel, uint32_t workDim,
@@ -51,11 +54,25 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object {
       uint32_t numUpdateCommands,
       const ur_exp_command_buffer_update_kernel_launch_desc_t *updateCommands);
 
+  ur_exp_command_buffer_sync_point_t getSyncPoint(ur_event_handle_t event);
+  ur_event_handle_t *getWaitListFromSyncPoints(
+      const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
+      uint32_t numSyncPointsInWaitList);
+
 private:
+  // Stores all sync points that are created by the command buffer.
+  std::vector<ur_event_handle_t> syncPoints;
+
+  // Temporary storage for sync points that are passed to function that require
+  // array of events. This is used to avoid allocating a new memory every time.
+  std::vector<ur_event_handle_t> syncPointWaitList;
+
   const ur_context_handle_t context;
   const ur_device_handle_t device;
+
   std::vector<std::unique_ptr<ur_exp_command_buffer_command_handle_t_>>
       commandHandles;
+
   // Indicates if command-buffer was finalized.
   bool isFinalized = false;
 

source/adapters/level_zero/v2/command_buffer.hpp

@@ -18,10 +18,11 @@
 ur_command_list_manager::ur_command_list_manager(
     ur_context_handle_t context, ur_device_handle_t device,
     v2::raii::command_list_unique_handle &&commandList, v2::event_flags_t flags,
-    ur_queue_t_ *queue)
-    : context(context), device(device),
-      eventPool(context->getEventPoolCache().borrow(device->Id.value(), flags)),
-      zeCommandList(std::move(commandList)), queue(queue) {
+    ur_queue_t_ *queue, PoolCacheType listType)
+    : context(context), device(device), zeCommandList(std::move(commandList)),
+      queue(queue) {
+  auto &eventPoolTmp = context->getEventPoolCache(listType);
+  eventPool = eventPoolTmp.borrow(device->Id.value(), flags);
   UR_CALL_THROWS(ur::level_zero::urContextRetain(context));
   UR_CALL_THROWS(ur::level_zero::urDeviceRetain(device));
 }
@@ -320,17 +321,18 @@ ur_result_t ur_command_list_manager::appendUSMPrefetch(
   return UR_RESULT_SUCCESS;
 }
 
-ur_result_t
-ur_command_list_manager::appendUSMAdvise(const void *pMem, size_t size,
-                                         ur_usm_advice_flags_t advice,
-                                         ur_event_handle_t *phEvent) {
+ur_result_t ur_command_list_manager::appendUSMAdvise(
+    const void *pMem, size_t size, ur_usm_advice_flags_t advice,
+    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
+    ur_event_handle_t *phEvent) {
   TRACK_SCOPE_LATENCY("ur_command_list_manager::appendUSMAdvise");
 
   auto zeAdvice = ur_cast<ze_memory_advice_t>(advice);
 
   auto zeSignalEvent = getSignalEvent(phEvent, UR_COMMAND_USM_ADVISE);
 
-  auto [pWaitEvents, numWaitEvents] = getWaitListView(nullptr, 0);
+  auto [pWaitEvents, numWaitEvents] =
+      getWaitListView(phEventWaitList, numEventsInWaitList);
 
   if (pWaitEvents) {
     ZE2UR_CALL(zeCommandListAppendWaitOnEvents,

source/adapters/level_zero/v2/command_list_manager.cpp

@@ -11,6 +11,7 @@
 
 #include "command_list_cache.hpp"
 #include "common.hpp"
+#include "context.hpp"
 #include "event_pool_cache.hpp"
 #include "memory.hpp"
 #include "queue_api.hpp"
@@ -39,7 +40,8 @@ struct ur_command_list_manager {
   ur_command_list_manager(ur_context_handle_t context,
                           ur_device_handle_t device,
                           v2::raii::command_list_unique_handle &&commandList,
-                          v2::event_flags_t flags, ur_queue_t_ *queue);
+                          v2::event_flags_t flags, ur_queue_t_ *queue,
+                          PoolCacheType listType);
   ur_command_list_manager(const ur_command_list_manager &src) = delete;
   ur_command_list_manager(ur_command_list_manager &&src) = default;
 
@@ -128,6 +130,8 @@ struct ur_command_list_manager {
 
   ur_result_t appendUSMAdvise(const void *pMem, size_t size,
                               ur_usm_advice_flags_t advice,
+                              uint32_t numEventsInWaitList,
+                              const ur_event_handle_t *phEventWaitList,
                               ur_event_handle_t *phEvent);
 
   ur_result_t appendBarrier(uint32_t numEventsInWaitList,

source/adapters/level_zero/v2/command_list_manager.hpp

@@ -53,16 +53,25 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext,
       commandListCache(hContext,
                        {phDevices[0]->Platform->ZeCopyOffloadExtensionSupported,
                         phDevices[0]->Platform->ZeMutableCmdListExt.Supported}),
-      eventPoolCache(
+      eventPoolCacheImmediate(
           this, phDevices[0]->Platform->getNumDevices(),
           [context = this](DeviceId /* deviceId*/, v2::event_flags_t flags)
               -> std::unique_ptr<v2::event_provider> {
-            assert((flags & v2::EVENT_FLAGS_COUNTER) != 0);
-
             // TODO: just use per-context id?
             return std::make_unique<v2::provider_normal>(
                 context, v2::QUEUE_IMMEDIATE, flags);
           }),
+      eventPoolCacheRegular(this, phDevices[0]->Platform->getNumDevices(),
+                            [context = this, platform = phDevices[0]->Platform](
+                                DeviceId deviceId, v2::event_flags_t flags)
+                                -> std::unique_ptr<v2::event_provider> {
+                              std::ignore = deviceId;
+                              std::ignore = platform;
+
+                              // TODO: just use per-context id?
+                              return std::make_unique<v2::provider_normal>(
+                                  context, v2::QUEUE_REGULAR, flags);
+                            }),
       nativeEventsPool(this, std::make_unique<v2::provider_normal>(
                                  this, v2::QUEUE_IMMEDIATE,
                                  v2::EVENT_FLAGS_PROFILING_ENABLED)),

source/adapters/level_zero/v2/context.cpp

@@ -17,6 +17,8 @@
 #include "event_pool_cache.hpp"
 #include "usm.hpp"
 
+enum class PoolCacheType { Immediate, Regular };
+
 struct ur_context_handle_t_ : ur_object {
   ur_context_handle_t_(ze_context_handle_t hContext, uint32_t numDevices,
                        const ur_device_handle_t *phDevices, bool ownZeContext);
@@ -34,9 +36,18 @@ struct ur_context_handle_t_ : ur_object {
   getP2PDevices(ur_device_handle_t hDevice) const;
 
   v2::event_pool &getNativeEventsPool() { return nativeEventsPool; }
-  v2::event_pool_cache &getEventPoolCache() { return eventPoolCache; }
   v2::command_list_cache_t &getCommandListCache() { return commandListCache; }
-
+  v2::event_pool_cache &getEventPoolCache(PoolCacheType type) {
+    switch (type) {
+    case PoolCacheType::Immediate:
+      return eventPoolCacheImmediate;
+    case PoolCacheType::Regular:
+      return eventPoolCacheRegular;
+    default:
+      assert(false && "Requested invalid event pool cache type");
+      throw UR_RESULT_ERROR_INVALID_VALUE;
+    }
+  }
   // Checks if Device is covered by this context.
   // For that the Device or its root devices need to be in the context.
   bool isValidDevice(ur_device_handle_t Device) const;
@@ -45,7 +56,8 @@ struct ur_context_handle_t_ : ur_object {
   const v2::raii::ze_context_handle_t hContext;
   const std::vector<ur_device_handle_t> hDevices;
   v2::command_list_cache_t commandListCache;
-  v2::event_pool_cache eventPoolCache;
+  v2::event_pool_cache eventPoolCacheImmediate;
+  v2::event_pool_cache eventPoolCacheRegular;
 
   // pool used for urEventCreateWithNativeHandle when native handle is NULL
   // (uses non-counter based events to allow for signaling from host)

source/adapters/level_zero/v2/context.hpp

@@ -76,7 +76,7 @@ ur_queue_immediate_in_order_t::ur_queue_immediate_in_order_t(
               ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS,
               getZePriority(pProps ? pProps->flags : ur_queue_flags_t{}),
               getZeIndex(pProps)),
-          eventFlagsFromQueueFlags(flags), this) {}
+          eventFlagsFromQueueFlags(flags), this, PoolCacheType::Immediate) {}
 
 ur_queue_immediate_in_order_t::ur_queue_immediate_in_order_t(
     ur_context_handle_t hContext, ur_device_handle_t hDevice,
@@ -93,7 +93,7 @@ ur_queue_immediate_in_order_t::ur_queue_immediate_in_order_t(
                   }
                 }
               }),
-          eventFlagsFromQueueFlags(flags), this) {}
+          eventFlagsFromQueueFlags(flags), this, PoolCacheType::Immediate) {}
 
 ze_event_handle_t ur_queue_immediate_in_order_t::getSignalEvent(
     locked<ur_command_list_manager> &commandList, ur_event_handle_t *hUserEvent,
@@ -605,7 +605,8 @@ ur_queue_immediate_in_order_t::enqueueUSMAdvise(const void *pMem, size_t size,
   TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueUSMAdvise");
 
   auto commandListLocked = commandListManager.lock();
-  UR_CALL(commandListLocked->appendUSMAdvise(pMem, size, advice, phEvent));
+  UR_CALL(commandListLocked->appendUSMAdvise(pMem, size, advice, 0, nullptr,
+                                             phEvent));
   return UR_RESULT_SUCCESS;
 }