[UR][L0v2] Implement IPC event sharing (#22445)

Adds the producer/consumer side of UR's IPC event extension on top of
Level Zero's counter-based event IPC APIs
(zeEventCounterBased{Create,Get/Open/CloseIpcHandle}).
* Adjusted urEventCreateExp for ipc events on top of changes introduced
by reusable-event feature.
* IPC events are not pooled. The primary use case is reusable IPC
events, where the user reuses one event many times, so pooling buys
little. And sharing pooled events across processes is unsafe because a
recycled event hands one process state another process is still relying
on.
* UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP is gated on v2/Linux, BMG+, and
latest driver - this might need to be revised in the future.

Conformance tests signal the producer event via APIs introduced by the
reusable-event extension.

Assisted-By: Claude

Author: againull

unified-runtime/source/adapters/level_zero/device.cpp

@@ -1396,6 +1396,19 @@ ur_result_t urDeviceGetInfo(
 #else
     return ReturnValue(false);
 #endif
+  case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP: {
+#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__)
+    constexpr uint32_t MinDriverBuild = 38646;
+    ZeStruct<ze_driver_properties_t> ZeDriverProperties;
+    ZE2UR_CALL(zeDriverGetProperties,
+               (Device->Platform->ZeDriver, &ZeDriverProperties));
+    const uint32_t DriverBuild = ZeDriverProperties.driverVersion & 0xFFFF;
+    return ReturnValue(static_cast<ur_bool_t>(Device->isBMGOrNewer() &&
+                                              DriverBuild >= MinDriverBuild));
+#else
+    return ReturnValue(false);
+#endif
+  }
   case UR_DEVICE_INFO_ASYNC_BARRIER:
     return ReturnValue(false);
   case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORT:

unified-runtime/source/adapters/level_zero/device.hpp

@@ -209,6 +209,11 @@ struct ur_device_handle_t_ : ur_object {
             ZeDeviceIpVersionExt->ipVersion >= 0x030f0000);
   }
 
+  bool isBMGOrNewer() {
+    return (ZeDeviceProperties->vendorId == 0x8086 &&
+            ZeDeviceIpVersionExt->ipVersion >= 0x05004000);
+  }
+
   bool isIntegrated() {
     return (ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED);
   }

unified-runtime/source/adapters/level_zero/v2/common.hpp

@@ -95,10 +95,14 @@ struct ze_handle_wrapper {
   bool ownZeHandle;
 };
 
+#define HANDLE_WRAPPER_TYPE_NAMED(Alias, ZeHandleT, DestroyFunc)               \
+  inline constexpr char Alias##_destroyName[] = #DestroyFunc;                  \
+  using Alias =                                                                \
+      ze_handle_wrapper<::ZeHandleT, DestroyFunc, Alias##_destroyName>;
+
+// Common case: the alias name matches the raw handle type name.
 #define HANDLE_WRAPPER_TYPE(ZeHandleT, DestroyFunc)                            \
-  inline constexpr char ZeHandleT##_destroyName[] = #DestroyFunc;              \
-  using ZeHandleT =                                                            \
-      ze_handle_wrapper<::ZeHandleT, DestroyFunc, ZeHandleT##_destroyName>;
+  HANDLE_WRAPPER_TYPE_NAMED(ZeHandleT, ZeHandleT, DestroyFunc)
 
 HANDLE_WRAPPER_TYPE(ze_kernel_handle_t, zeKernelDestroy)
 HANDLE_WRAPPER_TYPE(ze_event_handle_t, zeEventDestroy)
@@ -107,6 +111,12 @@ HANDLE_WRAPPER_TYPE(ze_context_handle_t, zeContextDestroy)
 HANDLE_WRAPPER_TYPE(ze_command_list_handle_t, zeCommandListDestroy)
 HANDLE_WRAPPER_TYPE(ze_image_handle_t, zeImageDestroy)
 
+// Counter-based event IPC handle opened via zeEventCounterBasedOpenIpcHandle.
+// Same underlying handle type as ze_event_handle_t but a different teardown
+// function, hence a distinct alias.
+HANDLE_WRAPPER_TYPE_NAMED(ipc_event_handle_t, ze_event_handle_t,
+                          zeEventCounterBasedCloseIpcHandle)
+
 template <typename RawHandle, ur_result_t (*retain)(RawHandle),
           ur_result_t (*release)(RawHandle)>
 struct ur_handle {

unified-runtime/source/adapters/level_zero/v2/event.cpp

@@ -17,6 +17,7 @@
 #include "queue_api.hpp"
 #include "queue_handle.hpp"
 
+#include "../device.hpp"
 #include "../ur_interface_loader.hpp"
 
 static uint64_t adjustEndEventTimestamp(uint64_t adjustedStartTimestamp,
@@ -122,7 +123,7 @@ void ur_event_handle_t_::setQueue(ur_queue_t_ *hQueue) {
 }
 
 ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t hContext,
-                                       v2::raii::ze_event_handle_t hZeEvent,
+                                       event_variant hZeEvent,
                                        v2::event_flags_t flags)
     : ur_event_handle_t_(hContext, std::move(hZeEvent), flags, nullptr) {}
 
@@ -186,6 +187,7 @@ struct event_teardown {
   }
 
   void operator()(const v2::raii::ze_event_handle_t &) { delete event; }
+  void operator()(const v2::raii::ipc_event_handle_t &) { delete event; }
 };
 
 ur_result_t ur_event_handle_t_::release() {
@@ -204,6 +206,14 @@ bool ur_event_handle_t_::isProfilingEnabled() const {
   return flags & v2::EVENT_FLAGS_PROFILING_ENABLED;
 }
 
+bool ur_event_handle_t_::isIpcCapable() const {
+  return flags & v2::EVENT_FLAGS_IPC;
+}
+
+bool ur_event_handle_t_::isIpcImported() const {
+  return flags & v2::EVENT_FLAGS_IPC_IMPORTED;
+}
+
 std::pair<uint64_t *, ze_event_handle_t>
 ur_event_handle_t_::getEventEndTimestampAndHandle() {
   return {profilingData.eventEndTimestampAddr(), getZeEvent()};
@@ -427,10 +437,10 @@ ur_result_t urEventCreateExp(ur_context_handle_t hContext,
                              ur_device_handle_t hDevice,
                              const ur_exp_event_desc_t *pEventDesc,
                              ur_event_handle_t *phEvent) try {
-  if (!hContext || !hDevice)
-    return UR_RESULT_ERROR_INVALID_NULL_HANDLE;
-  if (!pEventDesc || !phEvent)
-    return UR_RESULT_ERROR_INVALID_NULL_POINTER;
+  UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE);
+  UR_ASSERT(pEventDesc && phEvent, UR_RESULT_ERROR_INVALID_NULL_POINTER);
+  UR_ASSERT(!(pEventDesc->flags & UR_EXP_EVENT_FLAGS_MASK),
+            UR_RESULT_ERROR_INVALID_ENUMERATION);
 
   const v2::event_flags_t flags =
       v2::EVENT_FLAGS_COUNTER |
@@ -439,26 +449,18 @@ ur_result_t urEventCreateExp(ur_context_handle_t hContext,
            : 0) |
       (pEventDesc->flags & UR_EXP_EVENT_FLAG_IPC_EXP ? v2::EVENT_FLAGS_IPC : 0);
 
-  if (flags & v2::EVENT_FLAGS_IPC && flags & v2::EVENT_FLAGS_PROFILING_ENABLED)
-    return UR_RESULT_ERROR_INVALID_ENUMERATION;
+  UR_ASSERT(!(flags & v2::EVENT_FLAGS_IPC &&
+              flags & v2::EVENT_FLAGS_PROFILING_ENABLED),
+            UR_RESULT_ERROR_INVALID_VALUE);
 
   auto eventPool =
       hContext->getReusableEventPoolCache().borrow(hDevice->Id.value(), flags);
   assert(eventPool);
 
-  if (!(flags & v2::EVENT_FLAGS_IPC)) {
-    *phEvent = eventPool->allocate();
-    return UR_RESULT_SUCCESS;
-  }
-
-  v2::raii::cache_borrowed_event borrowed =
-      eventPool->getProvider()->allocate();
-  v2::raii::ze_event_handle_t ownedEvent(borrowed.release(),
-                                         /*ownZeHandle=*/true);
-  *phEvent = new ur_event_handle_t_(hContext, std::move(ownedEvent), flags);
-
-  // Handle IPC event creation specifics here...
-
+  // IPC events must not be recycled (their native handle may outlive this
+  // process's reference), so they get a detached, self-owning event.
+  *phEvent = (flags & v2::EVENT_FLAGS_IPC) ? eventPool->allocateDetached()
+                                           : eventPool->allocate();
   return UR_RESULT_SUCCESS;
 } catch (...) {
   return exceptionToResult(std::current_exception());

unified-runtime/source/adapters/level_zero/v2/event.hpp

@@ -60,8 +60,11 @@ struct ur_event_handle_t_ : ur_object {
   // - cache_borrowed_event: pooled event; it is returned to the pool.
   // - ze_event_handle_t: standalone/native event; the wrapper's ownZeHandle
   //   flag controls whether zeEventDestroy runs on destruction.
+  // - ipc_event_handle_t: opened from an IPC handle in another process;
+  //   torn down via zeEventCounterBasedCloseIpcHandle.
   using event_variant =
-      std::variant<v2::raii::cache_borrowed_event, v2::raii::ze_event_handle_t>;
+      std::variant<v2::raii::cache_borrowed_event, v2::raii::ze_event_handle_t,
+                   v2::raii::ipc_event_handle_t>;
 
   ur_event_handle_t_(ur_context_handle_t hContext,
                      v2::raii::cache_borrowed_event eventAllocation,
@@ -71,8 +74,7 @@ struct ur_event_handle_t_ : ur_object {
                      ur_native_handle_t hNativeEvent,
                      const ur_event_native_properties_t *pProperties);
 
-  ur_event_handle_t_(ur_context_handle_t hContext,
-                     v2::raii::ze_event_handle_t hZeEvent,
+  ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent,
                      v2::event_flags_t flags);
 
   // Set the queue and command that this event is associated with
@@ -105,6 +107,12 @@ struct ur_event_handle_t_ : ur_object {
   // Tells if this event comes from a pool that has profiling enabled.
   bool isProfilingEnabled() const;
 
+  // True for IPC-shareable events.
+  bool isIpcCapable() const;
+
+  // True for events opened via urIPCOpenEventHandleExp.
+  bool isIpcImported() const;
+
   // Queue associated with this event. Can be nullptr (for native events)
   ur_queue_t_ *getQueue() const;
 

unified-runtime/source/adapters/level_zero/v2/event_pool.cpp

@@ -42,6 +42,14 @@ ur_event_handle_t event_pool::allocate() {
   return event;
 }
 
+ur_event_handle_t event_pool::allocateDetached() {
+  TRACK_SCOPE_LATENCY("event_pool::allocateDetached");
+  raii::ze_event_handle_t ownedEvent(provider->allocate().release(),
+                                     /*ownZeHandle=*/true);
+  return new ur_event_handle_t_(hContext, std::move(ownedEvent),
+                                provider->eventFlags());
+}
+
 void event_pool::free(ur_event_handle_t event) {
   TRACK_SCOPE_LATENCY("event_pool::free");
 

unified-runtime/source/adapters/level_zero/v2/event_pool.hpp

@@ -43,6 +43,11 @@ class event_pool {
   // Allocate an event from the pool. Thread safe.
   ur_event_handle_t allocate();
 
+  // Allocate a detached event that owns its underlying ze_event_handle_t and is
+  // never returned to the pool for recycling, it is destroyed when released.
+  // Used for events that must not be recycled. Thread safe.
+  ur_event_handle_t allocateDetached();
+
   // Free an event back to the pool. Thread safe.
   void free(ur_event_handle_t event);
 

unified-runtime/source/adapters/level_zero/v2/event_provider.hpp

@@ -24,8 +24,14 @@ using event_flags_t = uint32_t;
 enum event_flag_t {
   EVENT_FLAGS_COUNTER = UR_BIT(0),
   EVENT_FLAGS_PROFILING_ENABLED = UR_BIT(1),
-  EVENT_FLAGS_IPC = UR_BIT(2)
+  // IPC-shareable producer event.
+  EVENT_FLAGS_IPC = UR_BIT(2),
+  // Event opened from an IPC handle.
+  EVENT_FLAGS_IPC_IMPORTED = UR_BIT(3),
 };
+// Number of flag bits that index into event_pool_cache.
+// EVENT_FLAGS_IPC_IMPORTED is excluded because imported events are created
+// directly.
 static constexpr size_t EVENT_FLAGS_USED_BITS = 3;
 
 enum queue_type {

unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp

@@ -49,6 +49,10 @@ static zex_counter_based_event_exp_flags_t createZeFlags(queue_type queueType,
     zeFlags |= ZEX_COUNTER_BASED_EVENT_FLAG_KERNEL_TIMESTAMP;
   }
 
+  if (flags & EVENT_FLAGS_IPC) {
+    zeFlags |= ZEX_COUNTER_BASED_EVENT_FLAG_IPC;
+  }
+
   if (queueType == QUEUE_IMMEDIATE) {
     zeFlags |= ZEX_COUNTER_BASED_EVENT_FLAG_IMMEDIATE;
   }

unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp

@@ -7,28 +7,89 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <memory>
+
 #include <unified-runtime/ur_api.h>
 
+#include "../platform.hpp"
 #include "../ur_interface_loader.hpp"
+#include "context.hpp"
+#include "event.hpp"
 
 namespace ur::level_zero {
 
-ur_result_t urIPCGetEventHandleExp(ur_event_handle_t /*hEvent*/,
-                                   void ** /*ppIPCEventHandleData*/,
-                                   size_t * /*pIPCEventHandleDataSizeRet*/) {
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+namespace {
+
+// Fixed L0 IPC handle size; reported by Get and validated by Open.
+constexpr size_t kIpcEventHandleDataSize =
+    sizeof(ze_ipc_event_counter_based_handle_t);
+
+} // namespace
+
+ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent,
+                                   void **ppIPCEventHandleData,
+                                   size_t *pIPCEventHandleDataSizeRet) try {
+  UR_ASSERT(hEvent, UR_RESULT_ERROR_INVALID_NULL_HANDLE);
+  UR_ASSERT(ppIPCEventHandleData && pIPCEventHandleDataSizeRet,
+            UR_RESULT_ERROR_INVALID_NULL_POINTER);
+
+  std::shared_lock<ur_shared_mutex> lock(hEvent->Mutex);
+
+  UR_ASSERT(hEvent->isIpcCapable() && !hEvent->isIpcImported(),
+            UR_RESULT_ERROR_INVALID_EVENT);
+  UR_ASSERT(!hEvent->isProfilingEnabled() && !hEvent->isTimestamped(),
+            UR_RESULT_ERROR_UNSUPPORTED_FEATURE);
+
+  auto handle = std::make_unique<ze_ipc_event_counter_based_handle_t>();
+  ZE2UR_CALL(zeEventCounterBasedGetIpcHandle,
+             (hEvent->getZeEvent(), handle.get()));
+
+  // Caller releases the buffer via urIPCPutEventHandleExp.
+  *ppIPCEventHandleData = handle.release();
+  *pIPCEventHandleDataSizeRet = kIpcEventHandleDataSize;
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
 }
 
 ur_result_t urIPCPutEventHandleExp(ur_context_handle_t /*hContext*/,
-                                   void * /*pIPCEventHandleData*/) {
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+                                   void *pIPCEventHandleData) try {
+  UR_ASSERT(pIPCEventHandleData, UR_RESULT_ERROR_INVALID_NULL_POINTER);
+  // Free the buffer allocated by urIPCGetEventHandleExp via RAII.
+  std::unique_ptr<ze_ipc_event_counter_based_handle_t> owner(
+      static_cast<ze_ipc_event_counter_based_handle_t *>(pIPCEventHandleData));
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
 }
 
-ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t /*hContext*/,
-                                    const void * /*pIPCEventHandleData*/,
-                                    size_t /*ipcEventHandleDataSize*/,
-                                    ur_event_handle_t * /*phEvent*/) {
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext,
+                                    const void *pIPCEventHandleData,
+                                    size_t ipcEventHandleDataSize,
+                                    ur_event_handle_t *phEvent) try {
+  UR_ASSERT(hContext, UR_RESULT_ERROR_INVALID_NULL_HANDLE);
+  UR_ASSERT(pIPCEventHandleData && phEvent,
+            UR_RESULT_ERROR_INVALID_NULL_POINTER);
+  UR_ASSERT(ipcEventHandleDataSize == kIpcEventHandleDataSize,
+            UR_RESULT_ERROR_INVALID_VALUE);
+
+  // The driver consumes the handle by value.
+  ze_ipc_event_counter_based_handle_t handle =
+      *static_cast<const ze_ipc_event_counter_based_handle_t *>(
+          pIPCEventHandleData);
+  ze_event_handle_t hZeEvent = nullptr;
+  ZE2UR_CALL(zeEventCounterBasedOpenIpcHandle,
+             (hContext->getZeHandle(), handle, &hZeEvent));
+
+  // The ipc_event_handle_t variant runs zeEventCounterBasedCloseIpcHandle on
+  // urEventRelease.
+  *phEvent =
+      new ur_event_handle_t_(hContext, v2::raii::ipc_event_handle_t{hZeEvent},
+                             v2::EVENT_FLAGS_COUNTER | v2::EVENT_FLAGS_IPC |
+                                 v2::EVENT_FLAGS_IPC_IMPORTED);
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
 }
 
 } // namespace ur::level_zero