[UR][Graph] Serialize submissions of the same command-buffer (#18295)

Update the UR command-buffer enqueue semantics such that command-buffer
submissions are ordered with a dependency on previous submissions of the
same command-buffer.

That is the same semantic as we currently specify for executable graph
objects in the SYCL-Graph extension. Allowing for a more efficient
SYCL-RT implementation on top of UR where less UR events need to be
created (future work).

OpenCL is the only adapter that needs updated to provide these
semantics, where we now track the `cl_event` returned from the most
recent submission of a command-buffer, so that it can be used as a
dependency of any future command-buffer submissions.

Author: Ewan Crawford

unified-runtime/scripts/core/EXP-COMMAND-BUFFER.rst

@@ -224,7 +224,7 @@ Enqueueing Command-Buffers
 --------------------------------------------------------------------------------
 
 Command-buffers are submitted for execution on a ${x}_queue_handle_t with an
-optional list of dependent events. An event is returned which tracks the
+optional list of dependent events. An event can be returned which tracks the
 execution of the command-buffer, and will be complete when all appended commands
 have finished executing.
 
@@ -238,16 +238,17 @@ of the same command-buffer is still awaiting completion. That is, the user is no
 required to do a blocking wait on the completion of the first command-buffer
 submission before making a second submission of the command-buffer.
 
-Submissions of the same command-buffer should be synchronized to prevent
-concurrent execution. For example, by using events, barriers, or in-order queue
-dependencies. The behavior of multiple submissions of the same command-buffer
-that can execute concurrently is undefined.
+Each submissions of a command-buffer is ordered behind previous submissions of
+the same command-buffer. As well as respecting the other synchronization
+dependencies set by the user, such as events, barriers, or in-order queue
+dependencies.
 
 .. parsed-literal::
-    // Valid usage if hQueue is in-order but undefined behavior is out-of-order
-    ${x}EnqueueCommandBufferExp(hQueue, hCommandBuffer, 0, nullptr,
+    // Submission of hCommandBuffer to hQueueB as an implicit dependency on
+    // prior submission to hQueueA.
+    ${x}EnqueueCommandBufferExp(hQueueA, hCommandBuffer, 0, nullptr,
                                 nullptr);
-    ${x}EnqueueCommandBufferExp(hQueue, hCommandBuffer, 0, nullptr,
+    ${x}EnqueueCommandBufferExp(hQueueB, hCommandBuffer, 0, nullptr,
                                 nullptr);
 
 

unified-runtime/source/adapters/opencl/command_buffer.cpp

@@ -21,6 +21,9 @@
 /// command-buffer to free the underlying object.
 ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() {
   urQueueRelease(hInternalQueue);
+  if (LastSubmission) {
+    clReleaseEvent(LastSubmission);
+  }
 
   cl_context CLContext = hContext->CLContext;
   cl_ext::clReleaseCommandBufferKHR_fn clReleaseCommandBufferKHR = nullptr;
@@ -501,16 +504,34 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp(
           ur::cl::getAdapter()->fnCache.clEnqueueCommandBufferKHRCache,
           cl_ext::EnqueueCommandBufferName, &clEnqueueCommandBufferKHR));
 
-  const uint32_t NumberOfQueues = 1;
-  cl_event Event;
-  std::vector<cl_event> CLWaitEvents(numEventsInWaitList);
+  // If we've submitted the command-buffer before, then add a dependency on the
+  // last submission.
+  bool AddExtraDep = hCommandBuffer->LastSubmission != nullptr;
+  uint32_t CLWaitListLength =
+      AddExtraDep ? numEventsInWaitList + 1 : numEventsInWaitList;
+  std::vector<cl_event> CLWaitEvents(CLWaitListLength);
   for (uint32_t i = 0; i < numEventsInWaitList; i++) {
     CLWaitEvents[i] = phEventWaitList[i]->CLEvent;
   }
+  if (AddExtraDep) {
+    CLWaitEvents[numEventsInWaitList] = hCommandBuffer->LastSubmission;
+  }
+
+  // Always get an event as we need it to serialize any future submissions
+  // of the command-buffer with.
+  cl_event Event;
   cl_command_queue CLQueue = hQueue->CLQueue;
-  CL_RETURN_ON_FAILURE(clEnqueueCommandBufferKHR(
-      NumberOfQueues, &CLQueue, hCommandBuffer->CLCommandBuffer,
-      numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event)));
+  CL_RETURN_ON_FAILURE(
+      clEnqueueCommandBufferKHR(1, &CLQueue, hCommandBuffer->CLCommandBuffer,
+                                CLWaitListLength, CLWaitEvents.data(), &Event));
+
+  // Retain event so that if a user manually destroys the returned UR
+  // event the adapter still has a valid handle.
+  clRetainEvent(Event);
+  if (hCommandBuffer->LastSubmission) {
+    clReleaseEvent(hCommandBuffer->LastSubmission);
+  }
+  hCommandBuffer->LastSubmission = Event;
 
   UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent));
   return UR_RESULT_SUCCESS;

unified-runtime/source/adapters/opencl/command_buffer.hpp

@@ -55,6 +55,8 @@ struct ur_exp_command_buffer_handle_t_ {
       CommandHandles;
   /// Object reference count
   std::atomic_uint32_t RefCount;
+  /// Track last submission of the command-buffer
+  cl_event LastSubmission;
 
   ur_exp_command_buffer_handle_t_(ur_queue_handle_t hQueue,
                                   ur_context_handle_t hContext,
@@ -63,7 +65,8 @@ struct ur_exp_command_buffer_handle_t_ {
                                   bool IsUpdatable, bool IsInOrder)
       : hInternalQueue(hQueue), hContext(hContext), hDevice(hDevice),
         CLCommandBuffer(CLCommandBuffer), IsUpdatable(IsUpdatable),
-        IsInOrder(IsInOrder), IsFinalized(false), RefCount(0) {}
+        IsInOrder(IsInOrder), IsFinalized(false), RefCount(0),
+        LastSubmission(nullptr) {}
 
   ~ur_exp_command_buffer_handle_t_();
 

unified-runtime/test/conformance/exp_command_buffer/CMakeLists.txt

@@ -16,6 +16,7 @@ add_conformance_test_with_kernels_environment(exp_command_buffer
   write.cpp
   rect_read.cpp
   rect_write.cpp
+  enqueue.cpp
   update/buffer_fill_kernel_update.cpp
   update/invalid_update.cpp
   update/kernel_handle_update.cpp
@@ -26,6 +27,7 @@ add_conformance_test_with_kernels_environment(exp_command_buffer
   update/event_sync.cpp
   update/kernel_event_sync.cpp
   update/local_memory_update.cpp
+  update/enqueue_update.cpp
   regression/usm_copy.cpp
 )
 

unified-runtime/test/conformance/exp_command_buffer/enqueue.cpp

@@ -0,0 +1,134 @@
+// Copyright (C) 2025 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
+// Exceptions. See LICENSE.TXT
+//
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "fixtures.h"
+#include <array>
+#include <cstring>
+
+// Tests that adapter implementation of urEnqueueCommandBufferExp serializes
+// submissions of the same UR command-buffer object with respect to previous
+// submissions.
+//
+// Done using a kernel that increments a value, so the ordering of the two
+// submission isn't tested, only that they didn't run concurrently. See the
+// enqueue_update.cpp test for a test verifying the order of submissions, as
+// the input/output to the kernels can be modified between the submissions.
+struct urEnqueueCommandBufferExpTest
+    : uur::command_buffer::urCommandBufferExpExecutionTest {
+  virtual void SetUp() override {
+    program_name = "increment";
+    UUR_RETURN_ON_FATAL_FAILURE(urCommandBufferExpExecutionTest::SetUp());
+
+    // Create an in-order queue
+    ur_queue_properties_t queue_properties = {
+        UR_STRUCTURE_TYPE_QUEUE_PROPERTIES, nullptr, 0};
+    ASSERT_SUCCESS(
+        urQueueCreate(context, device, &queue_properties, &in_order_queue));
+
+    // Create an out-of-order queue
+    queue_properties.flags = UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE;
+    ASSERT_SUCCESS(
+        urQueueCreate(context, device, &queue_properties, &out_of_order_queue));
+    ASSERT_NE(out_of_order_queue, nullptr);
+
+    ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr,
+                                    allocation_size, &device_ptr));
+    ASSERT_NE(device_ptr, nullptr);
+
+    uint32_t zero_pattern = 0;
+    ASSERT_SUCCESS(urEnqueueUSMFill(queue, device_ptr, sizeof(zero_pattern),
+                                    &zero_pattern, allocation_size, 0, nullptr,
+                                    nullptr));
+    ASSERT_SUCCESS(urQueueFinish(queue));
+
+    // Create command-buffer with a single kernel that does "Ptr[i] += 1;"
+    ASSERT_SUCCESS(urKernelSetArgPointer(kernel, 0, nullptr, device_ptr));
+    ASSERT_SUCCESS(urCommandBufferAppendKernelLaunchExp(
+        cmd_buf_handle, kernel, n_dimensions, &global_offset, &global_size,
+        nullptr, 0, nullptr, 0, nullptr, 0, nullptr, nullptr, nullptr,
+        nullptr));
+    ASSERT_SUCCESS(urCommandBufferFinalizeExp(cmd_buf_handle));
+  }
+
+  virtual void TearDown() override {
+    if (in_order_queue) {
+      EXPECT_SUCCESS(urQueueRelease(in_order_queue));
+    }
+
+    if (out_of_order_queue) {
+      EXPECT_SUCCESS(urQueueRelease(out_of_order_queue));
+    }
+
+    if (device_ptr) {
+      EXPECT_SUCCESS(urUSMFree(context, device_ptr));
+    }
+
+    UUR_RETURN_ON_FATAL_FAILURE(urCommandBufferExpExecutionTest::TearDown());
+  }
+
+  ur_queue_handle_t in_order_queue = nullptr;
+  ur_queue_handle_t out_of_order_queue = nullptr;
+
+  static constexpr size_t global_size = 16;
+  static constexpr size_t global_offset = 0;
+  static constexpr size_t n_dimensions = 1;
+  static constexpr size_t allocation_size = sizeof(uint32_t) * global_size;
+  void *device_ptr = nullptr;
+};
+
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urEnqueueCommandBufferExpTest);
+
+// Tests that the same command-buffer submitted across different in-order
+// queues has an implicit dependency on first submission
+TEST_P(urEnqueueCommandBufferExpTest, SerializeAcrossQueues) {
+  // Execute command-buffer to first in-order queue (created by parent
+  // urQueueTest fixture)
+  ASSERT_SUCCESS(
+      urEnqueueCommandBufferExp(queue, cmd_buf_handle, 0, nullptr, nullptr));
+
+  // Execute command-buffer to second in-order queue, should have implicit
+  // dependency on first submission.
+  ASSERT_SUCCESS(urEnqueueCommandBufferExp(in_order_queue, cmd_buf_handle, 0,
+                                           nullptr, nullptr));
+
+  // Wait for both submissions to complete
+  ASSERT_SUCCESS(urQueueFlush(queue));
+  ASSERT_SUCCESS(urQueueFinish(in_order_queue));
+
+  std::vector<uint32_t> Output(global_size);
+  ASSERT_SUCCESS(urEnqueueUSMMemcpy(queue, false, Output.data(), device_ptr,
+                                    allocation_size, 0, nullptr, nullptr));
+  ASSERT_SUCCESS(urQueueFinish(queue));
+
+  // Verify
+  const uint32_t reference = 2;
+  for (size_t i = 0; i < global_size; i++) {
+    ASSERT_EQ(reference, Output[i]);
+  }
+}
+
+// Tests that submitting a command-buffer twice to an out-of-order queue
+// relying on implicit serialization semantics for dependencies.
+TEST_P(urEnqueueCommandBufferExpTest, SerializeOutofOrderQueue) {
+  ASSERT_SUCCESS(urEnqueueCommandBufferExp(out_of_order_queue, cmd_buf_handle,
+                                           0, nullptr, nullptr));
+  ASSERT_SUCCESS(urEnqueueCommandBufferExp(out_of_order_queue, cmd_buf_handle,
+                                           0, nullptr, nullptr));
+
+  // Wait for both submissions to complete
+  ASSERT_SUCCESS(urQueueFinish(out_of_order_queue));
+
+  std::vector<uint32_t> Output(global_size);
+  ASSERT_SUCCESS(urEnqueueUSMMemcpy(out_of_order_queue, true, Output.data(),
+                                    device_ptr, allocation_size, 0, nullptr,
+                                    nullptr));
+
+  // Verify
+  const uint32_t reference = 2;
+  for (size_t i = 0; i < global_size; i++) {
+    ASSERT_EQ(reference, Output[i]);
+  }
+}