Map copy/fill to SVM (#18177)

There is currently no cl_khr_command_buffer or
cl_intel_unified_shared_memory entry-point for appending USM copy or
fill commands to a command-buffer. This prevents these commands from
being added to a graph for the OpenCL backend.

The long term solution to this is the OpenCL USVM extension which will
align USM and SVM, allowing the existing SVM entry-points to be used. To
prepare for this, map the UR entry-points to the cl_khr_command_buffer
SVM copy/fill commands.

This will work on OpenCL implementations that share an implementation
for USM and SVM.

Signed-off-by: Ewan Crawford <ewan@codeplay.com>

Author: Ewan Crawford

source/adapters/opencl/command_buffer.cpp

@@ -204,33 +204,59 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp(
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] void *pDst, [[maybe_unused]] const void *pSrc,
-    [[maybe_unused]] size_t size,
-    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
-    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
-        *pSyncPointWaitList,
-    [[maybe_unused]] uint32_t numEventsInWaitList,
-    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
-    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
-    [[maybe_unused]] ur_event_handle_t *phEvent,
-    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, const void *pSrc,
+    size_t size, uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t,
+    const ur_event_handle_t *, ur_exp_command_buffer_sync_point_t *pSyncPoint,
+    ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) {
+  // No extension entry-point exists for USM memcpy, use SVM memcpy in
+  // preparation for USVM.
+  cl_context CLContext = hCommandBuffer->hContext->CLContext;
+  cl_ext::clCommandSVMMemcpyKHR_fn clCommandSVMMemcpyKHR = nullptr;
+  UR_RETURN_ON_FAILURE(
+      cl_ext::getExtFuncFromContext<decltype(clCommandSVMMemcpyKHR)>(
+          CLContext, ur::cl::getAdapter()->fnCache.clCommandSVMMemcpyKHRCache,
+          cl_ext::CommandSVMMemcpyName, &clCommandSVMMemcpyKHR));
+
+  const bool IsInOrder = hCommandBuffer->IsInOrder;
+  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
+  const cl_sync_point_khr *SyncPointWaitList =
+      IsInOrder ? nullptr : pSyncPointWaitList;
+  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
+  CL_RETURN_ON_FAILURE(clCommandSVMMemcpyKHR(
+      hCommandBuffer->CLCommandBuffer, nullptr, nullptr, pDst, pSrc, size,
+      WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] void *pMemory, [[maybe_unused]] const void *pPattern,
-    [[maybe_unused]] size_t patternSize, [[maybe_unused]] size_t size,
-    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
-    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
-        *pSyncPointWaitList,
-    [[maybe_unused]] uint32_t numEventsInWaitList,
-    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
-    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
-    [[maybe_unused]] ur_event_handle_t *phEvent,
-    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory,
+    const void *pPattern, size_t patternSize, size_t size,
+    uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t,
+    const ur_event_handle_t *, ur_exp_command_buffer_sync_point_t *pSyncPoint,
+    ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) {
+  // No extension entry-point exists for USM fill, use SVM fill in preparation
+  // for USVM.
+  cl_context CLContext = hCommandBuffer->hContext->CLContext;
+  cl_ext::clCommandSVMMemFillKHR_fn clCommandSVMMemFillKHR = nullptr;
+  UR_RETURN_ON_FAILURE(
+      cl_ext::getExtFuncFromContext<decltype(clCommandSVMMemFillKHR)>(
+          CLContext, ur::cl::getAdapter()->fnCache.clCommandSVMMemFillKHRCache,
+          cl_ext::CommandSVMMemFillName, &clCommandSVMMemFillKHR));
+
+  const bool IsInOrder = hCommandBuffer->IsInOrder;
+  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
+  const cl_sync_point_khr *SyncPointWaitList =
+      IsInOrder ? nullptr : pSyncPointWaitList;
+  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
+  CL_RETURN_ON_FAILURE(
+      clCommandSVMMemFillKHR(hCommandBuffer->CLCommandBuffer, nullptr, nullptr,
+                             pMemory, pPattern, patternSize, size, WaitListSize,
+                             SyncPointWaitList, RetSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp(

source/adapters/opencl/common.hpp

@@ -201,6 +201,8 @@ CONSTFIX char CommandCopyBufferRectName[] = "clCommandCopyBufferRectKHR";
 CONSTFIX char CommandFillBufferName[] = "clCommandFillBufferKHR";
 CONSTFIX char CommandBarrierWithWaitListName[] =
     "clCommandBarrierWithWaitListKHR";
+CONSTFIX char CommandSVMMemcpyName[] = "clCommandSVMMemcpyKHR";
+CONSTFIX char CommandSVMMemFillName[] = "clCommandSVMMemFillKHR";
 CONSTFIX char EnqueueCommandBufferName[] = "clEnqueueCommandBufferKHR";
 CONSTFIX char GetCommandBufferInfoName[] = "clGetCommandBufferInfoKHR";
 CONSTFIX char UpdateMutableCommandsName[] = "clUpdateMutableCommandsKHR";
@@ -296,6 +298,21 @@ using clCommandBarrierWithWaitListKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
     const cl_sync_point_khr *sync_point_wait_list,
     cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
 
+using clCommandSVMMemcpyKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    const cl_command_properties_khr *properties, void *dst_ptr,
+    const void *src_ptr, size_t size, cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
+using clCommandSVMMemFillKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    const cl_command_properties_khr *properties, void *svm_ptr,
+    const void *pattern, size_t pattern_size, size_t size,
+    cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
 using clEnqueueCommandBufferKHR_fn = CL_API_ENTRY
 cl_int(CL_API_CALL *)(cl_uint num_queues, cl_command_queue *queues,
                       cl_command_buffer_khr command_buffer,

source/adapters/opencl/extension_functions.def

@@ -22,6 +22,8 @@ CL_EXTENSION_FUNC(clCommandCopyBufferKHR)
 CL_EXTENSION_FUNC(clCommandCopyBufferRectKHR)
 CL_EXTENSION_FUNC(clCommandFillBufferKHR)
 CL_EXTENSION_FUNC(clCommandBarrierWithWaitListKHR)
+CL_EXTENSION_FUNC(clCommandSVMMemFillKHR)
+CL_EXTENSION_FUNC(clCommandSVMMemcpyKHR)
 CL_EXTENSION_FUNC(clEnqueueCommandBufferKHR)
 CL_EXTENSION_FUNC(clGetCommandBufferInfoKHR)
 CL_EXTENSION_FUNC(clUpdateMutableCommandsKHR)

test/conformance/exp_command_buffer/commands.cpp

@@ -56,18 +56,12 @@ struct urCommandBufferCommandsTest
 UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCommandBufferCommandsTest);
 
 TEST_P(urCommandBufferCommandsTest, urCommandBufferAppendUSMMemcpyExp) {
-  // No USM memcpy command in cl_khr_command_buffer
-  UUR_KNOWN_FAILURE_ON(uur::OpenCL{});
-
   ASSERT_SUCCESS(urCommandBufferAppendUSMMemcpyExp(
       cmd_buf_handle, device_ptrs[0], device_ptrs[1], allocation_size, 0,
       nullptr, 0, nullptr, nullptr, nullptr, nullptr));
 }
 
 TEST_P(urCommandBufferCommandsTest, urCommandBufferAppendUSMFillExp) {
-  // No USM fill command in cl_khr_command_buffer
-  UUR_KNOWN_FAILURE_ON(uur::OpenCL{});
-
   uint32_t pattern = 42;
   ASSERT_SUCCESS(urCommandBufferAppendUSMFillExp(
       cmd_buf_handle, device_ptrs[0], &pattern, sizeof(pattern),

test/conformance/exp_command_buffer/fill.cpp

@@ -155,8 +155,11 @@ TEST_P(urCommandBufferFillCommandsTest, ExecuteTwice) {
 }
 
 TEST_P(urCommandBufferFillCommandsTest, USM) {
-  // No USM fill command in cl_khr_command_buffer
-  UUR_KNOWN_FAILURE_ON(uur::OpenCL{});
+  if (pattern_size > 128) {
+    // clCommandSVMMemFillKHR returns an error according to the spec if pattern
+    // size is not one of {1, 2, 4, 8, 16, 32, 64, 128}
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{});
+  }
 
   ASSERT_SUCCESS(urCommandBufferAppendUSMFillExp(
       cmd_buf_handle, device_ptr, pattern.data(), pattern_size, size, 0,