command_buffer.cpp

//===--------- command_buffer.cpp - OpenCL Adapter ---------------------===//
//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM
// Exceptions. See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "command_buffer.hpp"
#include "adapter.hpp"
#include "common.hpp"
#include "context.hpp"
#include "event.hpp"
#include "kernel.hpp"
#include "memory.hpp"
#include "queue.hpp"
#include "sampler.hpp"

namespace ur::opencl {

/// The ur_exp_command_buffer_handle_t_ destructor calls CL release
/// command-buffer to free the underlying object.
ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() {
  ur::opencl::urQueueRelease(cast(hInternalQueue));
  if (LastSubmission) {
    clReleaseEvent(LastSubmission);
  }

  cl_context CLContext = hContext->CLContext;
  cl_ext::clReleaseCommandBufferKHR_fn clReleaseCommandBufferKHR = nullptr;
  cl_int Res =
      cl_ext::getExtFuncFromContext<decltype(clReleaseCommandBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clReleaseCommandBufferKHRCache,
          cl_ext::ReleaseCommandBufferName, &clReleaseCommandBufferKHR);
  assert(Res == CL_SUCCESS);
  (void)Res;

  clReleaseCommandBufferKHR(CLCommandBuffer);
}

ur_result_t
urCommandBufferCreateExp(ur_context_handle_t hContext,
                         ur_device_handle_t hDevice,
                         const ur_exp_command_buffer_desc_t *pCommandBufferDesc,
                         ur_exp_command_buffer_handle_t *phCommandBuffer) {

  auto Context = cast(hContext);
  cl_context CLContext = Context->CLContext;
  cl_ext::clCreateCommandBufferKHR_fn clCreateCommandBufferKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCreateCommandBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCreateCommandBufferKHRCache,
          cl_ext::CreateCommandBufferName, &clCreateCommandBufferKHR));

  const bool IsUpdatable = pCommandBufferDesc->isUpdatable;

  ur_device_command_buffer_update_capability_flags_t UpdateCapabilities;
  auto Device = cast(hDevice);
  cl_device_id CLDevice = Device->CLDevice;
  CL_RETURN_ON_FAILURE(
      getDeviceCommandBufferUpdateCapabilities(CLDevice, UpdateCapabilities));
  bool DeviceSupportsUpdate = UpdateCapabilities > 0;

  if (IsUpdatable && !DeviceSupportsUpdate) {
    return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
  }

  // OpenCL command-buffer must be simultaneous use to match expectation of UR
  // command-buffer specification.
  cl_command_buffer_flags_khr Flags = CL_COMMAND_BUFFER_SIMULTANEOUS_USE_KHR;
  if (IsUpdatable) {
    Flags |= CL_COMMAND_BUFFER_MUTABLE_KHR;
  }

  cl_command_buffer_properties_khr Properties[3] = {CL_COMMAND_BUFFER_FLAGS_KHR,
                                                    Flags, 0};

  ur_queue_handle_t Queue = nullptr;
  ur_queue_properties_t QueueProperties = {UR_STRUCTURE_TYPE_QUEUE_PROPERTIES,
                                           nullptr, 0};
  const bool IsInOrder = pCommandBufferDesc->isInOrder;
  if (!IsInOrder) {
    QueueProperties.flags = UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE;
  }
  UR_RETURN_ON_FAILURE(
      ur::opencl::urQueueCreate(hContext, hDevice, &QueueProperties, &Queue));

  cl_int Res = CL_SUCCESS;
  auto QueuePtr = cast(Queue);
  const cl_command_queue CLQueue = QueuePtr->CLQueue;
  auto CLCommandBuffer =
      clCreateCommandBufferKHR(1, &CLQueue, Properties, &Res);
  if (Res != CL_SUCCESS) {
    ur::opencl::urQueueRelease(Queue);
    CL_RETURN_ON_FAILURE_AND_SET_NULL(Res, phCommandBuffer);
  }

  try {
    auto URCommandBuffer = std::make_unique<ur_exp_command_buffer_handle_t_>(
        QueuePtr, cast(hContext), cast(hDevice), CLCommandBuffer, IsUpdatable,
        IsInOrder);
    *phCommandBuffer = cast(URCommandBuffer.release());
  } catch (std::bad_alloc &) {
    return UR_RESULT_ERROR_OUT_OF_RESOURCES;
  } catch (...) {
    return UR_RESULT_ERROR_UNKNOWN;
  }

  CL_RETURN_ON_FAILURE(Res);
  return UR_RESULT_SUCCESS;
}

ur_result_t
urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
  auto CommandBuffer = cast(hCommandBuffer);
  CommandBuffer->RefCount.retain();
  return UR_RESULT_SUCCESS;
}

ur_result_t
urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
  auto CommandBuffer = cast(hCommandBuffer);
  if (CommandBuffer->RefCount.release()) {
    delete CommandBuffer;
  }

  return UR_RESULT_SUCCESS;
}

ur_result_t
urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
  auto CommandBuffer = cast(hCommandBuffer);
  UR_ASSERT(!CommandBuffer->IsFinalized, UR_RESULT_ERROR_INVALID_OPERATION);
  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;
  cl_ext::clFinalizeCommandBufferKHR_fn clFinalizeCommandBufferKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clFinalizeCommandBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clFinalizeCommandBufferKHRCache,
          cl_ext::FinalizeCommandBufferName, &clFinalizeCommandBufferKHR));

  CL_RETURN_ON_FAILURE(
      clFinalizeCommandBufferKHR(CommandBuffer->CLCommandBuffer));
  CommandBuffer->IsFinalized = true;
  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel,
    uint32_t workDim, const size_t *pGlobalWorkOffset,
    const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize,
    uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs,
    uint32_t /*numKernelAlternatives*/,
    ur_kernel_handle_t * /*phKernelAlternatives*/,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent,
    ur_exp_command_buffer_command_handle_t *phCommandHandle) {
  (void)numEventsInWaitList;
  (void)phEventWaitList;
  (void)phEvent;

  auto CommandBuffer = cast(hCommandBuffer);
  auto Kernel = cast(hKernel);
  // Command handles can only be obtained from updatable command-buffers
  UR_ASSERT(!(phCommandHandle && !CommandBuffer->IsUpdatable),
            UR_RESULT_ERROR_INVALID_OPERATION);

  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;

  clSetKernelArgMemPointerINTEL_fn SetKernelArgMemPointerPtr = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<clSetKernelArgMemPointerINTEL_fn>(
          CLContext,
          cast(ur::cl::getAdapter())
              ->fnCache.clSetKernelArgMemPointerINTELCache,
          cl_ext::SetKernelArgMemPointerName, &SetKernelArgMemPointerPtr));

  for (uint32_t i = 0; i < numArgs; i++) {
    switch (pArgs[i].type) {
    case UR_EXP_KERNEL_ARG_TYPE_LOCAL:
      CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel,
                                          static_cast<cl_uint>(pArgs[i].index),
                                          pArgs[i].size, nullptr));
      break;
    case UR_EXP_KERNEL_ARG_TYPE_VALUE:
      CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel,
                                          static_cast<cl_uint>(pArgs[i].index),
                                          pArgs[i].size, pArgs[i].value.value));
      break;
    case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: {
      auto Mem = pArgs[i].value.memObjTuple.hMem
                     ? cast(pArgs[i].value.memObjTuple.hMem)
                     : nullptr;
      cl_mem mem = Mem ? Mem->CLMemory : nullptr;
      CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel,
                                          static_cast<cl_uint>(pArgs[i].index),
                                          pArgs[i].size, &mem));
      break;
    }
    case UR_EXP_KERNEL_ARG_TYPE_POINTER:
      CL_RETURN_ON_FAILURE(SetKernelArgMemPointerPtr(
          Kernel->CLKernel, static_cast<cl_uint>(pArgs[i].index),
          pArgs[i].value.pointer));
      break;
    case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: {
      auto Sampler = cast(pArgs[i].value.sampler);
      CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel,
                                          static_cast<cl_uint>(pArgs[i].index),
                                          pArgs[i].size, &Sampler->CLSampler));
      break;
    }
    default:
      return UR_RESULT_ERROR_INVALID_ENUMERATION;
    }
  }

  return ur::opencl::urCommandBufferAppendKernelLaunchExp(
      hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize,
      pLocalWorkSize, 0, nullptr, numSyncPointsInWaitList, pSyncPointWaitList,
      numEventsInWaitList, phEventWaitList, pSyncPoint, phEvent,
      phCommandHandle);
}

ur_result_t urCommandBufferAppendKernelLaunchExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel,
    uint32_t workDim, const size_t *pGlobalWorkOffset,
    const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize,
    uint32_t /*numKernelAlternatives*/,
    ur_kernel_handle_t * /*phKernelAlternatives*/,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent,
    ur_exp_command_buffer_command_handle_t *phCommandHandle) {
  (void)numEventsInWaitList;
  (void)phEventWaitList;
  (void)phEvent;

  auto CommandBuffer = cast(hCommandBuffer);
  auto Kernel = cast(hKernel);
  // Command handles can only be obtained from updatable command-buffers
  UR_ASSERT(!(phCommandHandle && !CommandBuffer->IsUpdatable),
            UR_RESULT_ERROR_INVALID_OPERATION);

  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;
  cl_ext::clCommandNDRangeKernelKHR_fn clCommandNDRangeKernelKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandNDRangeKernelKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandNDRangeKernelKHRCache,
          cl_ext::CommandNRRangeKernelName, &clCommandNDRangeKernelKHR));

  cl_mutable_command_khr CommandHandle = nullptr;
  cl_mutable_command_khr *OutCommandHandle =
      CommandBuffer->IsUpdatable ? &CommandHandle : nullptr;

  cl_command_properties_khr UpdateProperties[] = {
      CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR,
      CL_MUTABLE_DISPATCH_GLOBAL_OFFSET_KHR |
          CL_MUTABLE_DISPATCH_GLOBAL_SIZE_KHR |
          CL_MUTABLE_DISPATCH_LOCAL_SIZE_KHR |
          CL_MUTABLE_DISPATCH_ARGUMENTS_KHR | CL_MUTABLE_DISPATCH_EXEC_INFO_KHR,
      0};

  cl_command_properties_khr *Properties =
      CommandBuffer->IsUpdatable ? UpdateProperties : nullptr;

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandNDRangeKernelKHR(
      CommandBuffer->CLCommandBuffer, nullptr, Properties, Kernel->CLKernel,
      workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, WaitListSize,
      SyncPointWaitList, RetSyncPoint, OutCommandHandle));

  try {
    auto Handle = std::make_unique<ur_exp_command_buffer_command_handle_t_>(
        CommandBuffer, CommandHandle, Kernel, workDim,
        pLocalWorkSize != nullptr);
    if (phCommandHandle) {
      *phCommandHandle = cast(Handle.get());
    }

    CommandBuffer->CommandHandles.push_back(std::move(Handle));
  } catch (...) {
    return UR_RESULT_ERROR_OUT_OF_RESOURCES;
  }

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendUSMMemcpyExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, const void *pSrc,
    size_t size, uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t,
    const ur_event_handle_t *, ur_exp_command_buffer_sync_point_t *pSyncPoint,
    ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) {
  // No extension entry-point exists for USM memcpy, use SVM memcpy in
  // preparation for USVM.
  auto CommandBuffer = cast(hCommandBuffer);
  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;
  cl_ext::clCommandSVMMemcpyKHR_fn clCommandSVMMemcpyKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandSVMMemcpyKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandSVMMemcpyKHRCache,
          cl_ext::CommandSVMMemcpyName, &clCommandSVMMemcpyKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandSVMMemcpyKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, pDst, pSrc, size,
      WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendUSMFillExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory,
    const void *pPattern, size_t patternSize, size_t size,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t,
    const ur_event_handle_t *, ur_exp_command_buffer_sync_point_t *pSyncPoint,
    ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) {
  // No extension entry-point exists for USM fill, use SVM fill in preparation
  // for USVM.
  auto CommandBuffer = cast(hCommandBuffer);
  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;
  cl_ext::clCommandSVMMemFillKHR_fn clCommandSVMMemFillKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandSVMMemFillKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandSVMMemFillKHRCache,
          cl_ext::CommandSVMMemFillName, &clCommandSVMMemFillKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(
      clCommandSVMMemFillKHR(CommandBuffer->CLCommandBuffer, nullptr, nullptr,
                             pMemory, pPattern, patternSize, size, WaitListSize,
                             SyncPointWaitList, RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendMemBufferCopyExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem,
    ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent,
    ur_exp_command_buffer_command_handle_t *phCommand) {
  (void)numEventsInWaitList;
  (void)phEventWaitList;
  (void)phEvent;
  (void)phCommand;
  auto CommandBuffer = cast(hCommandBuffer);
  auto SrcMem = cast(hSrcMem);
  auto DstMem = cast(hDstMem);
  auto Context = CommandBuffer->hContext;
  cl_context CLContext = Context->CLContext;
  cl_ext::clCommandCopyBufferKHR_fn clCommandCopyBufferKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandCopyBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandCopyBufferKHRCache,
          cl_ext::CommandCopyBufferName, &clCommandCopyBufferKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandCopyBufferKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, SrcMem->CLMemory,
      DstMem->CLMemory, srcOffset, dstOffset, size, WaitListSize,
      SyncPointWaitList, RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendMemBufferCopyRectExp(
    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
    [[maybe_unused]] ur_mem_handle_t hSrcMem,
    [[maybe_unused]] ur_mem_handle_t hDstMem,
    [[maybe_unused]] ur_rect_offset_t srcOrigin,
    [[maybe_unused]] ur_rect_offset_t dstOrigin,
    [[maybe_unused]] ur_rect_region_t region,
    [[maybe_unused]] size_t srcRowPitch, [[maybe_unused]] size_t srcSlicePitch,
    [[maybe_unused]] size_t dstRowPitch, [[maybe_unused]] size_t dstSlicePitch,
    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
        *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {

  size_t OpenCLOriginRect[3]{srcOrigin.x, srcOrigin.y, srcOrigin.z};
  size_t OpenCLDstRect[3]{dstOrigin.x, dstOrigin.y, dstOrigin.z};
  size_t OpenCLRegion[3]{region.width, region.height, region.depth};

  auto CommandBuffer = cast(hCommandBuffer);
  auto SrcMem = cast(hSrcMem);
  auto DstMem = cast(hDstMem);
  cl_context CLContext = CommandBuffer->hContext->CLContext;
  cl_ext::clCommandCopyBufferRectKHR_fn clCommandCopyBufferRectKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandCopyBufferRectKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandCopyBufferRectKHRCache,
          cl_ext::CommandCopyBufferRectName, &clCommandCopyBufferRectKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandCopyBufferRectKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, SrcMem->CLMemory,
      DstMem->CLMemory, OpenCLOriginRect, OpenCLDstRect, OpenCLRegion,
      srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, WaitListSize,
      SyncPointWaitList, RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendMemBufferWriteExp(
    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
    [[maybe_unused]] ur_mem_handle_t hBuffer, [[maybe_unused]] size_t offset,
    [[maybe_unused]] size_t size, [[maybe_unused]] const void *pSrc,
    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
        *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferAppendMemBufferReadExp(
    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
    [[maybe_unused]] ur_mem_handle_t hBuffer, [[maybe_unused]] size_t offset,
    [[maybe_unused]] size_t size, [[maybe_unused]] void *pDst,
    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
        *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferAppendMemBufferWriteRectExp(
    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
    [[maybe_unused]] ur_mem_handle_t hBuffer,
    [[maybe_unused]] ur_rect_offset_t bufferOffset,
    [[maybe_unused]] ur_rect_offset_t hostOffset,
    [[maybe_unused]] ur_rect_region_t region,
    [[maybe_unused]] size_t bufferRowPitch,
    [[maybe_unused]] size_t bufferSlicePitch,
    [[maybe_unused]] size_t hostRowPitch,
    [[maybe_unused]] size_t hostSlicePitch, [[maybe_unused]] void *pSrc,
    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
        *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferAppendMemBufferReadRectExp(
    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
    [[maybe_unused]] ur_mem_handle_t hBuffer,
    [[maybe_unused]] ur_rect_offset_t bufferOffset,
    [[maybe_unused]] ur_rect_offset_t hostOffset,
    [[maybe_unused]] ur_rect_region_t region,
    [[maybe_unused]] size_t bufferRowPitch,
    [[maybe_unused]] size_t bufferSlicePitch,
    [[maybe_unused]] size_t hostRowPitch,
    [[maybe_unused]] size_t hostSlicePitch, [[maybe_unused]] void *pDst,
    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
        *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferAppendMemBufferFillExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer,
    const void *pPattern, size_t patternSize, size_t offset, size_t size,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    [[maybe_unused]] uint32_t numEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint,
    [[maybe_unused]] ur_event_handle_t *phEvent,
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) {

  auto CommandBuffer = cast(hCommandBuffer);
  auto Buffer = cast(hBuffer);
  cl_context CLContext = CommandBuffer->hContext->CLContext;
  cl_ext::clCommandFillBufferKHR_fn clCommandFillBufferKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandFillBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clCommandFillBufferKHRCache,
          cl_ext::CommandFillBufferName, &clCommandFillBufferKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandFillBufferKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, Buffer->CLMemory,
      pPattern, patternSize, offset, size, WaitListSize, SyncPointWaitList,
      RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferAppendUSMPrefetchExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, const void *mem, size_t size,
    ur_usm_migration_flags_t flags, uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent,
    ur_exp_command_buffer_command_handle_t *phCommand) {
  (void)hCommandBuffer;
  (void)mem;
  (void)size;
  (void)flags;
  (void)numSyncPointsInWaitList;
  (void)pSyncPointWaitList;
  (void)numEventsInWaitList;
  (void)phEventWaitList;
  (void)pSyncPoint;
  (void)phEvent;
  (void)phCommand;

  // Not implemented
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferAppendUSMAdviseExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, const void *mem, size_t size,
    ur_usm_advice_flags_t advice, uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent,
    ur_exp_command_buffer_command_handle_t *phCommand) {
  (void)hCommandBuffer;
  (void)mem;
  (void)size;
  (void)advice;
  (void)numSyncPointsInWaitList;
  (void)pSyncPointWaitList;
  (void)numEventsInWaitList;
  (void)phEventWaitList;
  (void)pSyncPoint;
  (void)phEvent;
  (void)phCommand;

  // Not implemented
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urEnqueueCommandBufferExp(
    ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer,
    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
    ur_event_handle_t *phEvent) {

  auto CommandBuffer = cast(hCommandBuffer);
  auto Queue = cast(hQueue);
  cl_context CLContext = CommandBuffer->hContext->CLContext;
  cl_ext::clEnqueueCommandBufferKHR_fn clEnqueueCommandBufferKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clEnqueueCommandBufferKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clEnqueueCommandBufferKHRCache,
          cl_ext::EnqueueCommandBufferName, &clEnqueueCommandBufferKHR));

  // If we've submitted the command-buffer before, then add a dependency on the
  // last submission.
  bool AddExtraDep = CommandBuffer->LastSubmission != nullptr;
  uint32_t CLWaitListLength =
      AddExtraDep ? numEventsInWaitList + 1 : numEventsInWaitList;
  std::vector<cl_event> CLWaitEvents(CLWaitListLength);
  for (uint32_t i = 0; i < numEventsInWaitList; i++) {
    CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent;
  }
  if (AddExtraDep) {
    CLWaitEvents[numEventsInWaitList] = CommandBuffer->LastSubmission;
  }

  // Always get an event as we need it to serialize any future submissions
  // of the command-buffer with.
  cl_event Event;
  cl_command_queue CLQueue = Queue->CLQueue;
  CL_RETURN_ON_FAILURE(
      clEnqueueCommandBufferKHR(1, &CLQueue, CommandBuffer->CLCommandBuffer,
                                CLWaitListLength, CLWaitEvents.data(), &Event));

  // Retain event so that if a user manually destroys the returned UR
  // event the adapter still has a valid handle.
  clRetainEvent(Event);
  if (CommandBuffer->LastSubmission) {
    clReleaseEvent(CommandBuffer->LastSubmission);
  }
  CommandBuffer->LastSubmission = Event;

  UR_RETURN_ON_FAILURE(
      createUREvent(Event, cast(Queue->Context), hQueue, phEvent));
  return UR_RESULT_SUCCESS;
}

namespace {
void updateKernelPointerArgs(
    std::vector<cl_mutable_dispatch_arg_khr> &CLUSMArgs,
    const ur_exp_command_buffer_update_kernel_launch_desc_t
        &pUpdateKernelLaunch) {

  // WARNING - This relies on USM and SVM using the same implementation,
  // which is not guaranteed.
  // See https://github.com/KhronosGroup/OpenCL-Docs/issues/843
  const uint32_t NumPointerArgs = pUpdateKernelLaunch.numNewPointerArgs;
  const ur_exp_command_buffer_update_pointer_arg_desc_t *ArgPointerList =
      pUpdateKernelLaunch.pNewPointerArgList;

  CLUSMArgs.resize(NumPointerArgs);
  for (uint32_t i = 0; i < NumPointerArgs; i++) {
    const ur_exp_command_buffer_update_pointer_arg_desc_t &URPointerArg =
        ArgPointerList[i];
    cl_mutable_dispatch_arg_khr &USMArg = CLUSMArgs[i];
    USMArg.arg_index = URPointerArg.argIndex;
    USMArg.arg_value = *(void *const *)URPointerArg.pNewPointerArg;
  }
}

void updateKernelArgs(std::vector<cl_mutable_dispatch_arg_khr> &CLArgs,
                      const ur_exp_command_buffer_update_kernel_launch_desc_t
                          &pUpdateKernelLaunch) {
  const uint32_t NumMemobjArgs = pUpdateKernelLaunch.numNewMemObjArgs;
  const ur_exp_command_buffer_update_memobj_arg_desc_t *ArgMemobjList =
      pUpdateKernelLaunch.pNewMemObjArgList;
  const uint32_t NumValueArgs = pUpdateKernelLaunch.numNewValueArgs;
  const ur_exp_command_buffer_update_value_arg_desc_t *ArgValueList =
      pUpdateKernelLaunch.pNewValueArgList;

  for (uint32_t i = 0; i < NumMemobjArgs; i++) {
    const ur_exp_command_buffer_update_memobj_arg_desc_t &URMemObjArg =
        ArgMemobjList[i];
    cl_mem arg_value = cast(URMemObjArg.hNewMemObjArg)->CLMemory;
    cl_mutable_dispatch_arg_khr CLArg{
        URMemObjArg.argIndex, // arg_index
        sizeof(cl_mem),       // arg_size
        &arg_value            // arg_value
    };

    CLArgs.push_back(CLArg);
  }

  for (uint32_t i = 0; i < NumValueArgs; i++) {
    const ur_exp_command_buffer_update_value_arg_desc_t &URValueArg =
        ArgValueList[i];
    cl_mutable_dispatch_arg_khr CLArg{
        URValueArg.argIndex,    // arg_index
        URValueArg.argSize,     // arg_size
        URValueArg.pNewValueArg // arg_value
    };
    CLArgs.push_back(CLArg);
  }
}

ur_result_t validateCommandDesc(
    ur_exp_command_buffer_handle_t hCommandBuffer,
    const ur_exp_command_buffer_update_kernel_launch_desc_t &UpdateDesc) {
  auto CommandBuffer = cast(hCommandBuffer);
  if (!CommandBuffer->IsFinalized || !CommandBuffer->IsUpdatable) {
    return UR_RESULT_ERROR_INVALID_OPERATION;
  }

  auto Command = cast(UpdateDesc.hCommand);
  if (hCommandBuffer != cast(Command->hCommandBuffer)) {
    return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP;
  }

  // Kernel handle updates are not yet supported.
  if (UpdateDesc.hNewKernel && UpdateDesc.hNewKernel != cast(Command->Kernel)) {
    return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
  }

  // Error if work-dim has changed but a new global size/offset hasn't been set
  if (UpdateDesc.newWorkDim != Command->WorkDim &&
      (!UpdateDesc.pNewGlobalWorkOffset || !UpdateDesc.pNewGlobalWorkSize)) {
    return UR_RESULT_ERROR_INVALID_VALUE;
  }

  // Verify that the device supports updating the aspects of the kernel that
  // the user is requesting.
  auto URDevice = CommandBuffer->hDevice;
  cl_device_id CLDevice = URDevice->CLDevice;

  ur_device_command_buffer_update_capability_flags_t UpdateCapabilities = 0;
  CL_RETURN_ON_FAILURE(
      getDeviceCommandBufferUpdateCapabilities(CLDevice, UpdateCapabilities));

  size_t *NewGlobalWorkOffset = UpdateDesc.pNewGlobalWorkOffset;
  UR_ASSERT(
      !NewGlobalWorkOffset ||
          (UpdateCapabilities &
           UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET),
      UR_RESULT_ERROR_UNSUPPORTED_FEATURE);

  size_t *NewLocalWorkSize = UpdateDesc.pNewLocalWorkSize;
  UR_ASSERT(
      !NewLocalWorkSize ||
          (UpdateCapabilities &
           UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE),
      UR_RESULT_ERROR_UNSUPPORTED_FEATURE);

  size_t *NewGlobalWorkSize = UpdateDesc.pNewGlobalWorkSize;
  UR_ASSERT(
      !NewGlobalWorkSize ||
          (UpdateCapabilities &
           UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE),
      UR_RESULT_ERROR_UNSUPPORTED_FEATURE);
  UR_ASSERT(
      !(NewGlobalWorkSize && !NewLocalWorkSize) ||
          (UpdateCapabilities &
           UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE),
      UR_RESULT_ERROR_UNSUPPORTED_FEATURE);

  UR_ASSERT(
      (!UpdateDesc.numNewMemObjArgs && !UpdateDesc.numNewPointerArgs &&
       !UpdateDesc.numNewValueArgs) ||
          (UpdateCapabilities &
           UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS),
      UR_RESULT_ERROR_UNSUPPORTED_FEATURE);

  return UR_RESULT_SUCCESS;
}
} // end anonymous namespace

ur_result_t urCommandBufferUpdateKernelLaunchExp(
    ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates,
    const ur_exp_command_buffer_update_kernel_launch_desc_t
        *pUpdateKernelLaunch) {
  for (uint32_t i = 0; i < numKernelUpdates; i++) {
    UR_RETURN_ON_FAILURE(
        validateCommandDesc(hCommandBuffer, pUpdateKernelLaunch[i]));
  }

  auto CommandBuffer = cast(hCommandBuffer);
  cl_context CLContext = CommandBuffer->hContext->CLContext;

  cl_ext::clUpdateMutableCommandsKHR_fn clUpdateMutableCommandsKHR = nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clUpdateMutableCommandsKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())->fnCache.clUpdateMutableCommandsKHRCache,
          cl_ext::UpdateMutableCommandsName, &clUpdateMutableCommandsKHR));

  std::vector<cl_mutable_dispatch_config_khr> ConfigList(numKernelUpdates);
  std::vector<std::vector<cl_mutable_dispatch_arg_khr>> CLUSMArgsList(
      numKernelUpdates);
  std::vector<std::vector<cl_mutable_dispatch_arg_khr>> CLArgsList(
      numKernelUpdates);

  std::vector<std::vector<size_t>> CLGlobalWorkOffsetList(numKernelUpdates);
  std::vector<std::vector<size_t>> CLGlobalWorkSizeList(numKernelUpdates);
  std::vector<std::vector<size_t>> CLLocalWorkSizeList(numKernelUpdates);

  // Lambda for N-Dimensional update
  auto updateNDRange = [](std::vector<size_t> &NDRange, cl_uint WorkDim,
                          size_t *UpdatePtr) {
    NDRange.resize(WorkDim, 0);
    const size_t CopySize = sizeof(size_t) * WorkDim;
    std::memcpy(NDRange.data(), UpdatePtr, CopySize);
  };

  for (uint32_t i = 0; i < numKernelUpdates; i++) {
    cl_mutable_dispatch_config_khr &Config = ConfigList[i];
    std::vector<cl_mutable_dispatch_arg_khr> &CLUSMArgs = CLUSMArgsList[i];
    std::vector<cl_mutable_dispatch_arg_khr> &CLArgs = CLArgsList[i];
    std::vector<size_t> &CLGlobalWorkOffset = CLGlobalWorkOffsetList[i];
    std::vector<size_t> &CLGlobalWorkSize = CLGlobalWorkSizeList[i];
    std::vector<size_t> &CLLocalWorkSize = CLLocalWorkSizeList[i];

    const auto &UpdateDesc = pUpdateKernelLaunch[i];
    // Find the CL USM pointer arguments to the kernel to update
    updateKernelPointerArgs(CLUSMArgs, UpdateDesc);

    // Find the memory object and scalar arguments to the kernel to update
    updateKernelArgs(CLArgs, UpdateDesc);

    // Find the updated ND-Range configuration of the kernel.
    auto Command = cast(UpdateDesc.hCommand);
    cl_uint &CommandWorkDim = Command->WorkDim;

    if (auto GlobalWorkOffsetPtr = UpdateDesc.pNewGlobalWorkOffset) {
      updateNDRange(CLGlobalWorkOffset, CommandWorkDim, GlobalWorkOffsetPtr);
    }

    if (auto GlobalWorkSizePtr = UpdateDesc.pNewGlobalWorkSize) {
      updateNDRange(CLGlobalWorkSize, CommandWorkDim, GlobalWorkSizePtr);
    }

    if (auto LocalWorkSizePtr = UpdateDesc.pNewLocalWorkSize) {
      updateNDRange(CLLocalWorkSize, CommandWorkDim, LocalWorkSizePtr);
    }

    cl_mutable_command_khr CLCommand = Command->CLMutableCommand;
    Config = cl_mutable_dispatch_config_khr{
        CLCommand,
        static_cast<cl_uint>(CLArgs.size()),    // num_args
        static_cast<cl_uint>(CLUSMArgs.size()), // num_svm_args
        0,                                      // num_exec_infos
        CommandWorkDim,                         // work_dim
        CLArgs.data(),                          // arg_list
        CLUSMArgs.data(),                       // arg_svm_list
        nullptr,                                // exec_info_list
        CLGlobalWorkOffset.data(),              // global_work_offset
        CLGlobalWorkSize.data(),                // global_work_size
        CLLocalWorkSize.data(),                 // local_work_size
    };
  }

  cl_uint NumConfigs = ConfigList.size();
  std::vector<cl_command_buffer_update_type_khr> ConfigTypes(
      NumConfigs, CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR);
  std::vector<const void *> ConfigPtrs(NumConfigs);
  for (cl_uint i = 0; i < NumConfigs; i++) {
    ConfigPtrs[i] = &ConfigList[i];
  }
  CL_RETURN_ON_FAILURE(clUpdateMutableCommandsKHR(
      CommandBuffer->CLCommandBuffer, NumConfigs, ConfigTypes.data(),
      (const void **)ConfigPtrs.data()));

  return UR_RESULT_SUCCESS;
}

ur_result_t urCommandBufferUpdateSignalEventExp(
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t Command,
    [[maybe_unused]] ur_event_handle_t *Event) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t urCommandBufferUpdateWaitEventsExp(
    [[maybe_unused]] ur_exp_command_buffer_command_handle_t Command,
    [[maybe_unused]] uint32_t NumEventsInWaitList,
    [[maybe_unused]] const ur_event_handle_t *EventWaitList) {
  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

ur_result_t
urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer,
                          ur_exp_command_buffer_info_t propName,
                          size_t propSize, void *pPropValue,
                          size_t *pPropSizeRet) {

  UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet);
  auto CommandBuffer = cast(hCommandBuffer);

  switch (propName) {
  case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT:
    return ReturnValue(CommandBuffer->RefCount.getCount());
  case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: {
    ur_exp_command_buffer_desc_t Descriptor{};
    Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC;
    Descriptor.pNext = nullptr;
    Descriptor.isUpdatable = CommandBuffer->IsUpdatable;
    Descriptor.isInOrder = false;
    Descriptor.enableProfiling = false;

    return ReturnValue(Descriptor);
  }
  default:
    assert(false && "Command-buffer info request not implemented");
  }

  return UR_RESULT_ERROR_INVALID_ENUMERATION;
}

ur_result_t urCommandBufferAppendNativeCommandExp(
    ur_exp_command_buffer_handle_t hCommandBuffer,
    ur_exp_command_buffer_native_command_function_t pfnNativeCommand,
    void *pData, ur_exp_command_buffer_handle_t,
    uint32_t numSyncPointsInWaitList,
    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
    ur_exp_command_buffer_sync_point_t *pSyncPoint) {
  auto CommandBuffer = cast(hCommandBuffer);
  cl_context CLContext = CommandBuffer->hContext->CLContext;
  cl_ext::clCommandBarrierWithWaitListKHR_fn clCommandBarrierWithWaitListKHR =
      nullptr;
  UR_RETURN_ON_FAILURE(
      cl_ext::getExtFuncFromContext<decltype(clCommandBarrierWithWaitListKHR)>(
          CLContext,
          cast(ur::cl::getAdapter())
              ->fnCache.clCommandBarrierWithWaitListKHRCache,
          cl_ext::CommandBarrierWithWaitListName,
          &clCommandBarrierWithWaitListKHR));

  const bool IsInOrder = CommandBuffer->IsInOrder;
  cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint;
  const cl_sync_point_khr *SyncPointWaitList =
      IsInOrder ? nullptr : pSyncPointWaitList;
  uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList;
  CL_RETURN_ON_FAILURE(clCommandBarrierWithWaitListKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, WaitListSize,
      SyncPointWaitList, nullptr, nullptr));

  // Call user-defined function immediately
  pfnNativeCommand(pData);

  // Barrier on all commands after user defined commands.
  CL_RETURN_ON_FAILURE(clCommandBarrierWithWaitListKHR(
      CommandBuffer->CLCommandBuffer, nullptr, nullptr, 0, nullptr,
      RetSyncPoint, nullptr));

  return UR_RESULT_SUCCESS;
}

ur_result_t
urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer,
                                  ur_native_handle_t *phNativeCommandBuffer) {
  auto CommandBuffer = cast(hCommandBuffer);
  *phNativeCommandBuffer =
      reinterpret_cast<ur_native_handle_t>(CommandBuffer->CLCommandBuffer);
  return UR_RESULT_SUCCESS;
}

} // namespace ur::opencl