Merge branch 'adapters' into review/yang/urAdapterGet

source/adapters/level_zero/adapter.cpp

@@ -38,7 +38,7 @@ ur_result_t adapterStateTeardown() {
   // Print the balance of various create/destroy native calls.
   // The idea is to verify if the number of create(+) and destroy(-) calls are
   // matched.
-  if (ZeCallCount && (UrL0Debug & UR_L0_DEBUG_CALL_COUNT) != 0) {
+  if (ZeCallCount && (UrL0LeaksDebug) != 0) {
     // clang-format off
     //
     // The format of this table is such that each row accounts for a
@@ -79,8 +79,7 @@ ur_result_t adapterStateTeardown() {
     //
     // clang-format on
 
-    fprintf(stderr, "ZE_DEBUG=%d: check balance of create/destroy calls\n",
-            UR_L0_DEBUG_CALL_COUNT);
+    fprintf(stderr, "Check balance of create/destroy calls\n");
     fprintf(stderr,
             "----------------------------------------------------------\n");
     for (const auto &Row : CreateDestroySet) {

source/adapters/level_zero/common.cpp

@@ -149,7 +149,7 @@ ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName,
                            const char *ZeArgs, bool TraceError) {
   urPrint("ZE ---> %s%s\n", ZeName, ZeArgs);
 
-  if (UrL0Debug & UR_L0_DEBUG_CALL_COUNT) {
+  if (UrL0LeaksDebug) {
     ++(*ZeCallCount)[ZeName];
   }
 

source/adapters/level_zero/common.hpp

@@ -187,7 +187,6 @@ enum UrDebugLevel {
   UR_L0_DEBUG_NONE = 0x0,
   UR_L0_DEBUG_BASIC = 0x1,
   UR_L0_DEBUG_VALIDATION = 0x2,
-  UR_L0_DEBUG_CALL_COUNT = 0x4,
   UR_L0_DEBUG_ALL = -1
 };
 
@@ -203,6 +202,13 @@ const int UrL0Debug = [] {
   return DebugMode;
 }();
 
+const int UrL0LeaksDebug = [] {
+  const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG");
+  if (!UrRet)
+    return 0;
+  return std::atoi(UrRet);
+}();
+
 // Controls Level Zero calls serialization to w/a Level Zero driver being not MT
 // ready. Recognized values (can be used as a bit mask):
 enum {

source/adapters/level_zero/platform.cpp

@@ -30,7 +30,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformGet(
   static std::once_flag ZeCallCountInitialized;
   try {
     std::call_once(ZeCallCountInitialized, []() {
-      if (UrL0Debug & UR_L0_DEBUG_CALL_COUNT) {
+      if (UrL0LeaksDebug) {
         ZeCallCount = new std::map<std::string, int>;
       }
     });

source/adapters/opencl/command_buffer.hpp

@@ -8,8 +8,17 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <CL/cl_ext.h>
 #include <ur/ur.hpp>
 
-/// Stub implementation of command-buffers for OpenCL
+struct ur_exp_command_buffer_handle_t_ {
+  ur_queue_handle_t hInternalQueue;
+  ur_context_handle_t hContext;
+  cl_command_buffer_khr CLCommandBuffer;
 
-struct ur_exp_command_buffer_handle_t_ {};
+  ur_exp_command_buffer_handle_t_(ur_queue_handle_t hQueue,
+                                  ur_context_handle_t hContext,
+                                  cl_command_buffer_khr CLCommandBuffer)
+      : hInternalQueue(hQueue), hContext(hContext),
+        CLCommandBuffer(CLCommandBuffer) {}
+};

source/adapters/opencl/common.cpp

@@ -77,6 +77,10 @@ ur_result_t mapCLErrorToUR(cl_int Result) {
     return UR_RESULT_ERROR_PROGRAM_LINK_FAILURE;
   case CL_INVALID_ARG_INDEX:
     return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX;
+  case CL_INVALID_COMMAND_BUFFER_KHR:
+    return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP;
+  case CL_INVALID_SYNC_POINT_WAIT_LIST_KHR:
+    return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP;
   default:
     return UR_RESULT_ERROR_UNKNOWN;
   }

source/adapters/opencl/common.hpp

@@ -192,6 +192,16 @@ CONSTFIX char EnqueueReadGlobalVariableName[] =
 // Names of host pipe functions queried from OpenCL
 CONSTFIX char EnqueueReadHostPipeName[] = "clEnqueueReadHostPipeINTEL";
 CONSTFIX char EnqueueWriteHostPipeName[] = "clEnqueueWriteHostPipeINTEL";
+// Names of command buffer functions queried from OpenCL
+CONSTFIX char CreateCommandBufferName[] = "clCreateCommandBufferKHR";
+CONSTFIX char RetainCommandBufferName[] = "clRetainCommandBufferKHR";
+CONSTFIX char ReleaseCommandBufferName[] = "clReleaseCommandBufferKHR";
+CONSTFIX char FinalizeCommandBufferName[] = "clFinalizeCommandBufferKHR";
+CONSTFIX char CommandNRRangeKernelName[] = "clCommandNDRangeKernelKHR";
+CONSTFIX char CommandCopyBufferName[] = "clCommandCopyBufferKHR";
+CONSTFIX char CommandCopyBufferRectName[] = "clCommandCopyBufferRectKHR";
+CONSTFIX char CommandFillBufferName[] = "clCommandFillBufferKHR";
+CONSTFIX char EnqueueCommandBufferName[] = "clEnqueueCommandBufferKHR";
 
 #undef CONSTFIX
 
@@ -226,6 +236,58 @@ cl_int(CL_API_CALL *)(cl_command_queue queue, cl_program program,
                       cl_uint num_events_in_waitlist,
                       const cl_event *events_waitlist, cl_event *event);
 
+using clCreateCommandBufferKHR_fn = CL_API_ENTRY cl_command_buffer_khr(
+    CL_API_CALL *)(cl_uint num_queues, const cl_command_queue *queues,
+                   const cl_command_buffer_properties_khr *properties,
+                   cl_int *errcode_ret);
+
+using clRetainCommandBufferKHR_fn = CL_API_ENTRY
+cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer);
+
+using clReleaseCommandBufferKHR_fn = CL_API_ENTRY
+cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer);
+
+using clFinalizeCommandBufferKHR_fn = CL_API_ENTRY
+cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer);
+
+using clCommandNDRangeKernelKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    const cl_ndrange_kernel_command_properties_khr *properties,
+    cl_kernel kernel, cl_uint work_dim, const size_t *global_work_offset,
+    const size_t *global_work_size, const size_t *local_work_size,
+    cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
+using clCommandCopyBufferKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    cl_mem src_buffer, cl_mem dst_buffer, size_t src_offset, size_t dst_offset,
+    size_t size, cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
+using clCommandCopyBufferRectKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    cl_mem src_buffer, cl_mem dst_buffer, const size_t *src_origin,
+    const size_t *dst_origin, const size_t *region, size_t src_row_pitch,
+    size_t src_slice_pitch, size_t dst_row_pitch, size_t dst_slice_pitch,
+    cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
+using clCommandFillBufferKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)(
+    cl_command_buffer_khr command_buffer, cl_command_queue command_queue,
+    cl_mem buffer, const void *pattern, size_t pattern_size, size_t offset,
+    size_t size, cl_uint num_sync_points_in_wait_list,
+    const cl_sync_point_khr *sync_point_wait_list,
+    cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle);
+
+using clEnqueueCommandBufferKHR_fn = CL_API_ENTRY
+cl_int(CL_API_CALL *)(cl_uint num_queues, cl_command_queue *queues,
+                      cl_command_buffer_khr command_buffer,
+                      cl_uint num_events_in_wait_list,
+                      const cl_event *event_wait_list, cl_event *event);
+
 template <typename T> struct FuncPtrCache {
   std::map<cl_context, T> Map;
   std::mutex Mutex;
@@ -255,6 +317,15 @@ struct ExtFuncPtrCacheT {
   FuncPtrCache<clEnqueueWriteHostPipeINTEL_fn> clEnqueueWriteHostPipeINTELCache;
   FuncPtrCache<clSetProgramSpecializationConstant_fn>
       clSetProgramSpecializationConstantCache;
+  FuncPtrCache<clCreateCommandBufferKHR_fn> clCreateCommandBufferKHRCache;
+  FuncPtrCache<clRetainCommandBufferKHR_fn> clRetainCommandBufferKHRCache;
+  FuncPtrCache<clReleaseCommandBufferKHR_fn> clReleaseCommandBufferKHRCache;
+  FuncPtrCache<clFinalizeCommandBufferKHR_fn> clFinalizeCommandBufferKHRCache;
+  FuncPtrCache<clCommandNDRangeKernelKHR_fn> clCommandNDRangeKernelKHRCache;
+  FuncPtrCache<clCommandCopyBufferKHR_fn> clCommandCopyBufferKHRCache;
+  FuncPtrCache<clCommandCopyBufferRectKHR_fn> clCommandCopyBufferRectKHRCache;
+  FuncPtrCache<clCommandFillBufferKHR_fn> clCommandFillBufferKHRCache;
+  FuncPtrCache<clEnqueueCommandBufferKHR_fn> clEnqueueCommandBufferKHRCache;
 };
 // A raw pointer is used here since the lifetime of this map has to be tied to
 // piTeardown to avoid issues with static destruction order (a user application

source/adapters/opencl/device.cpp

@@ -886,7 +886,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
   case UR_DEVICE_INFO_PROFILE:
   case UR_DEVICE_INFO_VERSION:
   case UR_EXT_DEVICE_INFO_OPENCL_C_VERSION:
-  case UR_DEVICE_INFO_EXTENSIONS:
   case UR_DEVICE_INFO_BUILT_IN_KERNELS:
   case UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES:
   case UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL:
@@ -908,6 +907,22 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
 
     return UR_RESULT_SUCCESS;
   }
+  case UR_DEVICE_INFO_EXTENSIONS: {
+    cl_device_id Dev = cl_adapter::cast<cl_device_id>(hDevice);
+    size_t ExtSize = 0;
+    CL_RETURN_ON_FAILURE(
+        clGetDeviceInfo(Dev, CL_DEVICE_EXTENSIONS, 0, nullptr, &ExtSize));
+
+    std::string ExtStr(ExtSize, '\0');
+    CL_RETURN_ON_FAILURE(clGetDeviceInfo(Dev, CL_DEVICE_EXTENSIONS, ExtSize,
+                                         ExtStr.data(), nullptr));
+
+    std::string SupportedExtensions(ExtStr.c_str());
+    if (ExtStr.find("cl_khr_command_buffer") != std::string::npos) {
+      SupportedExtensions += " ur_exp_command_buffer";
+    }
+    return ReturnValue(SupportedExtensions.c_str());
+  }
   /* TODO: Check regularly to see if support is enabled in OpenCL. Intel GPU
    * EU device-specific information extensions. Some of the queries are
    * enabled by cl_intel_device_attribute_query extension, but it's not yet in