[SYCL][UR][L0 v2] Implement ooo queue using ooo command-list

Author: igchor

unified-runtime/source/adapters/level_zero/v2/queue_create.cpp

@@ -45,7 +45,8 @@ static ze_command_queue_priority_t getZePriority(ur_queue_flags_t flags) {
 }
 
 static event_flags_t eventFlagsFromQueueFlags(ur_queue_flags_t flags) {
-  event_flags_t eventFlags = EVENT_FLAGS_COUNTER;
+  bool ooo = flags & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE;
+  event_flags_t eventFlags = ooo ? 0 : EVENT_FLAGS_COUNTER;
   if (flags & UR_QUEUE_FLAG_PROFILING_ENABLE)
     eventFlags |= EVENT_FLAGS_PROFILING_ENABLED;
   return eventFlags;
@@ -90,8 +91,6 @@ ur_result_t urQueueCreateWithNativeHandle(
     ur_native_handle_t hNativeQueue, ur_context_handle_t hContext,
     ur_device_handle_t hDevice, const ur_queue_native_properties_t *pProperties,
     ur_queue_handle_t *phQueue) try {
-  // TODO: For now, always assume it's immediate, in-order
-
   bool ownNativeHandle = pProperties ? pProperties->isNativeHandleOwned : false;
   ur_queue_flags_t flags = 0;
 
@@ -119,9 +118,12 @@ ur_result_t urQueueCreateWithNativeHandle(
         }
       });
 
-  *phQueue = ur_queue_handle_t_::create<v2::ur_queue_immediate_in_order_t>(
-      hContext, hDevice, std::move(commandListHandle),
-      v2::eventFlagsFromQueueFlags(flags), flags);
+  v2::event_flags_t eventFlags = 0;
+  if (flags & UR_QUEUE_FLAG_PROFILING_ENABLE)
+    eventFlags = v2::EVENT_FLAGS_PROFILING_ENABLED;
+
+  *phQueue = ur_queue_handle_t_::create<v2::ur_queue_immediate_out_of_order_t>(
+      hContext, hDevice, std::move(commandListHandle), eventFlags, flags);
 
   return UR_RESULT_SUCCESS;
 } catch (...) {

unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp

@@ -14,34 +14,30 @@
 
 namespace v2 {
 
-template <size_t N>
-std::array<ur_command_list_manager, N> createCommandListManagers(
-    ur_context_handle_t hContext, ur_device_handle_t hDevice, uint32_t ordinal,
-    ze_command_queue_priority_t priority, std::optional<int32_t> index) {
-  return createArrayOf<ur_command_list_manager, N>([&](size_t) {
-    return ur_command_list_manager(
-        hContext, hDevice,
-        hContext->getCommandListCache().getImmediateCommandList(
-            hDevice->ZeDevice,
-            {true, ordinal, true /* always enable copy offload */},
-            ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS, priority, index));
-  });
-}
-
 ur_queue_immediate_out_of_order_t::ur_queue_immediate_out_of_order_t(
     ur_context_handle_t hContext, ur_device_handle_t hDevice, uint32_t ordinal,
     ze_command_queue_priority_t priority, std::optional<int32_t> index,
     event_flags_t eventFlags, ur_queue_flags_t flags)
     : hContext(hContext), hDevice(hDevice),
       eventPool(hContext->getEventPoolCache(PoolCacheType::Immediate)
                     .borrow(hDevice->Id.value(), eventFlags)),
-      commandListManagers(createCommandListManagers<numCommandLists>(
-          hContext, hDevice, ordinal, priority, index)),
-      flags(flags) {
-  for (size_t i = 0; i < numCommandLists; i++) {
-    barrierEvents[i] = eventPool->allocate();
-  }
-}
+      commandListManager(
+          hContext, hDevice,
+          hContext->getCommandListCache().getImmediateCommandList(
+              hDevice->ZeDevice,
+              {false, ordinal, true /* always enable copy offload */},
+              ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS, priority, index)),
+      flags(flags) {}
+
+ur_queue_immediate_out_of_order_t::ur_queue_immediate_out_of_order_t(
+    ur_context_handle_t hContext, ur_device_handle_t hDevice,
+    raii::command_list_unique_handle commandListHandle,
+    event_flags_t eventFlags, ur_queue_flags_t flags)
+    : hContext(hContext), hDevice(hDevice),
+      eventPool(hContext->getEventPoolCache(PoolCacheType::Immediate)
+                    .borrow(hDevice->Id.value(), eventFlags)),
+      commandListManager(hContext, hDevice, std::move(commandListHandle)),
+      flags(flags) {}
 
 ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo(
     ur_queue_info_t propName, size_t propSize, void *pPropValue,
@@ -72,13 +68,7 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo(
       }
     };
 
-    auto commandListManagersLocked = commandListManagers.lock();
-
-    bool empty = std::all_of(
-        commandListManagersLocked->begin(), commandListManagersLocked->end(),
-        [&](auto &cmdListManager) {
-          return isCmdListEmpty(cmdListManager.getZeCommandList());
-        });
+    bool empty = isCmdListEmpty(commandListManager.lock()->getZeCommandList());
 
     return ReturnValue(empty);
   }
@@ -96,8 +86,7 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo(
 ur_result_t ur_queue_immediate_out_of_order_t::queueGetNativeHandle(
     ur_queue_native_desc_t *pDesc, ur_native_handle_t *phNativeQueue) {
   *phNativeQueue = reinterpret_cast<ur_native_handle_t>(
-      (*commandListManagers.get_no_lock())[getNextCommandListId()]
-          .getZeCommandList());
+      commandListManager.get_no_lock()->getZeCommandList());
   if (pDesc && pDesc->pNativeData) {
     // pNativeData == isImmediateQueue
     *(reinterpret_cast<int32_t *>(pDesc->pNativeData)) = 1;
@@ -108,13 +97,15 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetNativeHandle(
 ur_result_t ur_queue_immediate_out_of_order_t::queueFinish() {
   TRACK_SCOPE_LATENCY("ur_queue_immediate_out_of_order_t::queueFinish");
 
-  auto commandListManagersLocked = commandListManagers.lock();
+  auto commandListManagerLocked = commandListManager.lock();
+
+  ZE2UR_CALL(zeCommandListHostSynchronize,
+             (commandListManagerLocked->getZeCommandList(), UINT64_MAX));
 
-  for (size_t i = 0; i < numCommandLists; i++) {
-    ZE2UR_CALL(zeCommandListHostSynchronize,
-               (commandListManagersLocked[i].getZeCommandList(), UINT64_MAX));
-    UR_CALL(commandListManagersLocked[i].releaseSubmittedKernels());
+  for (auto &event : pendingEvents) {
+    event->release();
   }
+  pendingEvents.clear();
 
   hContext->getAsyncPool()->cleanupPoolsForQueue(this);
   hContext->forEachUsmPool([this](ur_usm_pool_handle_t hPool) {
@@ -132,10 +123,6 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueFlush() {
 ur_queue_immediate_out_of_order_t::~ur_queue_immediate_out_of_order_t() {
   try {
     UR_CALL_THROWS(queueFinish());
-
-    for (size_t i = 0; i < numCommandLists; i++) {
-      barrierEvents[i]->release();
-    }
   } catch (...) {
     // Ignore errors during destruction
   }
@@ -146,43 +133,9 @@ ur_result_t ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrier(
     ur_event_handle_t *phEvent) {
   TRACK_SCOPE_LATENCY(
       "ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrier");
-  // Since we use L0 in-order command lists, we don't need a real L0 barrier,
-  // just wait for requested events in potentially different queues and add a
-  // "barrier" event signal because it is already guaranteed that previous
-  // commands in this queue are completed when the signal is started. However,
-  // we do need to use barrier if profiling is enabled: see
-  // zeCommandListAppendWaitOnEvents
-  bool needsRealBarrier = (flags & UR_QUEUE_FLAG_PROFILING_ENABLE) != 0;
-  auto barrierFn = needsRealBarrier
-                       ? &ur_command_list_manager::appendEventsWaitWithBarrier
-                       : &ur_command_list_manager::appendEventsWait;
-
-  auto commandListManagersLocked = commandListManagers.lock();
-
-  // Enqueue wait for the user-provider events on the first command list.
-  UR_CALL(commandListManagersLocked[0].appendEventsWait(
-      numEventsInWaitList, phEventWaitList, barrierEvents[0]));
-
-  // Request barrierEvents[id] to be signaled on remaining command lists.
-  for (size_t id = 1; id < numCommandLists; id++) {
-    UR_CALL(commandListManagersLocked[id].appendEventsWait(0, nullptr,
-                                                           barrierEvents[id]));
-  }
-
-  // Enqueue barriers on all command lists by waiting on barrierEvents.
-
-  if (phEvent) {
-    UR_CALL(
-        std::invoke(barrierFn, commandListManagersLocked[0], numCommandLists,
-                    barrierEvents.data(),
-                    createEventIfRequested(eventPool.get(), phEvent, this)));
-  }
-
-  for (size_t id = phEvent ? 1 : 0; id < numCommandLists; id++) {
-    UR_CALL(std::invoke(barrierFn, commandListManagersLocked[0],
-                        numCommandLists, barrierEvents.data(), nullptr));
-  }
-
+  UR_CALL(commandListManager.lock()->appendEventsWaitWithBarrier(
+      numEventsInWaitList, phEventWaitList,
+      createEventAndStoreIfRequested(phEvent)));
   return UR_RESULT_SUCCESS;
 }