[UR][L0v2] Add initial record & replay implementation

unified-runtime/CMakeLists.txt

@@ -324,6 +324,7 @@ file(GLOB_RECURSE license_src ${license_glob})
 
 # Add license check target
 list(FILTER license_src EXCLUDE REGEX "registry.yml")
+list(FILTER format_src EXCLUDE REGEX "source/adapters/level_zero/external")
 add_custom_target(verify-licenses
     COMMAND ${Python3_EXECUTABLE}
         "${PROJECT_SOURCE_DIR}/scripts/verify_license.py"

unified-runtime/source/adapters/level_zero/CMakeLists.txt

@@ -149,7 +149,6 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/graph.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
@@ -169,6 +168,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_provider_normal.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_provider.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/event.hpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/graph.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp

unified-runtime/source/adapters/level_zero/common.hpp

@@ -219,8 +219,6 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString);
   {                                                                            \
     ze_result_t ZeResult = ZeName ZeArgs;                                      \
     if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) {     \
-      UR_DFAILURE("failed ZE call " #ZeName " with " #ZeArgs ", with result:"  \
-                  << Result);                                                  \
       throw ze2urResult(Result);                                               \
     }                                                                          \
   }

unified-runtime/source/adapters/level_zero/external/README.md

@@ -0,0 +1,3 @@
+This subdirectory contains a copy of Level Zero experimental headers.
+Repo: https://github.com/intel/compute-runtime
+Tag: 25.44.36015.5

unified-runtime/source/adapters/level_zero/external/driver_experimental/mcl_ext/ze_mutable_command_list_exp.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <level_zero/ze_api.h>
+
+typedef ze_result_t(ZE_APICALL *ze_pfnCommandListGetNextCommandIdExpCb_t)(
+    ze_command_list_handle_t hCommandList,
+    const ze_mutable_command_id_exp_desc_t *desc, uint64_t *pCommandId);
+
+typedef ze_result_t(ZE_APICALL *ze_pfnCommandListUpdateMutableCommandsExpCb_t)(
+    ze_command_list_handle_t hCommandList,
+    const ze_mutable_commands_exp_desc_t *desc);
+
+typedef ze_result_t(
+    ZE_APICALL *ze_pfnCommandListUpdateMutableCommandSignalEventExpCb_t)(
+    ze_command_list_handle_t hCommandList, uint64_t commandId,
+    ze_event_handle_t hSignalEvent);
+
+typedef ze_result_t(
+    ZE_APICALL *ze_pfnCommandListUpdateMutableCommandWaitEventsExpCb_t)(
+    ze_command_list_handle_t hCommandList, uint64_t commandId,
+    uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents);
+
+typedef ze_result_t(
+    ZE_APICALL *ze_pfnCommandListGetNextCommandIdWithKernelsExpCb_t)(
+    ze_command_list_handle_t hCommandList,
+    const ze_mutable_command_id_exp_desc_t *desc, uint32_t numKernels,
+    ze_kernel_handle_t *phKernels, uint64_t *pCommandId);
+
+typedef ze_result_t(
+    ZE_APICALL *ze_pfnCommandListUpdateMutableCommandKernelsExpCb_t)(
+    ze_command_list_handle_t hCommandList, uint32_t numKernels,
+    uint64_t *pCommandId, ze_kernel_handle_t *phKernels);

unified-runtime/source/adapters/level_zero/external/driver_experimental/mcl_ext/zex_mutable_cmdlist_control_flow_ext.h

@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/ze_stypes.h"
+#include <level_zero/ze_api.h>
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Label handle.
+typedef struct _zex_label_handle_t *zex_label_handle_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Label descriptor.
+typedef struct _zex_label_desc_t {
+  ze_structure_type_ext_t stype =
+      ZEX_STRUCTURE_TYPE_LABEL_DESCRIPTOR; ///< [in] type of this structure
+  const void *pNext =
+      nullptr; ///< [in][optional] pointer to extension-specific structure
+
+  const char *name;   ///< [in][optional] null-terminated name of the label
+  uint32_t alignment; ///< [in][optional] minimum alignment of the label
+} zex_label_desc_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Operand flags
+typedef uint32_t zex_operand_flags_t;
+typedef enum _zex_operand_flag_t {
+  ZEX_OPERAND_FLAG_USES_VARIABLE =
+      ZE_BIT(0), // variable is being used - passed via memory
+  ZEX_OPERAND_FLAG_JUMP_ON_CLEAR =
+      ZE_BIT(1) // jump on '0', instead of default '1'
+} zex_operand_flag_t;
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Operand descriptor.
+typedef struct _zex_operand_desc_t {
+  ze_structure_type_ext_t stype =
+      ZEX_STRUCTURE_TYPE_OPERAND_DESCRIPTOR; ///< [in] type of this structure
+  const void *pNext =
+      nullptr; ///< [in][optional] pointer to extension-specific structure
+
+  void *memory;  // if memory is NULL then offset is interpreted as MMIO
+                 // if flag ZEX_OPERAND_FLAG_USES_VARIABLE is set memory is
+                 // interpreted as variable
+  size_t offset; // offset within memory or register MMIO
+  uint32_t size; // operand size - dword
+  zex_operand_flags_t flags; // flags
+} zex_operand_desc_t;
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Creates/returns label based on name provided in
+///        label descriptor.
+///
+/// @details
+///     - When label with the name provided in label descriptor does not
+///       exist new label is created.
+///     - If label with provided name exists it's returned.
+///     - Label at creation is undefined (points to nothing).
+///
+/// @returns
+///     - ZE_RESULT_SUCCESS
+///     - ZE_RESULT_ERROR_INVALID_ARGUMENT
+///         + nullptr == hCommandList
+///         + nullptr == pLabelDesc
+///         + nullptr == phLabel
+///         + pLabelDesc->alignment & (pLabelDesc->alignment - 1) != 0
+///     - ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY
+///         + bad_alloc
+ze_result_t ZE_APICALL zexCommandListGetLabel(
+    ze_command_list_handle_t hCommandList, ///< [in] handle of command list
+    const zex_label_desc_t *pLabelDesc,    ///< [in] pointer to label descriptor
+    zex_label_handle_t *phLabel            ///< [out] pointer to handle of label
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Sets label to point to current location in command list.
+///
+/// @details
+///     - Sets label address to current location.
+///     - All previous jumps to this label are patched with label's address.
+///     - Future jumps to this label will be patched with label's address.
+///     - Label can be set only once.
+///     - Label must be located in the same command list, as it was created
+///
+/// @returns
+///     - ZE_RESULT_SUCCESS
+///     - ZE_RESULT_ERROR_INVALID_ARGUMENT
+///         + nullptr == hCommandList
+///         + nullptr == hLabel
+///     - ZE_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + variable already set
+///     - ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT
+///         + label's address is not aligned to it's alignment
+ze_result_t ZE_APICALL zexCommandListSetLabel(
+    ze_command_list_handle_t hCommandList, ///< [in] handle of command list
+    zex_label_handle_t hLabel              ///< [in] handle of label
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Appends jump to label predicated by condition.
+///
+/// @details
+///     If condition is present:
+///        If condition->memory is present:
+///            If ZEX_OPERAND_FLAG_USES_VARIABLE flag is set:
+///              - Append MI_LOAD_REGISTER_MEM
+///                 reg = PREDICATE_RESULT_2
+///                 memAddr = gpu address of buffer variable
+///            Else:
+///              - Append MI_LOAD_REGISTER_MEM
+///                reg = PREDICATE_RESULT_2
+///                memAddr = pCondition->memory + pCondition->offset
+///        Else:
+///          - Append MI_LOAD_REGISTER_REG
+///            regDst = PREDICATE_RESULT_2
+///            regSrc = pCondition->offset (need to pass MMIO)
+///
+///        If ZEX_OPERAND_FLAG_JUMP_ON_CLEAR flag is set: //  jumps when '0'
+///          - Append MI_SET_PREDICATE(ENABLE_ON_SET)
+///        Else:                                          //  jumps when '1'
+///          - Append MI_SET_PREDICATE(ENABLE_ON_CLEAR)
+///
+///     - Append MI_BATCH_BUFFER_START
+///       predicationEnabled = condition is present
+///       jumpAddress = label's address
+///     - Append MI_SET_PREDICATE(DISABLE) - after BB_START
+///     - Append MI_SET_PREDICATE(DISABLE) - at label's address
+///
+/// @returns
+///     - ZE_RESULT_SUCCESS
+///     - ZE_RESULT_ERROR_INVALID_ARGUMENT
+///         + nullptr == hCommandList
+///         + nullptr == hLabel
+ze_result_t ZE_APICALL zexCommandListAppendJump(
+    ze_command_list_handle_t hCommandList, ///< [in] handle of command list
+    zex_label_handle_t hLabel,             ///< [in] handle of label
+    zex_operand_desc_t *pCondition         ///< [in][opt] pointer to operand
+);
+
+typedef ze_result_t(ZE_APICALL *zex_pfnCommandListGetLabelCb_t)(
+    ze_command_list_handle_t hCommandList, const zex_label_desc_t *pLabelDesc,
+    zex_label_handle_t *phLabel);
+
+typedef ze_result_t(ZE_APICALL *zex_pfnCommandListSetLabelCb_t)(
+    ze_command_list_handle_t hCommandList, zex_label_handle_t hLabel);
+
+typedef ze_result_t(ZE_APICALL *zex_pfnCommandListAppendJumpCb_t)(
+    ze_command_list_handle_t hCommandList, zex_label_handle_t hLabel,
+    zex_operand_desc_t *pCondition);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif

unified-runtime/source/adapters/level_zero/external/driver_experimental/mcl_ext/zex_mutable_cmdlist_dispatch_ext.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_variable_ext.h"
+#include <level_zero/ze_api.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Sets kernel group size to provided variable.
+ze_result_t ZE_APICALL zexKernelSetVariableGroupSize(
+    ze_kernel_handle_t hKernel,              ///< [in] handle of kernel
+    zex_variable_handle_t hGroupSizeVariable ///< [in] handle of variable
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Appends kernel launch to command list with variable group size.
+ze_result_t ZE_APICALL zexCommandListAppendVariableLaunchKernel(
+    ze_command_list_handle_t
+        hCommandList,           ///< [in] hnadle of mutable command list
+    ze_kernel_handle_t hKernel, ///< [in] handle of kernel
+    zex_variable_handle_t hGroupCountVariable, ///< [in] handle of variable
+    ze_event_handle_t hSignalEvent,            ///< [in] handle to signal event
+    uint32_t numWaitEvents,                    ///< [in] num events to wait for
+    ze_event_handle_t *phWaitEvents); ///< [in] array of events to wait for
+
+typedef ze_result_t(ZE_APICALL *zex_pfnKernelSetVariableGroupSizeCb_t)(
+    ze_kernel_handle_t hKernel, zex_variable_handle_t hGroupSizeVariable);
+
+typedef ze_result_t(
+    ZE_APICALL *zex_pfnCommandListAppendVariableLaunchKernelCb_t)(
+    ze_command_list_handle_t hCommandList, ze_kernel_handle_t hKernel,
+    zex_variable_handle_t hGroupCountVariable, ze_event_handle_t hSignalEvent,
+    uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif

unified-runtime/source/adapters/level_zero/external/driver_experimental/mcl_ext/zex_mutable_cmdlist_ext.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/driver_experimental/mcl_ext/ze_mutable_command_list_exp.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_control_flow_ext.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_dispatch_ext.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_serialization_ext.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_temp_mem_ext.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_variable_ext.h"
+#include "level_zero/driver_experimental/mcl_ext/zex_mutable_cmdlist_variable_info_ext.h"

unified-runtime/source/adapters/level_zero/external/driver_experimental/mcl_ext/zex_mutable_cmdlist_serialization_ext.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2025 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <level_zero/ze_api.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieve native MCL binary from cmdlist.
+ze_result_t ZE_APICALL zexCommandListGetNativeBinary(
+    ze_command_list_handle_t
+        hCommandList, ///< [in] handle of mutable command list
+    void *pBinary,    ///< [in,out][optional] byte pointer to native MCL binary
+    size_t *pBinarySize, ///< [in,out] size of native MCL binary in bytes
+    const void *pModule, ///< [in] byte pointer to module containing kernels'
+                         ///< used in cmdlist
+    size_t moduleSize    ///< [in] size in bytes of module containing kernels's
+                         ///< used in cmdlist
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Load command list from native MCL binary.
+ze_result_t ZE_APICALL zexCommandListLoadNativeBinary(
+    ze_command_list_handle_t
+        hCommandList,    ///< [in] handle of mutable command list
+    const void *pBinary, ///< [in] byte pointer to native MCL binary
+    size_t binarySize    ///< [in] size of native MCL binary in bytes
+);
+
+typedef ze_result_t(ZE_APICALL *zex_pfnCommandListGetNativeBinaryCb_t)(
+    ze_command_list_handle_t hCommandList, void *pBinary, size_t *pBinarySize,
+    const void *pModule, size_t moduleSize);
+
+typedef ze_result_t(ZE_APICALL *zex_pfnCommandListLoadNativeBinaryCb_t)(
+    ze_command_list_handle_t hCommandList, const void *pBinary,
+    size_t binarySize);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif