diff --git a/sycl/cmake/modules/BuildUnifiedRuntime.cmake b/sycl/cmake/modules/BuildUnifiedRuntime.cmake index f4c3849efb027..fc8ff1c20eb9e 100644 --- a/sycl/cmake/modules/BuildUnifiedRuntime.cmake +++ b/sycl/cmake/modules/BuildUnifiedRuntime.cmake @@ -223,6 +223,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) -DUR_BUILD_ADAPTER_HIP:BOOL=${UR_BUILD_ADAPTER_HIP} -DUR_BUILD_ADAPTER_NATIVE_CPU:BOOL=${UR_BUILD_ADAPTER_NATIVE_CPU} -DUR_STATIC_LOADER:BOOL=${UR_STATIC_LOADER} + -DUR_STATIC_ADAPTER_L0:BOOL=${UR_STATIC_ADAPTER_L0} + -DUR_STATIC_ADAPTER_L0_V2:BOOL=${UR_STATIC_ADAPTER_L0_V2} -DUR_STATIC_ADAPTER_OPENCL:BOOL=${UR_STATIC_ADAPTER_OPENCL} -DUMF_BUILD_EXAMPLES:BOOL=${UMF_BUILD_EXAMPLES} -DUMF_BUILD_SHARED_LIBRARY:BOOL=${UMF_BUILD_SHARED_LIBRARY} @@ -272,16 +274,50 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) ${LLVM_BINARY_DIR}/lib/ur_loaderd.lib ${LLVM_BINARY_DIR}/lib/ur_commond.lib dbghelp) + set(_urd_static_l0 FALSE) # Link static adapters into the loader foreach(adapter ${SYCL_ENABLE_BACKENDS}) ur_adapter_is_static(${adapter} adapter_is_static) - set(build_var "UR_BUILD_ADAPTER_${adapter}") - string(TOUPPER "${build_var}" build_var) + # The Level Zero adapters use the abbreviated build flags + # UR_BUILD_ADAPTER_L0 / _L0_V2 rather than the backend name. + string(TOUPPER "${adapter}" adapter_upper) + if(adapter_upper STREQUAL "LEVEL_ZERO") + set(build_var "UR_BUILD_ADAPTER_L0") + elseif(adapter_upper STREQUAL "LEVEL_ZERO_V2") + set(build_var "UR_BUILD_ADAPTER_L0_V2") + else() + set(build_var "UR_BUILD_ADAPTER_${adapter_upper}") + endif() if(adapter_is_static AND DEFINED ${build_var} AND ${build_var}) target_link_libraries(UnifiedRuntimeLoaderDebug INTERFACE ${LLVM_BINARY_DIR}/lib/ur_adapter_${adapter}d.lib) + if(build_var STREQUAL "UR_BUILD_ADAPTER_L0" OR + build_var STREQUAL "UR_BUILD_ADAPTER_L0_V2") + set(_urd_static_l0 TRUE) + endif() endif() endforeach() + # A static umfd.lib does not absorb its private helper archives, so link + # them too. umfd and the helpers are copied below. + target_link_libraries(UnifiedRuntimeLoaderDebug INTERFACE + ${LLVM_BINARY_DIR}/lib/umfd.lib + ${LLVM_BINARY_DIR}/lib/umf_bad.lib + ${LLVM_BINARY_DIR}/lib/umf_utilsd.lib + ${LLVM_BINARY_DIR}/lib/umf_coarsed.lib) + if(_urd_static_l0) + # The static L0 adapters need ze_loader. Copy the debug-CRT ze_loader + # from the debug sub-build under a 'd' name (it has no debug postfix, so + # copying it as-is would collide with the release in-tree ze_loader.lib). + list(APPEND URD_COPY_FILES ${LLVM_BINARY_DIR}/lib/ze_loaderd.lib) + add_custom_command( + OUTPUT ${LLVM_BINARY_DIR}/lib/ze_loaderd.lib + COMMAND ${CMAKE_COMMAND} -E copy + ${URD_INSTALL_DIR}/lib/ze_loader.lib + ${LLVM_BINARY_DIR}/lib/ze_loaderd.lib + ) + target_link_libraries(UnifiedRuntimeLoaderDebug INTERFACE + ${LLVM_BINARY_DIR}/lib/ze_loaderd.lib) + endif() endif() foreach(adapter ${SYCL_ENABLE_BACKENDS}) ur_adapter_is_static(${adapter} adapter_is_static) @@ -294,6 +330,22 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) endforeach() # Also copy umfd.dll/umfd.lib urd_copy_library_to_build(umfd ${UMF_BUILD_SHARED_LIBRARY}) + # A static umfd.lib doesn't absorb its private helper archives (umf_ba / + # umf_utils / umf_coarse), so a consumer linking umfd.lib must link them too. + # They aren't installed, but the debug sub-build produces them (debug CRT) at + # ${URD_BINARY_DIR}/lib without a 'd' postfix, so copy them under a 'd' name. + if(NOT UMF_BUILD_SHARED_LIBRARY) + foreach(_umfd_helper umf_ba umf_utils umf_coarse) + list(APPEND URD_COPY_FILES ${LLVM_BINARY_DIR}/lib/${_umfd_helper}d.lib) + add_custom_command( + OUTPUT ${LLVM_BINARY_DIR}/lib/${_umfd_helper}d.lib + COMMAND ${CMAKE_COMMAND} -E copy + ${URD_BINARY_DIR}/lib/${_umfd_helper}.lib + ${LLVM_BINARY_DIR}/lib/${_umfd_helper}d.lib + DEPENDS unified-runtimed + ) + endforeach() + endif() add_custom_target(unified-runtimed-build ALL DEPENDS ${URD_COPY_FILES}) add_dependencies(unified-runtimed-build unified-runtimed) diff --git a/sycl/test-e2e/Adapters/dll-detach-order.cpp b/sycl/test-e2e/Adapters/dll-detach-order.cpp index 41a310206156d..dfe83540407cf 100644 --- a/sycl/test-e2e/Adapters/dll-detach-order.cpp +++ b/sycl/test-e2e/Adapters/dll-detach-order.cpp @@ -8,6 +8,7 @@ // whatever adapter THIS is // CHECK: ---> urAdapterRelease -// CHECK: [INFO]: unloaded adapter +// Statically-linked adapters have no library to unload, so the loader's +// "unloaded adapter" message is only emitted for dynamically-loaded adapters. // CHECK: ---> urLoaderTearDown diff --git a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp index aeff4cafa5a04..c832150281dcf 100644 --- a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp +++ b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp @@ -151,6 +151,8 @@ void preloadLibraries() { // unloaded prematurely. the Only trusted loader is the one that is loaded // from the system32 directory. LoadLibraryExW(L"ze_loader.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + // Load the OpenCL ICD loader dynamic library, for the same reason as above. + LoadLibraryExW(L"OpenCL.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); // Restore system error handling. (void)SetErrorMode(SavedMode); diff --git a/unified-runtime/CMakeLists.txt b/unified-runtime/CMakeLists.txt index 28256344244a4..73701c6c3cf48 100644 --- a/unified-runtime/CMakeLists.txt +++ b/unified-runtime/CMakeLists.txt @@ -47,8 +47,9 @@ option(UR_BUILD_ADAPTER_NATIVE_CPU "Build the Native-CPU adapter" OFF) option(UR_BUILD_ADAPTER_ALL "Build all currently supported adapters" OFF) option(UR_BUILD_ADAPTER_L0_V2 "Build the (experimental) Level-Zero v2 adapter" OFF) option(UR_BUILD_ADAPTER_OFFLOAD "Build the experimental Offload adapter" OFF) -option(UR_STATIC_ADAPTER_L0 "Build the Level-Zero adapter as static and embed in the loader" OFF) +option(UR_STATIC_ADAPTER_L0 "Build the Level-Zero adapter as static and embed in the loader" ON) option(UR_STATIC_ADAPTER_OPENCL "Build the OpenCL adapter as static library with dynamic OpenCL loading" ON) +option(UR_STATIC_ADAPTER_L0_V2 "Build the Level-Zero v2 adapter as static and embed in the loader" ON) option(UR_BUILD_EXAMPLE_CODEGEN "Build the codegen example." OFF) option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF) option(UR_ENABLE_ASSERTIONS "Enable assertions for all build types" OFF) diff --git a/unified-runtime/cmake/FetchLevelZero.cmake b/unified-runtime/cmake/FetchLevelZero.cmake index 093b8c3319c48..40d346a4cd987 100644 --- a/unified-runtime/cmake/FetchLevelZero.cmake +++ b/unified-runtime/cmake/FetchLevelZero.cmake @@ -90,8 +90,16 @@ if(NOT LEVEL_ZERO_LIB_NAME AND NOT LEVEL_ZERO_LIBRARY) target_compile_options(ze_loader PRIVATE $<$,GNU;Clang;Intel;IntelLLVM>:-Wno-error> - $<$:/WX- /UUNICODE> + $<$:/WX- /UUNICODE /GL-> ) + # Neutralize the dllexport macros so a consuming DLL doesn't re-export the + # ze*/zes*/zel* symbols from this statically-linked loader. + if(WIN32 AND (UR_STATIC_ADAPTER_L0 OR UR_STATIC_ADAPTER_L0_V2)) + target_compile_definitions(ze_loader PRIVATE + ZE_APIEXPORT= + ZE_DLLEXPORT= + ) + endif() endif() add_library(LevelZeroLoader INTERFACE) diff --git a/unified-runtime/cmake/helpers.cmake b/unified-runtime/cmake/helpers.cmake index 6d9df20ed5e10..8d921bb20e657 100644 --- a/unified-runtime/cmake/helpers.cmake +++ b/unified-runtime/cmake/helpers.cmake @@ -303,9 +303,13 @@ function(ur_adapter_is_static adapter out_var) set(static_var "UR_STATIC_ADAPTER_${adapter}") string(TOUPPER "${static_var}" static_var) string(TOUPPER "${adapter}" adapter) - # Special case: LEVEL_ZERO uses UR_STATIC_ADAPTER_L0 + # Special case: the Level Zero adapters use the abbreviated flag names + # UR_STATIC_ADAPTER_L0 / UR_STATIC_ADAPTER_L0_V2 rather than the backend + # name (LEVEL_ZERO / LEVEL_ZERO_V2). if(adapter STREQUAL "LEVEL_ZERO") set(static_var "UR_STATIC_ADAPTER_L0") + elseif(adapter STREQUAL "LEVEL_ZERO_V2") + set(static_var "UR_STATIC_ADAPTER_L0_V2") endif() if(DEFINED ${static_var} AND ${static_var}) set(${out_var} TRUE PARENT_SCOPE) diff --git a/unified-runtime/scripts/generate_code.py b/unified-runtime/scripts/generate_code.py index 08568eab81447..ae0952580e145 100644 --- a/unified-runtime/scripts/generate_code.py +++ b/unified-runtime/scripts/generate_code.py @@ -417,10 +417,9 @@ def generate_loader(path, section, namespace, tags, version, specs, meta): def _mako_interface_loader_api( - path, adapter, ext, namespace, tags, version, specs, meta + dstpath, adapter, ext, namespace, tags, version, specs, meta ): """generates c/c++ files from the specification documents""" - dstpath = os.path.join(path, adapter) os.makedirs(dstpath, exist_ok=True) template = f"ur_interface_loader.{ext}.mako" @@ -462,11 +461,33 @@ def generate_adapters(path, section, namespace, tags, version, specs, meta): dstpath, "adapter", "def", namespace, tags, version, specs, meta ) + level_zero_dstpath = os.path.join(path, "adapters", "level_zero") loc += _mako_interface_loader_api( - dstpath, "level_zero", "cpp", namespace, tags, version, specs, meta + level_zero_dstpath, "level_zero", "cpp", namespace, tags, version, specs, meta ) loc += _mako_interface_loader_api( - dstpath, "level_zero", "hpp", namespace, tags, version, specs, meta + level_zero_dstpath, "level_zero", "hpp", namespace, tags, version, specs, meta + ) + level_zero_v2_dstpath = os.path.join(level_zero_dstpath, "v2") + loc += _mako_interface_loader_api( + level_zero_v2_dstpath, + "level_zero_v2", + "cpp", + namespace, + tags, + version, + specs, + meta, + ) + loc += _mako_interface_loader_api( + level_zero_v2_dstpath, + "level_zero_v2", + "hpp", + namespace, + tags, + version, + specs, + meta, ) loc += _mako_interface_loader_api( dstpath, "opencl", "cpp", namespace, tags, version, specs, meta diff --git a/unified-runtime/scripts/templates/helper.py b/unified-runtime/scripts/templates/helper.py index d3933dc953000..b4adc3727af7f 100644 --- a/unified-runtime/scripts/templates/helper.py +++ b/unified-runtime/scripts/templates/helper.py @@ -967,6 +967,7 @@ def make_param_lines( format: List[str] = ["type", "name", "delim", "desc"], delim: str = ",", replacements: dict = {}, + global_handles: bool = False, ): """ Public: @@ -984,6 +985,12 @@ def make_param_lines( tname = _get_type_name(namespace, tags, obj, item) + # Qualify handle types with leading '::' so they resolve to the global + # scope, not an enclosing adapter namespace's same-named internal type. + if global_handles and type_traits.is_handle(tname): + base = type_traits.base(tname) # strips const/* -> ur_X_handle_t + tname = tname.replace(base, "::" + base, 1) + words = [] if "type*" in format: ptname = tname + "*" @@ -1707,8 +1714,14 @@ def get_queue_related_functions(specs: List[dict], namespace: str, tags) -> List return funcs +# Handle types that have no adapter-internal alias: they resolve to the +# global type even inside an adapter namespace, so they are forwarded as-is +# (no cast) when bridging the public API into the internal queue interface. +_QUEUE_NO_CAST_HANDLES = ("native_handle_t", "external_semaphore_handle_t") + + def transform_queue_related_function_name( - namespace, tags, obj, format=["name", "type"] + namespace, tags, obj, format=["name", "type"], cast_handles=False ) -> str: """ Public: @@ -1716,6 +1729,10 @@ def transform_queue_related_function_name( - remove $x prefix - make first letter lowercase - remove first param (queue) + - if cast_handles, wrap each remaining handle argument in v2_cast() + so the global public-API handle is converted to the internal type + the queue interface method expects (native/external-semaphore + handles have no internal alias and are left as-is) """ function_name = make_func_name(namespace, tags, obj).replace(namespace, "") function_name = function_name[0].lower() + function_name[1:] @@ -1723,6 +1740,18 @@ def transform_queue_related_function_name( if obj["params"][0]["type"] != "$x_queue_handle_t": raise ValueError("First parameter is not a queue handle.") + if cast_handles: + args = [] + for item in obj["params"][1:]: + name = _get_param_name(namespace, tags, item) + if type_traits.is_handle(item["type"]) and not any( + h in item["type"] for h in _QUEUE_NO_CAST_HANDLES + ): + args.append("v2_cast(%s)" % name) + else: + args.append(name) + return "{}({})".format(function_name, ", ".join(args)) + params = make_param_lines(namespace, tags, obj, format=format) params = params[1:] diff --git a/unified-runtime/scripts/templates/queue_api.cpp.mako b/unified-runtime/scripts/templates/queue_api.cpp.mako index b32af92077bf8..29667daf540f4 100644 --- a/unified-runtime/scripts/templates/queue_api.cpp.mako +++ b/unified-runtime/scripts/templates/queue_api.cpp.mako @@ -25,10 +25,11 @@ from templates import helper as th #include "queue_handle.hpp" #include "ur_util.hpp" +namespace ur::level_zero::v2 { + ur_queue_t_::~ur_queue_t_() {} ## FUNCTION ################################################################### -namespace ${x}::level_zero { %for obj in th.get_queue_related_functions(specs, n, tags): %if not 'Release' in obj['name'] and not 'Retain' in obj['name']: %if 'guard' in obj: @@ -36,12 +37,12 @@ namespace ${x}::level_zero { %endif ${x}_result_t ${th.make_func_name(n, tags, obj)}( - %for line in th.make_param_lines(n, tags, obj, format=["name", "type", "delim"]): + %for line in th.make_param_lines(n, tags, obj, format=["name", "type", "delim"], global_handles=True): ${line} %endfor ) try { - return ${obj['params'][0]['name']}->get().${th.transform_queue_related_function_name(n, tags, obj, format=["name"])}; + return v2_cast(${obj['params'][0]['name']})->get().${th.transform_queue_related_function_name(n, tags, obj, format=["name"], cast_handles=True)}; } catch(...) { return exceptionToResult(std::current_exception()); } @@ -54,12 +55,12 @@ try { %endif ${x}_result_t ${th.make_func_name(n, tags, obj)}( - %for line in th.make_param_lines(n, tags, obj, format=["name", "type", "delim"]): + %for line in th.make_param_lines(n, tags, obj, format=["name", "type", "delim"], global_handles=True): ${line} %endfor ) try { - return ${obj['params'][0]['name']}->${th.transform_queue_related_function_name(n, tags, obj, format=["name"])}; + return v2_cast(${obj['params'][0]['name']})->${th.transform_queue_related_function_name(n, tags, obj, format=["name"], cast_handles=True)}; } catch(...) { return exceptionToResult(std::current_exception()); } @@ -68,4 +69,5 @@ try { %endif %endif %endfor -} + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/scripts/templates/queue_api.hpp.mako b/unified-runtime/scripts/templates/queue_api.hpp.mako index 08dbefb78dfb9..d6e0c4c713f7b 100644 --- a/unified-runtime/scripts/templates/queue_api.hpp.mako +++ b/unified-runtime/scripts/templates/queue_api.hpp.mako @@ -24,8 +24,11 @@ from templates import helper as th #pragma once #include +#include "common.hpp" #include "queue_extensions.hpp" +namespace ur::level_zero::v2 { + struct ur_queue_t_ : ur_queue_extensions { virtual ~ur_queue_t_(); @@ -41,3 +44,5 @@ struct ur_queue_t_ : ur_queue_extensions { %endif %endfor }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/scripts/templates/ur_interface_loader.cpp.mako b/unified-runtime/scripts/templates/ur_interface_loader.cpp.mako index b750cad5c7da2..2208a360bdd44 100644 --- a/unified-runtime/scripts/templates/ur_interface_loader.cpp.mako +++ b/unified-runtime/scripts/templates/ur_interface_loader.cpp.mako @@ -8,6 +8,12 @@ from templates import helper as th x=tags['$x'] X=x.upper() Adapter=adapter.upper() + adapter_namespace={ + 'level_zero': 'ur::level_zero::v1', + 'level_zero_v2': 'ur::level_zero::v2', + }.get(adapter, 'ur::'+adapter) + is_level_zero = adapter in ('level_zero', 'level_zero_v2') + %>//===--------- ${n}_interface_loader.cpp - Level Zero Adapter ------------===// // // @@ -35,8 +41,11 @@ static ur_result_t validateProcInputs(ur_api_version_t version, void *pDdiTable) return UR_RESULT_SUCCESS; } +%if is_level_zero: +#include "ur_interface_loader_common_forwarders.hpp" +%endif #ifdef UR_STATIC_ADAPTER_${Adapter} -namespace ${n}::${adapter} { +namespace ${adapter_namespace} { #else extern "C" { #endif @@ -60,7 +69,7 @@ ${X}_APIEXPORT ${x}_result_t ${X}_APICALL ${tbl['export']['name']}( %if 'guard' in obj and 'guard' not in tbl: #if ${obj['guard']} %endif - pDdiTable->${th.append_ws(th.make_pfn_name(n, tags, obj), 43)} = ${n}::${adapter}::${th.make_func_name(n, tags, obj)}; + pDdiTable->${th.append_ws(th.make_pfn_name(n, tags, obj), 43)} = ${adapter_namespace + '::' if is_level_zero else ''}${th.make_func_name(n, tags, obj)}; %if 'guard' in obj and 'guard' not in tbl: #endif // ${obj['guard']} %endif @@ -75,7 +84,7 @@ ${X}_APIEXPORT ${x}_result_t ${X}_APICALL ${tbl['export']['name']}( %endfor #ifdef UR_STATIC_ADAPTER_${Adapter} -} // namespace ur::${adapter} +} // namespace ${adapter_namespace} #else } // extern "C" #endif @@ -89,7 +98,7 @@ ur_result_t populateDdiTable(ur_dditable_t *ddi) { ur_result_t result; #ifdef UR_STATIC_ADAPTER_${Adapter} -#define NAMESPACE_ ::ur::${adapter} +#define NAMESPACE_ ::${adapter_namespace} #else #define NAMESPACE_ #endif @@ -113,7 +122,7 @@ ur_result_t populateDdiTable(ur_dditable_t *ddi) { } -namespace ur::${adapter} { +namespace ${adapter_namespace} { const ${x}_dditable_t *ddi_getter::value() { static std::once_flag flag; static ${x}_dditable_t table; @@ -127,4 +136,4 @@ ur_result_t urAdapterGetDdiTables(${x}_dditable_t *ddi) { return populateDdiTable(ddi); } #endif -} +} // namespace ${adapter_namespace} diff --git a/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako b/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako index aa27674f2c923..3a9b4f7d1b26c 100644 --- a/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako +++ b/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako @@ -8,6 +8,13 @@ from templates import helper as th x=tags['$x'] X=x.upper() Adapter=adapter.upper() + + # See ur_interface_loader.cpp.mako: map the Level Zero v1/v2 adapter ids to + # their real nested C++ namespace. + adapter_namespace={ + 'level_zero': 'ur::level_zero::v1', + 'level_zero_v2': 'ur::level_zero::v2', + }.get(adapter, 'ur::'+adapter) %>//===--------- ${n}_interface_loader.hpp - Level Zero Adapter ------------===// // // @@ -22,7 +29,7 @@ from templates import helper as th #include #include -namespace ${n}::${adapter} { +namespace ${adapter_namespace} { %for s in specs: %for obj in th.filter_items(s['objects'], 'type', 'function'): %if 'guard' in obj: @@ -30,7 +37,7 @@ namespace ${n}::${adapter} { %endif %if not th.obj_traits.is_loader_only(obj): ${x}_result_t ${th.make_func_name(n, tags, obj)}( - %for line in th.make_param_lines(n, tags, obj, format=["type", "name", "delim"]): + %for line in th.make_param_lines(n, tags, obj, format=["type", "name", "delim"], global_handles=True): ${line} %endfor ); @@ -47,4 +54,4 @@ ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); struct ddi_getter { const static ${x}_dditable_t *value(); }; -} +} // namespace ${adapter_namespace} diff --git a/unified-runtime/source/adapters/cuda/common.cpp b/unified-runtime/source/adapters/cuda/common.cpp index 759d2c4671152..1d510b93b8169 100644 --- a/unified-runtime/source/adapters/cuda/common.cpp +++ b/unified-runtime/source/adapters/cuda/common.cpp @@ -172,14 +172,6 @@ void setPluginSpecificMessage(CUresult cu_res) { } namespace umf { -ur_result_t getProviderNativeError(const char *providerName, int32_t error) { - if (strcmp(providerName, "CUDA") == 0) { - return mapErrorUR(static_cast(error)); - } - - return UR_RESULT_ERROR_UNKNOWN; -} - ur_result_t CreateProviderPool(int cuDevice, void *cuContext, umf_usm_memory_type_t type, umf_memory_provider_handle_t *provider, diff --git a/unified-runtime/source/adapters/cuda/common.hpp b/unified-runtime/source/adapters/cuda/common.hpp index 544c4ac345ce7..a5a1da12be8b8 100644 --- a/unified-runtime/source/adapters/cuda/common.hpp +++ b/unified-runtime/source/adapters/cuda/common.hpp @@ -54,8 +54,24 @@ ur_result_t CreateProviderPool(int cuDevice, void *cuContext, umf_usm_memory_type_t type, umf_memory_provider_handle_t *provider, umf_memory_pool_handle_t *pool); + +// Per-adapter translator for umf2urResult's provider-specific error path. +namespace { +[[maybe_unused]] inline ur_result_t +cudaGetProviderNativeError(const char *providerName, int32_t error) { + if (strcmp(providerName, "CUDA") == 0) { + return mapErrorUR(static_cast(error)); + } + return UR_RESULT_ERROR_UNKNOWN; +} +} // namespace } // namespace umf +#define UMF_GET_PROVIDER_NATIVE_ERROR ::umf::cudaGetProviderNativeError + +#include +#include + namespace ur::cuda { struct ddi_getter { const static ur_dditable_t *value(); diff --git a/unified-runtime/source/adapters/hip/usm.cpp b/unified-runtime/source/adapters/hip/usm.cpp index 78bbe9434816a..74d482bebe0f2 100644 --- a/unified-runtime/source/adapters/hip/usm.cpp +++ b/unified-runtime/source/adapters/hip/usm.cpp @@ -8,27 +8,34 @@ //===----------------------------------------------------------------------===// #include +#include -#include "adapter.hpp" -#include "common.hpp" -#include "context.hpp" -#include "device.hpp" -#include "platform.hpp" -#include "ur_util.hpp" -#include "usm.hpp" +#include +// Per-adapter translator for umf2urResult's provider-specific error path. namespace umf { -ur_result_t getProviderNativeError(const char *providerName, - int32_t nativeError) { +namespace { +[[maybe_unused]] inline ur_result_t +hipGetProviderNativeError(const char *providerName, int32_t nativeError) { if (strcmp(providerName, "HIP") == 0) { // HIP provider stores native errors of ur_result_t type return static_cast(nativeError); } - return UR_RESULT_ERROR_UNKNOWN; } +} // namespace } // namespace umf +#define UMF_GET_PROVIDER_NATIVE_ERROR ::umf::hipGetProviderNativeError + +#include "adapter.hpp" +#include "common.hpp" +#include "context.hpp" +#include "device.hpp" +#include "platform.hpp" +#include "ur_util.hpp" +#include "usm.hpp" + /// USM: Implements USM Host allocations using HIP Pinned Memory UR_APIEXPORT ur_result_t UR_APICALL urUSMHostAlloc(ur_context_handle_t hContext, const ur_usm_desc_t *pUSMDesc, diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 908bb021f0a8c..a6c2bdfb48704 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -2,6 +2,95 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_L0_V2) + set(UR_L0_COMMON_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/common/device.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/platform.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/common/adapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/platform.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/program.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/image_common.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/virtual_mem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/sampler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/enqueued_pool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/memory_export.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/kernel_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/memory_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/mutable_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/shared_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/command_buffer_command.cpp + ) + + add_library(ur_adapter_level_zero_common OBJECT ${UR_L0_COMMON_SOURCES}) + + add_ur_target_compile_options(ur_adapter_level_zero_common) + + # The common code needs the same include paths as adapters + target_include_directories(ur_adapter_level_zero_common PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/../../" + "${CMAKE_CURRENT_SOURCE_DIR}/../.." + "${CMAKE_CURRENT_SOURCE_DIR}/../../ur" + LevelZeroLoader-Headers + ) + + # If shared code depends on this define, keep it here so both adapters inherit + target_compile_definitions(ur_adapter_level_zero_common PUBLIC + UR_STATIC_LEVEL_ZERO + ) + + # Keep the warning suppression that applied to the shared code + target_compile_options(ur_adapter_level_zero_common PRIVATE + $<$:/wd4805 /wd4244> + ) + + target_link_libraries(ur_adapter_level_zero_common PUBLIC + ${PROJECT_NAME}::headers + ${PROJECT_NAME}::common + umf::umf + umf::headers + LevelZeroLoader + LevelZeroLoader-Headers + ComputeRuntimeLevelZero-Headers + ) + + set_target_properties(ur_adapter_level_zero_common PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) +endif() + +if(UR_STATIC_ADAPTER_L0 OR UR_STATIC_ADAPTER_L0_V2) + set(ADAPTER_L0_TARGETS_TO_INSTALL LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers) + # 'utils' target from 'level-zero-loader' includes path which is prefixed + # in the source directory, this breaks the installation of 'utils' target. + if(TARGET level_zero_utils) + set_target_properties(level_zero_utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") + list(APPEND ADAPTER_L0_TARGETS_TO_INSTALL level_zero_utils) + endif() + if(TARGET ze_loader) + list(APPEND ADAPTER_L0_TARGETS_TO_INSTALL ze_loader) + endif() + # Use the 'unified-runtime' component so a component-scoped install (the + # Windows debug sub-build) captures the debug-CRT ze_loader: a static + # adapter doesn't absorb its PRIVATE ze_loader dep, so the consumer links it. + install(TARGETS ${ADAPTER_L0_TARGETS_TO_INSTALL} + EXPORT ${PROJECT_NAME}-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT unified-runtime + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT unified-runtime + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT unified-runtime + ) + + install(TARGETS ur_adapter_level_zero_common + EXPORT ${PROJECT_NAME}-targets + OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif() + if(UR_BUILD_ADAPTER_L0) set(ADAPTER_LIB_TYPE SHARED) if(UR_STATIC_ADAPTER_L0) @@ -9,76 +98,51 @@ if(UR_BUILD_ADAPTER_L0) endif() add_ur_adapter(ur_adapter_level_zero ${ADAPTER_LIB_TYPE} - ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/adapter.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/image_common.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/physical_mem.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/sampler.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/kernel_helpers.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/memory_helpers.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/common/helpers/mutable_helpers.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/async_alloc.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer_command.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer_command.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/common.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/context.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/event.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/ur_level_zero.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/context.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_native.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader_common_forwarders.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/graph.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/enqueued_pool.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ur_level_zero.cpp ) install_ur_library(ur_adapter_level_zero) + target_link_libraries(ur_adapter_level_zero PRIVATE + ur_adapter_level_zero_common + ) + target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_LEVEL_ZERO) if(UR_STATIC_ADAPTER_L0) target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO) - set(ADAPTER_L0_TARGETS_TO_INSTALL LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers) - # 'utils' target from 'level-zero-loader' includes path which is prefixed - # in the source directory, this breaks the installation of 'utils' target. - if(TARGET level_zero_utils) - set_target_properties(level_zero_utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") - list(APPEND ADAPTER_L0_TARGETS_TO_INSTALL level_zero_utils) - endif() - if(TARGET ze_loader) - list(APPEND ADAPTER_L0_TARGETS_TO_INSTALL ze_loader) - endif() - install(TARGETS ${ADAPTER_L0_TARGETS_TO_INSTALL} - EXPORT ${PROJECT_NAME}-targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + # Neutralize dllexport/dllimport annotations on MSVC: the generated .cpp + # files use UR_APIEXPORT/UR_APICALL but a static library must not have + # __declspec(dllimport) on definitions (causes C2375) and does not need + # __declspec(dllexport) either. + target_compile_definitions(ur_adapter_level_zero PRIVATE + UR_APIEXPORT= + UR_APICALL= + UR_DLLEXPORT= ) endif() @@ -123,39 +187,12 @@ if(UR_BUILD_ADAPTER_L0) endif() if(UR_BUILD_ADAPTER_L0_V2) - add_ur_adapter(ur_adapter_level_zero_v2 - SHARED - # sources shared with legacy adapter - ${CMAKE_CURRENT_SOURCE_DIR}/adapter.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer_command.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/common.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/command_buffer_command.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/enqueued_pool.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp + set(ADAPTER_V2_LIB_TYPE SHARED) + if(UR_STATIC_ADAPTER_L0_V2) + set(ADAPTER_V2_LIB_TYPE STATIC) + endif() + + add_ur_adapter(ur_adapter_level_zero_v2 ${ADAPTER_V2_LIB_TYPE} # v2-only sources ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_buffer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_cache.hpp @@ -170,6 +207,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/graph.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.hpp @@ -189,6 +227,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/ipc_event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_batched.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_create.cpp @@ -196,9 +235,18 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm_p2p.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/ur_interface_loader_common_forwarders.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/ur_interface_loader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/adapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/platform.cpp ) install_ur_library(ur_adapter_level_zero_v2) + target_link_libraries(ur_adapter_level_zero_v2 PRIVATE + ur_adapter_level_zero_common + ) + if(NOT WIN32) # api.cpp contains NOT_SUPPORTED functions-only set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/v2/api.cpp @@ -206,7 +254,18 @@ if(UR_BUILD_ADAPTER_L0_V2) endif() target_compile_definitions(ur_adapter_level_zero_v2 PUBLIC UR_STATIC_LEVEL_ZERO) - target_compile_definitions(ur_adapter_level_zero_v2 PUBLIC UR_ADAPTER_LEVEL_ZERO_V2) + if(UR_STATIC_ADAPTER_L0_V2) + target_compile_definitions(ur_adapter_level_zero_v2 PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO_V2) + # Neutralize dllexport/dllimport annotations on MSVC: the generated .cpp + # files use UR_APIEXPORT/UR_APICALL but a static library must not have + # __declspec(dllimport) on definitions (causes C2375) and does not need + # __declspec(dllexport) either. + target_compile_definitions(ur_adapter_level_zero_v2 PRIVATE + UR_APIEXPORT= + UR_APICALL= + UR_DLLEXPORT= + ) + endif() # TODO: fix level_zero adapter conversion warnings target_compile_options(ur_adapter_level_zero_v2 PRIVATE diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index dbf15849c7b56..bca94e401a029 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -1,4 +1,4 @@ -//===--------- adapter.cpp - Level Zero Adapter ---------------------------===// +//===--------- adapter.cpp - Level Zero V1 Adapter -----------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -7,691 +7,95 @@ // //===----------------------------------------------------------------------===// -#include "adapter.hpp" -#include "common.hpp" -#include "ur_level_zero.hpp" -#include - -// The default value of the loader versions skiplist -// (SYCL_UR_L0_LOADER_SKIPLIST) -static const std::vector LoaderSkiplistDefault = {}; - -// As windows order of unloading dlls is reversed from linux, windows will call -// umfTearDown before it could release umf objects in level_zero, so we call -// umfInit on urAdapterGet and umfAdapterTearDown to enforce the teardown of umf -// after umf objects are destructed. -#if defined(_WIN32) -#include -#endif +#include +#include +#include +#include -ZeUSMImportExtension ZeUSMImport; -ur_adapter_handle_t_ *GlobalAdapter; +#include "common.hpp" +#include "common/adapter.hpp" +#include "common/device.hpp" +#include "common/platform.hpp" +#include "ur_interface_loader.hpp" -// This is a temporary workaround on windows, where UR adapter is teardowned -// before the UR loader, which will result in access violation when we use print -// function as the overrided print function was already released with the UR -// adapter. -// TODO: Change adapters to use a common sink class in the loader instead of -// using thier own sink class that inherit from logger::Sink. -class ur_legacy_sink : public logger::Sink { -public: - ur_legacy_sink(std::string logger_name = "", bool skip_prefix = true) - : Sink(std::move(logger_name), skip_prefix) { - this->ostream = &std::cerr; - } +namespace ur::level_zero::v1 { - virtual void print([[maybe_unused]] ur_logger_level_t level, - const std::string &msg) override { - fprintf(stderr, "%s", msg.c_str()); - } -}; +ur_result_t urAdapterGet( + /// [in] the number of platforms to be added to phAdapters. If phAdapters is + /// not NULL, then NumEntries should be greater than zero, otherwise + /// ::UR_RESULT_ERROR_INVALID_SIZE, will be returned. + uint32_t NumEntries, + /// [out][optional][range(0, NumEntries)] array of handle of adapters. + /// If NumEntries is less than the number of adapters available, then + /// ::urAdapterGet shall only retrieve that number of platforms. + ::ur_adapter_handle_t *AdaptersOpque, + /// [out][optional] returns the total number of adapters available. + uint32_t *NumAdapters) try { + std::lock_guard Lock(GlobalAdapterMutex); -// Find the corresponding ZesDevice Handle for a given ZeDevice -ur_result_t getZesDeviceHandle(ur_adapter_handle_t_ *adapter, - zes_uuid_t coreDeviceUuid, - zes_device_handle_t *ZesDevice, - uint32_t *SubDeviceId, ze_bool_t *SubDevice) { - uint32_t ZesDriverCount = 0; - std::vector ZesDrivers; - std::vector ZesDevices; - ze_result_t ZesResult = ZE_RESULT_ERROR_INVALID_ARGUMENT; - ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, (&ZesDriverCount, nullptr)); - ZesDrivers.resize(ZesDriverCount); - ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, - (&ZesDriverCount, ZesDrivers.data())); - for (uint32_t I = 0; I < ZesDriverCount; ++I) { - ZesResult = ZE_CALL_NOCHECK( - adapter->getDeviceByUUIdFunctionPtr, - (ZesDrivers[I], coreDeviceUuid, ZesDevice, SubDevice, SubDeviceId)); - if (ZesResult == ZE_RESULT_SUCCESS) { + // L0v2 adapter may have already been selected. Return adapter only if it + // carries our DDI. If the GlobalAdapter is not set, then create a new one + // and return it. + if (GlobalAdapter) { + if (GlobalAdapter->ddi_table != ddi_getter::value()) { + if (NumAdapters) + *NumAdapters = 0; return UR_RESULT_SUCCESS; } - } - return UR_RESULT_ERROR_INVALID_ARGUMENT; -} - -ur_result_t checkDeviceIntelGPUIpVersionOrNewer(uint32_t ipVersion) { - uint32_t ZeDriverCount = 0; - ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, nullptr)); - if (ZeDriverCount == 0) { - return UR_RESULT_SUCCESS; - } - - std::vector ZeDrivers; - std::vector ZeDevices; - ZeDrivers.resize(ZeDriverCount); - - ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, ZeDrivers.data())); - for (uint32_t I = 0; I < ZeDriverCount; ++I) { - ze_device_properties_t device_properties{}; - ze_device_ip_version_ext_t ipVersionExt{}; - ipVersionExt.stype = ZE_STRUCTURE_TYPE_DEVICE_IP_VERSION_EXT; - ipVersionExt.pNext = nullptr; - device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - device_properties.pNext = &ipVersionExt; - uint32_t ZeDeviceCount = 0; - ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, nullptr)); - ZeDevices.resize(ZeDeviceCount); - ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, ZeDevices.data())); - // Check if this driver has GPU Devices that have this IP Version or newer. - for (uint32_t D = 0; D < ZeDeviceCount; ++D) { - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevices[D], &device_properties)); - if (device_properties.type == ZE_DEVICE_TYPE_GPU && - device_properties.vendorId == 0x8086) { - ze_device_ip_version_ext_t *ipVersionExt = - (ze_device_ip_version_ext_t *)device_properties.pNext; - if (ipVersionExt->ipVersion >= ipVersion) { - return UR_RESULT_SUCCESS; - } - } + if (NumEntries && AdaptersOpque) { + *AdaptersOpque = common_cast(GlobalAdapter); + GlobalAdapter->RefCount.retain(); } - } - return UR_RESULT_ERROR_UNSUPPORTED_VERSION; -} - -/** - * @brief Initializes the platforms by querying Level Zero drivers and devices. - * - * This function initializes the platforms by querying the available Level Zero - * drivers and devices. It handles different behaviors based on the presence of - * drivers obtained through `zeDriverGet` and initialized drivers through - * `zeInitDrivers`. - * - * @param platforms A vector to store the initialized platform handles. - * @param ZesResult The result of a previous ZES (Level Zero System) operation. - * @return ur_result_t The result of the initialization process. - * - * The function performs the following steps: - * 1. Queries the number of Level Zero drivers using `zeDriverGet`. - * 2. If drivers are found, it retrieves their handles. - * 3. If no drivers are found in either `zeInitDrivers` or `zeDriverGet`, - * it logs a message and returns success. - * 4. If `zeInitDrivers` is supported by the global adapter, it retrieves - * their handles and properties. - * 5. It compares the drivers obtained from `zeDriverGet` and `zeInitDrivers`, - * adding unique drivers to the list. - * 6. If `zeInitDrivers` is not supported, it uses the drivers obtained - * from `zeDriverGet`. - * 7. For each driver, it queries the devices and checks if they are GPU - * devices. - * 8. If a GPU device is found, it initializes a platform for the driver and - * adds it to the platforms vector. - * 9. If ZES operations are successful, it populates the ZES/ZE device mapping - * for the devices into the platform. - * 10. The function handles exceptions and returns the appropriate result. - */ -ur_result_t initPlatforms(ur_adapter_handle_t_ *adapter, PlatformVec &platforms, - ze_result_t ZesResult) noexcept try { - std::vector ZeDrivers; - std::vector ZeDriverGetHandles; - std::vector ZeInitDriversHandles; - std::vector ZeDevices; - uint32_t ZeDriverCount = 0; - uint32_t ZeDriverGetCount = 0; - - auto ZeDriverGetResult = - ZE_CALL_NOCHECK(zeDriverGet, (&ZeDriverGetCount, nullptr)); - if (ZeDriverGetCount > 0 && ZeDriverGetResult == ZE_RESULT_SUCCESS) { - ZeDriverGetHandles.resize(ZeDriverGetCount); - ZE2UR_CALL(zeDriverGet, (&ZeDriverGetCount, ZeDriverGetHandles.data())); - } - if (ZeDriverGetCount == 0 && adapter->ZeInitDriversCount == 0) { - UR_LOG(ERR, "\nNo Valid L0 Drivers found.\n"); + if (NumAdapters) + *NumAdapters = 1; return UR_RESULT_SUCCESS; } - if (adapter->InitDriversSupported) { - ZeInitDriversHandles.resize(adapter->ZeInitDriversCount); - ZeDrivers.resize(adapter->ZeInitDriversCount); - ZE2UR_CALL(adapter->initDriversFunctionPtr, - (&adapter->ZeInitDriversCount, ZeInitDriversHandles.data(), - &adapter->InitDriversDesc)); - ZeDrivers.assign(ZeInitDriversHandles.begin(), ZeInitDriversHandles.end()); - if (ZeDriverGetCount > 0 && adapter->ZeInitDriversCount > 0) { - for (uint32_t X = 0; X < adapter->ZeInitDriversCount; ++X) { - // zeDriverGet and zeInitDrivers can return the driver handles in - // reverse order based on driver ordering causing an issue if the - // drivers are expected to be in the same order. To resolve, this loop - // checks if the driver from zeDriverGet already exists in zeInitDrivers - // and only adds it if it is not found. - bool unMatchedDriverHandle = false; - ze_driver_handle_t driverGetHandle = nullptr; - for (uint32_t Y = 0; Y < ZeDriverGetCount; ++Y) { - unMatchedDriverHandle = true; - ZeStruct ZeDriverGetProperties; - ZeStruct ZeInitDriverProperties; - ZE2UR_CALL(zeDriverGetProperties, - (ZeDriverGetHandles[Y], &ZeDriverGetProperties)); - ZE2UR_CALL(zeDriverGetProperties, - (ZeInitDriversHandles[X], &ZeInitDriverProperties)); - driverGetHandle = ZeDriverGetHandles[Y]; - // If zeDriverGet driver is the same version as zeInitDriver driver, - // then do not add it again. - if (ZeDriverGetProperties.driverVersion == - ZeInitDriverProperties.driverVersion) { - UR_LOG(DEBUG, - "zeDriverHandle {} matched between zeDriverGet and " - "zeInitDrivers. Not adding duplicate driver to list", - ZeDriverGetHandles[Y]); - unMatchedDriverHandle = false; - break; - } - } - if (unMatchedDriverHandle) { - UR_LOG(DEBUG, - "zeDriverHandle {} not found in zeInitDrivers. Adding to " - "driver list.", - driverGetHandle); - ZeDrivers.push_back(driverGetHandle); - } - } - } - } else { - ZeDrivers.resize(ZeDriverGetCount); - ZeDrivers.assign(ZeDriverGetHandles.begin(), ZeDriverGetHandles.end()); - } - ZeDriverCount = ZeDrivers.size(); - UR_LOG(DEBUG, "{} L0 Drivers found.", ZeDriverCount); - for (uint32_t I = 0; I < ZeDriverCount; ++I) { - // Keep track of the first platform init for this Driver - bool DriverPlatformInit = false; - ze_device_properties_t device_properties{}; - device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - uint32_t ZeDeviceCount = 0; - ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, nullptr)); - ZeDevices.resize(ZeDeviceCount); - ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, ZeDevices.data())); - auto platform = std::make_unique(ZeDrivers[I]); - // Check if this driver has GPU Devices - for (uint32_t D = 0; D < ZeDeviceCount; ++D) { - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevices[D], &device_properties)); - if (ZE_DEVICE_TYPE_GPU == device_properties.type) { - // Check if this driver's platform has already been init. - if (!DriverPlatformInit) { - // If this Driver is a GPU, save it as a usable platform. - ur_result_t Result; - UR_CALL_NOCHECK(Result = platform->initialize()); - - // Forget platform with L0 driver version listed in the skiplist - if (platform->IsDriverVersionSkipListed) { - break; - } - - if (Result != UR_RESULT_SUCCESS) { - return Result; - } - - // Save a copy in the cache for future uses. - platforms.push_back(std::move(platform)); - // Mark this driver's platform as init to prevent additional platforms - // from being created per driver. - DriverPlatformInit = true; - } - if (ZesResult == ZE_RESULT_SUCCESS) { - // Populate the Zes/Ze device mapping for this Ze Device into the last - // added platform which represents the current driver being queried. - ur_zes_device_handle_data_t ZesDeviceData; - zes_uuid_t ZesUUID; - std::memcpy(&ZesUUID, &device_properties.uuid, sizeof(zes_uuid_t)); - if (getZesDeviceHandle(adapter, ZesUUID, &ZesDeviceData.ZesDevice, - &ZesDeviceData.SubDeviceId, - &ZesDeviceData.SubDevice) == - UR_RESULT_SUCCESS) { - platforms.back()->ZedeviceToZesDeviceMap.insert( - std::make_pair(ZeDevices[D], std::move(ZesDeviceData))); - } - } - } - } - } - return UR_RESULT_SUCCESS; -} catch (...) { - return exceptionToResult(std::current_exception()); -} - -ur_result_t adapterStateInit() { - -#if defined(_WIN32) - umfInit(); -#endif - - return UR_RESULT_SUCCESS; -} + auto *adapter = new ur_adapter_handle_t_(); + if (adapter->Platforms.empty()) { + delete adapter; + releaseStaticLoaderResourcesIfNeeded(); -static bool isBMGorNewer() { - auto urResult = checkDeviceIntelGPUIpVersionOrNewer(0x05004000); - if (urResult != UR_RESULT_SUCCESS && - urResult != UR_RESULT_ERROR_UNSUPPORTED_VERSION) { - UR_LOG(ERR, "Intel GPU IP Version check failed: {}\n", urResult); - throw urResult; - } - - return urResult == UR_RESULT_SUCCESS; -} - -static void releaseStaticLoaderResourcesIfNeeded() { -#ifdef UR_STATIC_LEVEL_ZERO - // Given static linking of the L0 Loader, we must release the static loader - // resources if the adapter is not used. - zelLoaderContextTeardown(); -#endif -} - -// returns a pair indicating whether to use the V1 adapter and a string -// indicating the reason for the decision. -static std::pair shouldUseV1Adapter() { - auto specificAdapterVersionRequested = - ur_getenv("UR_LOADER_USE_LEVEL_ZERO_V2").has_value() || - ur_getenv("SYCL_UR_USE_LEVEL_ZERO_V2").has_value(); - - auto v2Requested = getenv_tobool("UR_LOADER_USE_LEVEL_ZERO_V2", false); - v2Requested |= getenv_tobool("SYCL_UR_USE_LEVEL_ZERO_V2", false); - - std::string reason = - specificAdapterVersionRequested - ? "Specific adapter version requested by UR_LOADER_USE_LEVEL_ZERO_V2 " - "or SYCL_UR_USE_LEVEL_ZERO_V2" - : "Using default adapter version based on device IP version"; - - if (v2Requested) { - return {false, reason}; - } - - if (!v2Requested && specificAdapterVersionRequested) { - // v1 specifically requested - return {true, reason}; - } - - // default: only enable for devices older than BMG - return {!isBMGorNewer(), reason}; -} - -[[maybe_unused]] static std::pair shouldUseV2Adapter() { - auto [useV1, reason] = shouldUseV1Adapter(); - return {!useV1, reason}; -} - -/* -This constructor initializes the `ur_adapter_handle_t_` object and -sets up the environment for Level Zero (L0) initialization. -The behavior of the initialization process is influenced by two -environment variables: -`UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` and `UR_L0_ENABLE_ZESINIT_DEFAULT`. - -| Environment Variable | Value | Behavior | -|--------------------------------|-------|----------------------------| -| UR_L0_ENABLE_SYSMAN_ENV_DEFAULT| 1 | Enables the default SysMan | -| | | environment initialization | -| | | by setting | -| | | `ZES_ENABLE_SYSMAN` to "1".| -| | 0 | Disables the default SysMan| -| | | environment initialization.| -| | unset | Defaults to 1, enabling the| -| | | SysMan environment | -| | | initialization. | -| UR_L0_ENABLE_ZESINIT_DEFAULT | 1 | Enables the default SysMan | -| | | initialization by loading | -| | | SysMan-related functions | -| | | and calling `zesInit`. | -| | 0 | Disables the default SysMan| -| | | initialization with zesInit| -| | unset | Defaults to 0, disabling | -| | | the SysMan initialization | -| | | thru zesInit. | - -Behavior Summary: -- If `UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` is set to 1 or is unset, - `ZES_ENABLE_SYSMAN` is set to "1". -- If `UR_L0_ENABLE_ZESINIT_DEFAULT` is set to 1 and - `UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` is not set to 1, - SysMan-related functions are loaded and `zesInit` is called. -- If `UR_L0_ENABLE_ZESINIT_DEFAULT` is set to 0 or is unset, - SysMan initialization is skipped. -*/ -ur_adapter_handle_t_::ur_adapter_handle_t_() - : handle_base(), logger(logger::get_logger("level_zero")), RefCount(0) { - auto ZeInitDriversResult = ZE_RESULT_ERROR_UNINITIALIZED; - auto ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; - auto ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; - -#ifdef UR_STATIC_LEVEL_ZERO - // Given static linking of the L0 Loader, we must delay the loader's - // destruction of its context until after the UR Adapter is destroyed. - zelSetDelayLoaderContextTeardown(); -#endif - - if (UrL0Debug & UR_L0_DEBUG_BASIC) { - logger.setLegacySink(std::make_unique()); - setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1"); - setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace"); - setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1"); - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - }; - - if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); - } - - if (UrL0LeaksDebug) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); - } - - uint32_t UserForcedSysManInit = 0; - // Check if the user has disabled the default L0 Env initialization. - const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { - const char *UrRet = std::getenv("UR_L0_ENABLE_SYSMAN_ENV_DEFAULT"); - if (!UrRet) - return 1; - UserForcedSysManInit &= 1; - return std::atoi(UrRet); - }(); - - // Dynamically load the new L0 apis separately. - // This must be done to avoid attempting to use symbols that do - // not exist in older loader runtimes. -#ifndef UR_STATIC_LEVEL_ZERO -#ifdef _WIN32 - processHandle = GetModuleHandle(NULL); -#else - processHandle = nullptr; -#endif -#endif - - // Setting these environment variables before running zeInit will enable - // the validation layer in the Level Zero loader. - if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); - } - - if (getenv("SYCL_ENABLE_PCI") != nullptr) { - UR_LOG(WARN, - "WARNING: SYCL_ENABLE_PCI is deprecated and no longer needed.\n"); - } - - // TODO: We can still safely recover if something goes wrong during the - // init. Implement handling segfault using sigaction. - - // We must only initialize the driver once, even if urPlatformGet() is - // called multiple times. Declaring the return value as "static" ensures - // it's only called once. - - // Set ZES_ENABLE_SYSMAN by default if the user has not set it. - if (UrSysManEnvInitEnabled) { - setEnvVar("ZES_ENABLE_SYSMAN", "1"); - } - - // Init with all flags set to enable for all driver types to be init in - // the application. - ze_init_flags_t L0InitFlags = ZE_INIT_FLAG_GPU_ONLY; - if (UrL0InitAllDrivers) { - L0InitFlags |= ZE_INIT_FLAG_VPU_ONLY; - } - UR_LOG(DEBUG, "zeInit with flags value of {}", static_cast(L0InitFlags)); - ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); - if (ZeInitResult != ZE_RESULT_SUCCESS) { - const char *ErrorString = "Unknown"; - zeParseError(ZeInitResult, ErrorString); - UR_LOG(ERR, "zeInit failed with {}", ErrorString); - } - - bool useInitDrivers = false; - zel_version_t loader_version = {}; - size_t num_components; - auto result = zelLoaderGetVersions(&num_components, nullptr); - if (result == ZE_RESULT_SUCCESS) { - zel_component_version_t *versions = - new zel_component_version_t[num_components]; - result = zelLoaderGetVersions(&num_components, versions); - if (result == ZE_RESULT_SUCCESS) { - for (size_t i = 0; i < num_components; ++i) { - if (strncmp(versions[i].component_name, "loader", strlen("loader")) == - 0) { - loader_version = versions[i].component_lib_version; - UR_LOG(DEBUG, "Level Zero Loader version: {}.{}.{}", - loader_version.major, loader_version.minor, - loader_version.patch); - break; - } - } - } - delete[] versions; - - auto LoaderSkiplist = getenv_to_vec("SYCL_UR_L0_LOADER_SKIPLIST"); - if (!LoaderSkiplist.has_value()) { - LoaderSkiplist = LoaderSkiplistDefault; - - if ((loader_version.major == 1 && loader_version.minor < 8)) { - UR_LOG(ERR, - "ERROR: Level Zero Loader version older than 1.8.0 is not " - "supported (current version: {}.{}.{}). Please update Intel GPU " - "compute drivers.", - loader_version.major, loader_version.minor, - loader_version.patch); - // Return 0 platforms as the loader is too old to be supported. - return; - } - } - - auto LoaderVersionString = std::to_string(loader_version.major) + "." + - std::to_string(loader_version.minor) + "." + - std::to_string(loader_version.patch); - for (const auto &version : LoaderSkiplist.value()) { - UR_LOG(DEBUG, "Checking loader version {} against skiplist entry {}", - LoaderVersionString, version); - if (version == LoaderVersionString) { - UR_LOG(WARN, - "Skipping Level Zero adapter due to the loader version: {} " - "matches a skiplist entry (set SYCL_UR_L0_LOADER_SKIPLIST to " - "override)", - version); - // Return 0 platforms - return; - } - } - - if (loader_version.major > 1 || - (loader_version.major == 1 && loader_version.minor > 19) || - (loader_version.major == 1 && loader_version.minor == 19 && - loader_version.patch >= 2)) { - useInitDrivers = true; - } - - if ((loader_version.major == 1 && loader_version.minor < 21) || - (loader_version.major == 1 && loader_version.minor == 21 && - loader_version.patch < 2)) { - UR_LOG(WARN, - "WARNING: Level Zero Loader version is older than 1.21.2. " - "Please update to the latest version for API logging support."); - } - } - - if (useInitDrivers) { -#ifdef UR_STATIC_LEVEL_ZERO - initDriversFunctionPtr = zeInitDrivers; -#else - initDriversFunctionPtr = - (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zeInitDrivers"); -#endif - if (initDriversFunctionPtr) { - UR_LOG(DEBUG, "zeInitDrivers with flags value of {}", - static_cast(InitDriversDesc.flags)); - ZeInitDriversResult = - ZE_CALL_NOCHECK(initDriversFunctionPtr, - (&ZeInitDriversCount, nullptr, &InitDriversDesc)); - if (ZeInitDriversResult == ZE_RESULT_SUCCESS) { - InitDriversSupported = true; - } else { - const char *ErrorString = "Unknown"; - zeParseError(ZeInitDriversResult, ErrorString); - UR_LOG(ERR, "zeInitDrivers failed with {}", ErrorString); - } - } - } - - if (ZeInitResult != ZE_RESULT_SUCCESS && - ZeInitDriversResult != ZE_RESULT_SUCCESS) { - // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. - UR_LOG(ERR, "Level Zero Uninitialized"); - return; + if (NumAdapters) + *NumAdapters = 0; + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } - PlatformVec platforms; - - bool forceLoadedAdapter = ur_getenv("UR_ADAPTERS_FORCE_LOAD").has_value(); - if (!forceLoadedAdapter) { -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - auto [useV2, reason] = shouldUseV2Adapter(); - if (!useV2) { - UR_LOG(INFO, "Skipping L0 V2 adapter: {}", reason); - releaseStaticLoaderResourcesIfNeeded(); - return; - } -#else + if (!ur_getenv("UR_ADAPTERS_FORCE_LOAD").has_value()) { auto [useV1, reason] = shouldUseV1Adapter(); if (!useV1) { UR_LOG(INFO, "Skipping L0 V1 adapter: {}", reason); + delete adapter; +#ifndef UR_STATIC_ADAPTER_LEVEL_ZERO releaseStaticLoaderResourcesIfNeeded(); - return; - } #endif - } - - // Check if the user has enabled the default L0 SysMan initialization. - const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { - const char *UrRet = std::getenv("UR_L0_ENABLE_ZESINIT_DEFAULT"); - if (!UrRet) - return 0; - UserForcedSysManInit &= 2; - return std::atoi(UrRet); - }(); - - bool ZesInitNeeded = UrSysmanZesinitEnable && !UrSysManEnvInitEnabled; - // Unless the user has forced the SysMan init, we will check the device - // version to see if the zesInit is needed. - if (UserForcedSysManInit == 0 && - checkDeviceIntelGPUIpVersionOrNewer(0x05004000) == UR_RESULT_SUCCESS) { - if (UrSysManEnvInitEnabled) { - setEnvVar("ZES_ENABLE_SYSMAN", "0"); + if (NumAdapters) + *NumAdapters = 0; + return UR_RESULT_SUCCESS; } - ZesInitNeeded = true; - } - if (ZesInitNeeded) { -#ifdef UR_STATIC_LEVEL_ZERO - getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp; - getSysManDriversFunctionPtr = zesDriverGet; - sysManInitFunctionPtr = zesInit; -#else - getDeviceByUUIdFunctionPtr = - (zes_pfnDriverGetDeviceByUuidExp_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zesDriverGetDeviceByUuidExp"); - getSysManDriversFunctionPtr = - (zes_pfnDriverGet_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zesDriverGet"); - sysManInitFunctionPtr = (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zesInit"); -#endif } - if (getDeviceByUUIdFunctionPtr && getSysManDriversFunctionPtr && - sysManInitFunctionPtr) { - ze_init_flags_t L0ZesInitFlags = 0; - UR_LOG(DEBUG, "\nzesInit with flags value of {}\n", - static_cast(L0ZesInitFlags)); - ZesResult = ZE_CALL_NOCHECK(sysManInitFunctionPtr, (L0ZesInitFlags)); - } else { - ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; - } - - ur_result_t err = initPlatforms(this, platforms, ZesResult); - if (err == UR_RESULT_SUCCESS) { - UR_LOG(DEBUG, "Initialized {} platforms", platforms.size()); - Platforms = std::move(platforms); - } else { - UR_LOG(ERR, "Failed to initialize Platforms"); - throw err; - } -} -void globalAdapterOnDemandCleanup() { - if (GlobalAdapter) { - delete GlobalAdapter; - } -} + // Stamp L0v1 DDI table on the created adapter and its platforms. + adapter->ddi_table = ddi_getter::value(); + for (auto &p : adapter->Platforms) + p->ddi_table = adapter->ddi_table; + GlobalAdapter = adapter; -ur_result_t adapterStateTeardown() { - // Due to multiple DLLMain definitions with SYCL, register to cleanup the - // Global Adapter after refcnt is 0 #if defined(_WIN32) - umfTearDown(); - std::atexit(globalAdapterOnDemandCleanup); + // On Windows the adapter library is torn down before the UR loader, + // so register an atexit cleanup that frees GlobalAdapter ourselves. + std::atexit(ur::level_zero::globalAdapterOnDemandCleanup); #endif - return UR_RESULT_SUCCESS; -} - -namespace ur::level_zero { -ur_result_t urAdapterGet( - /// [in] the number of platforms to be added to phAdapters. If phAdapters is - /// not NULL, then NumEntries should be greater than zero, otherwise - /// ::UR_RESULT_ERROR_INVALID_SIZE, will be returned. - uint32_t NumEntries, - /// [out][optional][range(0, NumEntries)] array of handle of adapters. - /// If NumEntries is less than the number of adapters available, then - /// ::urAdapterGet shall only retrieve that number of platforms. - ur_adapter_handle_t *Adapters, - /// [out][optional] returns the total number of adapters available. - uint32_t *NumAdapters) try { - static std::mutex AdapterConstructionMutex{}; - - // We need to initialize the adapter even if user only queries - // the number of adapters to decided whether to use V1 or V2. - std::lock_guard Lock{AdapterConstructionMutex}; - - if (!GlobalAdapter) { - GlobalAdapter = new ur_adapter_handle_t_(); - } - - if (GlobalAdapter->Platforms.size() == 0) { - if (NumAdapters) { - *NumAdapters = 0; - } - return UR_RESULT_ERROR_UNSUPPORTED_VERSION; - } - - if (NumEntries && Adapters) { - *Adapters = GlobalAdapter; - + if (NumEntries && AdaptersOpque) { + *AdaptersOpque = common_cast(GlobalAdapter); if (GlobalAdapter->RefCount.retain() == 0) { adapterStateInit(); } } - if (NumAdapters) { + if (NumAdapters) *NumAdapters = 1; - } return UR_RESULT_SUCCESS; } catch (ur_result_t result) { @@ -700,94 +104,4 @@ ur_result_t urAdapterGet( return UR_RESULT_ERROR_UNKNOWN; } -ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { - assert(GlobalAdapter && GlobalAdapter == Adapter); - - // NOTE: This does not require guarding with a mutex; the instant the ref - // count hits zero, both Get and Retain are UB. - if (GlobalAdapter->RefCount.release()) { - auto result = adapterStateTeardown(); -#ifdef UR_STATIC_LEVEL_ZERO - // Given static linking of the L0 Loader, we must delay the loader's - // destruction of its context until after the UR Adapter is destroyed. - zelLoaderContextTeardown(); -#endif - - delete GlobalAdapter; - GlobalAdapter = nullptr; - - return result; - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) { - assert(GlobalAdapter && GlobalAdapter == Adapter); - GlobalAdapter->RefCount.retain(); - - return UR_RESULT_SUCCESS; -} - -ur_result_t urAdapterGetLastError( - /// [in] handle of the platform instance - ur_adapter_handle_t, - /// [out] pointer to a C string where the adapter specific error message - /// will be stored. - const char **Message, - /// [out] pointer to an integer where the adapter specific error code will - /// be stored. - int32_t *Error) { - *Message = ErrorMessage; - *Error = ErrorAdapterNativeCode; - - return UR_RESULT_SUCCESS; -} - -ur_result_t urAdapterGetInfo(ur_adapter_handle_t, ur_adapter_info_t PropName, - size_t PropSize, void *PropValue, - size_t *PropSizeRet) { - UrReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet); - - switch (PropName) { - case UR_ADAPTER_INFO_BACKEND: - return ReturnValue(UR_BACKEND_LEVEL_ZERO); - case UR_ADAPTER_INFO_REFERENCE_COUNT: - return ReturnValue(GlobalAdapter->RefCount.getCount()); - case UR_ADAPTER_INFO_VERSION: { -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - uint32_t adapterVersion = 2; -#else - uint32_t adapterVersion = 1; -#endif - return ReturnValue(adapterVersion); - } - default: - return UR_RESULT_ERROR_INVALID_ENUMERATION; - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urAdapterSetLoggerCallback( - ur_adapter_handle_t, ur_logger_callback_t pfnLoggerCallback, - void *pUserData, ur_logger_level_t level = UR_LOGGER_LEVEL_QUIET) { - - if (GlobalAdapter) { - GlobalAdapter->logger.setCallbackSink(pfnLoggerCallback, pUserData, level); - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urAdapterSetLoggerCallbackLevel(ur_adapter_handle_t, - ur_logger_level_t level) { - - if (GlobalAdapter) { - GlobalAdapter->logger.setCallbackLevel(level); - } - - return UR_RESULT_SUCCESS; -} - -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/async_alloc.cpp b/unified-runtime/source/adapters/level_zero/async_alloc.cpp index 64f7385977835..754752a53d7dc 100644 --- a/unified-runtime/source/adapters/level_zero/async_alloc.cpp +++ b/unified-runtime/source/adapters/level_zero/async_alloc.cpp @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// +#include "common/enqueued_pool.hpp" #include "context.hpp" -#include "enqueued_pool.hpp" #include "event.hpp" #include "logger/ur_logger.hpp" @@ -16,7 +16,7 @@ #include #include -namespace ur::level_zero { +namespace ur::level_zero::v1 { static ur_result_t enqueueUSMAllocHelper( ur_queue_handle_t Queue, ur_usm_pool_handle_t Pool, const size_t Size, @@ -115,110 +115,121 @@ static ur_result_t enqueueUSMAllocHelper( } ur_result_t urEnqueueUSMDeviceAllocExp( - ur_queue_handle_t Queue, ///< [in] handle of the queue object - ur_usm_pool_handle_t Pool, ///< [in][optional] USM pool descriptor + ::ur_queue_handle_t QueueOpque, ///< [in] handle of the queue object + ::ur_usm_pool_handle_t PoolOpque, ///< [in][optional] USM pool descriptor const size_t Size, ///< [in] minimum size in bytes of the USM memory object ///< to be allocated const ur_exp_async_usm_alloc_properties_t *Properties, ///< [in][optional] pointer to the enqueue asynchronous ///< USM allocation properties uint32_t NumEventsInWaitList, ///< [in] size of the event wait list - const ur_event_handle_t - *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)] - ///< pointer to a list of events that must be complete - ///< before the kernel execution. If nullptr, the - ///< numEventsInWaitList must be 0, indicating no wait - ///< events. - void **Mem, ///< [out] pointer to USM memory object - ur_event_handle_t *OutEvent ///< [out][optional] return an event object that - ///< identifies the async alloc + const ::ur_event_handle_t + *EventWaitListOpque, ///< [in][optional][range(0, numEventsInWaitList)] + ///< pointer to a list of events that must be + ///< complete before the kernel execution. If + ///< nullptr, the numEventsInWaitList must be 0, + ///< indicating no wait events. + void **Mem, ///< [out] pointer to USM memory object + ::ur_event_handle_t + *OutEventOpque ///< [out][optional] return an event + ///< object that identifies the async alloc ) { - return enqueueUSMAllocHelper(Queue, Pool, Size, Properties, - NumEventsInWaitList, EventWaitList, Mem, - OutEvent, UR_USM_TYPE_DEVICE); + return enqueueUSMAllocHelper(v1_cast(QueueOpque), v1_cast(PoolOpque), Size, + Properties, NumEventsInWaitList, + v1_cast(EventWaitListOpque), Mem, + v1_cast(OutEventOpque), UR_USM_TYPE_DEVICE); } ur_result_t urEnqueueUSMSharedAllocExp( - ur_queue_handle_t Queue, ///< [in] handle of the queue object - ur_usm_pool_handle_t Pool, ///< [in][optional] USM pool descriptor + ::ur_queue_handle_t QueueOpque, ///< [in] handle of the queue object + ::ur_usm_pool_handle_t PoolOpque, ///< [in][optional] USM pool descriptor const size_t Size, ///< [in] minimum size in bytes of the USM memory object ///< to be allocated const ur_exp_async_usm_alloc_properties_t *Properties, ///< [in][optional] pointer to the enqueue asynchronous ///< USM allocation properties uint32_t NumEventsInWaitList, ///< [in] size of the event wait list - const ur_event_handle_t - *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)] - ///< pointer to a list of events that must be complete - ///< before the kernel execution. If nullptr, the - ///< numEventsInWaitList must be 0, indicating no wait - ///< events. - void **Mem, ///< [out] pointer to USM memory object - ur_event_handle_t *OutEvent ///< [out][optional] return an event object that - ///< identifies the async alloc + const ::ur_event_handle_t + *EventWaitListOpque, ///< [in][optional][range(0, numEventsInWaitList)] + ///< pointer to a list of events that must be + ///< complete before the kernel execution. If + ///< nullptr, the numEventsInWaitList must be 0, + ///< indicating no wait events. + void **Mem, ///< [out] pointer to USM memory object + ::ur_event_handle_t + *OutEventOpque ///< [out][optional] return an event + ///< object that identifies the async alloc ) { - return enqueueUSMAllocHelper(Queue, Pool, Size, Properties, - NumEventsInWaitList, EventWaitList, Mem, - OutEvent, UR_USM_TYPE_SHARED); + return enqueueUSMAllocHelper(v1_cast(QueueOpque), v1_cast(PoolOpque), Size, + Properties, NumEventsInWaitList, + v1_cast(EventWaitListOpque), Mem, + v1_cast(OutEventOpque), UR_USM_TYPE_SHARED); } ur_result_t urEnqueueUSMHostAllocExp( - ur_queue_handle_t Queue, ///< [in] handle of the queue object - ur_usm_pool_handle_t Pool, ///< [in][optional] handle of the USM memory pool + ::ur_queue_handle_t QueueOpque, ///< [in] handle of the queue object + ::ur_usm_pool_handle_t + PoolOpque, ///< [in][optional] handle of the USM memory pool const size_t Size, ///< [in] minimum size in bytes of the USM memory object ///< to be allocated const ur_exp_async_usm_alloc_properties_t *Properties, ///< [in][optional] pointer to the enqueue asynchronous ///< USM allocation properties uint32_t NumEventsInWaitList, ///< [in] size of the event wait list - const ur_event_handle_t - *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)] - ///< pointer to a list of events that must be complete - ///< before the kernel execution. If nullptr, the - ///< numEventsInWaitList must be 0, indicating no wait - ///< events. - void **Mem, ///< [out] pointer to USM memory object - ur_event_handle_t - *OutEvent ///< [out][optional] return an event object that identifies - ///< the asynchronous USM device allocation + const ::ur_event_handle_t + *EventWaitListOpque, ///< [in][optional][range(0, numEventsInWaitList)] + ///< pointer to a list of events that must be + ///< complete before the kernel execution. If + ///< nullptr, the numEventsInWaitList must be 0, + ///< indicating no wait events. + void **Mem, ///< [out] pointer to USM memory object + ::ur_event_handle_t + *OutEventOpque ///< [out][optional] return an event object that + ///< identifies the asynchronous USM device allocation ) { - return enqueueUSMAllocHelper(Queue, Pool, Size, Properties, - NumEventsInWaitList, EventWaitList, Mem, - OutEvent, UR_USM_TYPE_HOST); + return enqueueUSMAllocHelper(v1_cast(QueueOpque), v1_cast(PoolOpque), Size, + Properties, NumEventsInWaitList, + v1_cast(EventWaitListOpque), Mem, + v1_cast(OutEventOpque), UR_USM_TYPE_HOST); } ur_result_t urEnqueueUSMFreeExp( - ur_queue_handle_t Queue, ///< [in] handle of the queue object - ur_usm_pool_handle_t, ///< [in][optional] USM pool descriptor - void *Mem, ///< [in] pointer to USM memory object - uint32_t NumEventsInWaitList, ///< [in] size of the event wait list - const ur_event_handle_t - *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)] - ///< pointer to a list of events that must be complete - ///< before the kernel execution. If nullptr, the - ///< numEventsInWaitList must be 0, indicating no wait - ///< events. - ur_event_handle_t *OutEvent ///< [out][optional] return an event object that - ///< identifies the async alloc + ::ur_queue_handle_t QueueOpque, ///< [in] handle of the queue object + ::ur_usm_pool_handle_t, ///< [in][optional] USM pool descriptor + void *Mem, ///< [in] pointer to USM memory object + uint32_t NumEventsInWaitList, ///< [in] size of the event wait list + const ::ur_event_handle_t + *EventWaitListOpque, ///< [in][optional][range(0, numEventsInWaitList)] + ///< pointer to a list of events that must be + ///< complete before the kernel execution. If + ///< nullptr, the numEventsInWaitList must be 0, + ///< indicating no wait events. + ::ur_event_handle_t + *OutEventOpque ///< [out][optional] return an event + ///< object that identifies the async alloc ) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitListOpque); + auto OutEventInternal = v1_cast(OutEventOpque); std::scoped_lock lock(Queue->Mutex); bool UseCopyEngine = false; ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); bool OkToBatch = false; // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, - OkToBatch, nullptr /*ForcedCmdQueue*/)); + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent{}; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_ENQUEUE_USM_FREE_EXP, @@ -252,7 +263,7 @@ ur_result_t urEnqueueUSMFreeExp( } (*Event)->RefCount.retain(); - UsmPool->AsyncPool.insert(Mem, Size, *Event, Queue); + UsmPool->AsyncPool.insert(Mem, Size, v1_cast(*Event), Queue); // Signal that USM free event was finished ZE2UR_CALL(zeCommandListAppendSignalEvent, (ZeCommandList, ZeEvent)); @@ -261,4 +272,4 @@ ur_result_t urEnqueueUSMFreeExp( return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index 2973b33077165..ba101696e272b 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -8,15 +8,18 @@ // //===----------------------------------------------------------------------===// #include "command_buffer.hpp" -#include "command_buffer_command.hpp" -#include "helpers/kernel_helpers.hpp" -#include "helpers/mutable_helpers.hpp" +#include "common/command_buffer_command.hpp" +#include "common/device.hpp" +#include "common/helpers/kernel_helpers.hpp" +#include "common/helpers/mutable_helpers.hpp" #include "logger/ur_logger.hpp" #include "unified-runtime/ur_api.h" #include "ur/ur.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" +namespace ur::level_zero::v1 { + /* L0 Command-buffer Extension Doc see: https://github.com/intel/llvm/blob/sycl/sycl/doc/design/CommandGraph.md#level-zero */ @@ -26,20 +29,20 @@ namespace { -ur_result_t -getMemoryAccessType(const ur_kernel_arg_mem_obj_properties_t *Properties, - ur_mem_handle_t_::access_mode_t &UrAccessMode) { - UrAccessMode = ur_mem_handle_t_::read_write; +ur_result_t getMemoryAccessType( + const ur_kernel_arg_mem_obj_properties_t *Properties, + ur::level_zero::v1::ur_mem_handle_t_::access_mode_t &UrAccessMode) { + UrAccessMode = ur::level_zero::v1::ur_mem_handle_t_::read_write; if (Properties) { switch (Properties->memoryAccess) { case UR_MEM_FLAG_READ_WRITE: - UrAccessMode = ur_mem_handle_t_::read_write; + UrAccessMode = ur::level_zero::v1::ur_mem_handle_t_::read_write; break; case UR_MEM_FLAG_WRITE_ONLY: - UrAccessMode = ur_mem_handle_t_::write_only; + UrAccessMode = ur::level_zero::v1::ur_mem_handle_t_::write_only; break; case UR_MEM_FLAG_READ_ONLY: - UrAccessMode = ur_mem_handle_t_::read_only; + UrAccessMode = ur::level_zero::v1::ur_mem_handle_t_::read_only; break; default: return UR_RESULT_ERROR_INVALID_ARGUMENT; @@ -48,20 +51,23 @@ getMemoryAccessType(const ur_kernel_arg_mem_obj_properties_t *Properties, return UR_RESULT_SUCCESS; } -ur_result_t getZeKernelWrapped(ur_kernel_handle_t Kernel, +ur_result_t getZeKernelWrapped(::ur_kernel_handle_t KernelOpque, ze_kernel_handle_t &ZeKernel, - ur_device_handle_t Device) { + ur::level_zero::ur_device_handle_t Device) { + auto Kernel = v1_cast(KernelOpque); ze_kernel_handle_t Tmp{}; UR_CALL(getZeKernel(Device->ZeDevice, Kernel, &Tmp)); ZeKernel = Tmp; return UR_RESULT_SUCCESS; } -ur_result_t getMemPtr(ur_mem_handle_t MemObj, +ur_result_t getMemPtr(::ur_mem_handle_t MemObjOpque, const ur_kernel_arg_mem_obj_properties_t *Properties, - char **&ZeHandlePtr, ur_device_handle_t Device, + char **&ZeHandlePtr, + ur::level_zero::ur_device_handle_t Device, device_ptr_storage_t *) { - ur_mem_handle_t_::access_mode_t UrAccessMode; + auto MemObj = v1_cast(MemObjOpque); + ur::level_zero::v1::ur_mem_handle_t_::access_mode_t UrAccessMode; UR_CALL(getMemoryAccessType(Properties, UrAccessMode)); if (MemObj) { @@ -464,17 +470,18 @@ ur_result_t enqueueCommandBufferFillHelper( } } // namespace -ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_( - ur_context_handle_t Context, ur_device_handle_t Device, - ze_command_list_handle_t CommandList, - ze_command_list_handle_t CommandListTranslated, - ze_command_list_handle_t CommandListResetEvents, - ze_command_list_handle_t CopyCommandList, - ur_event_handle_t ExecutionFinishedEvent, ur_event_handle_t WaitEvent, - ur_event_handle_t AllResetEvent, ur_event_handle_t CopyFinishedEvent, - ur_event_handle_t ComputeFinishedEvent, - const ur_exp_command_buffer_desc_t *Desc, const bool IsInOrderCmdList, - const bool UseImmediateAppendPath) +ur::level_zero::v1::ur_exp_command_buffer_handle_t_:: + ur_exp_command_buffer_handle_t_( + ur_context_handle_t Context, ur_device_handle_t Device, + ze_command_list_handle_t CommandList, + ze_command_list_handle_t CommandListTranslated, + ze_command_list_handle_t CommandListResetEvents, + ze_command_list_handle_t CopyCommandList, + ur_event_handle_t ExecutionFinishedEvent, ur_event_handle_t WaitEvent, + ur_event_handle_t AllResetEvent, ur_event_handle_t CopyFinishedEvent, + ur_event_handle_t ComputeFinishedEvent, + const ur_exp_command_buffer_desc_t *Desc, const bool IsInOrderCmdList, + const bool UseImmediateAppendPath) : Context(Context), Device(Device), ZeComputeCommandList(CommandList), ZeComputeCommandListTranslated(CommandListTranslated), ZeCommandListResetEvents(CommandListResetEvents), @@ -486,16 +493,17 @@ ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_( IsUpdatable(Desc->isUpdatable), IsProfilingEnabled(Desc->enableProfiling), InOrderRequested(Desc->isInOrder), IsInOrderCmdList(IsInOrderCmdList), UseImmediateAppendPath(UseImmediateAppendPath) { - UR_CALL_NOCHECK(ur::level_zero::urContextRetain(Context)); - UR_CALL_NOCHECK(ur::level_zero::urDeviceRetain(Device)); + UR_CALL_NOCHECK(ur::level_zero::v1::urContextRetain(v1_cast(Context))); + UR_CALL_NOCHECK(ur::level_zero::urDeviceRetain(common_cast(Device))); } -void ur_exp_command_buffer_handle_t_::cleanupCommandBufferResources() { +void ur::level_zero::v1::ur_exp_command_buffer_handle_t_:: + cleanupCommandBufferResources() { // Release the memory allocated to the Context stored in the command_buffer - UR_CALL_NOCHECK(ur::level_zero::urContextRelease(Context)); + UR_CALL_NOCHECK(ur::level_zero::v1::urContextRelease(v1_cast(Context))); // Release the device - UR_CALL_NOCHECK(ur::level_zero::urDeviceRelease(Device)); + UR_CALL_NOCHECK(ur::level_zero::urDeviceRelease(common_cast(Device))); // Release the memory allocated to the CommandList stored in the // command_buffer @@ -569,7 +577,8 @@ void ur_exp_command_buffer_handle_t_::cleanupCommandBufferResources() { // some other thread) then release referenced memory allocations. As a // result, memory can be deallocated and context can be removed from // container in the platform. That's why we need to lock a mutex here. - ur_platform_handle_t Platform = Kernel->Program->Context->getPlatform(); + ur_platform_handle_t Platform = + v1_cast(Kernel->Program->Context)->getPlatform(); std::scoped_lock ContextsLock(Platform->ContextsMutex); if (--Kernel->SubmissionsCount == 0) { @@ -587,11 +596,12 @@ void ur_exp_command_buffer_handle_t_::cleanupCommandBufferResources() { for (auto &AssociatedKernel : KernelsList) { ReleaseIndirectMem(AssociatedKernel); - UR_CALL_NOCHECK(ur::level_zero::urKernelRelease(AssociatedKernel)); + UR_CALL_NOCHECK( + ur::level_zero::v1::urKernelRelease(v1_cast(AssociatedKernel))); } } -void ur_exp_command_buffer_handle_t_::registerSyncPoint( +void ur::level_zero::v1::ur_exp_command_buffer_handle_t_::registerSyncPoint( ur_exp_command_buffer_sync_point_t SyncPoint, ur_event_handle_t Event) { SyncPoints[SyncPoint] = Event; NextSyncPoint++; @@ -599,7 +609,8 @@ void ur_exp_command_buffer_handle_t_::registerSyncPoint( } ze_command_list_handle_t -ur_exp_command_buffer_handle_t_::chooseCommandList(bool PreferCopyEngine) { +ur::level_zero::v1::ur_exp_command_buffer_handle_t_::chooseCommandList( + bool PreferCopyEngine) { if (PreferCopyEngine && useCopyEngine()) { // We indicate that ZeCopyCommandList contains commands to be submitted. MCopyCommandListEmpty = false; @@ -608,7 +619,8 @@ ur_exp_command_buffer_handle_t_::chooseCommandList(bool PreferCopyEngine) { return ZeComputeCommandList; } -ur_result_t ur_exp_command_buffer_handle_t_::getFenceForQueue( +ur_result_t +ur::level_zero::v1::ur_exp_command_buffer_handle_t_::getFenceForQueue( ze_command_queue_handle_t &ZeCommandQueue, ze_fence_handle_t &ZeFence) { // If we already have created a fence for this queue, first reset then reuse // it, otherwise create a new fence. @@ -625,8 +637,6 @@ ur_result_t ur_exp_command_buffer_handle_t_::getFenceForQueue( return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - /** * Creates a L0 command list * @param[in] Context The Context associated with the command-list @@ -708,8 +718,9 @@ bool canBeInOrder(ur_context_handle_t Context, * @param CommandBuffer The CommandBuffer * @return UR_RESULT_SUCCESS or an error code on failure. */ -ur_result_t appendExecutionWaits(ur_exp_command_buffer_handle_t CommandBuffer) { - +ur_result_t +appendExecutionWaits(::ur_exp_command_buffer_handle_t CommandBufferOpque) { + auto CommandBuffer = v1_cast(CommandBufferOpque); std::vector PrecondEvents; if (CommandBuffer->ZeCommandListResetEvents) { PrecondEvents.push_back(CommandBuffer->AllResetEvent->ZeEvent); @@ -732,9 +743,12 @@ ur_result_t appendExecutionWaits(ur_exp_command_buffer_handle_t CommandBuffer) { } ur_result_t -urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, +urCommandBufferCreateExp(::ur_context_handle_t ContextOpque, + ::ur_device_handle_t DeviceOpque, const ur_exp_command_buffer_desc_t *CommandBufferDesc, - ur_exp_command_buffer_handle_t *CommandBuffer) { + ::ur_exp_command_buffer_handle_t *CommandBuffer) { + auto Context = v1_cast(ContextOpque); + auto Device = common_cast(DeviceOpque); bool IsInOrder = canBeInOrder(Context, CommandBufferDesc); bool EnableProfiling = CommandBufferDesc->enableProfiling && !IsInOrder; bool IsUpdatable = CommandBufferDesc->isUpdatable; @@ -822,11 +836,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, } try { - *CommandBuffer = new ur_exp_command_buffer_handle_t_( - Context, Device, ZeComputeCommandList, ZeComputeCommandListTranslated, - ZeCommandListResetEvents, ZeCopyCommandList, ExecutionFinishedEvent, - WaitEvent, AllResetEvent, CopyFinishedEvent, ComputeFinishedEvent, - CommandBufferDesc, IsInOrder, ImmediateAppendPath); + *CommandBuffer = reinterpret_cast<::ur_exp_command_buffer_handle_t>( + new ur::level_zero::v1::ur_exp_command_buffer_handle_t_( + Context, Device, ZeComputeCommandList, + ZeComputeCommandListTranslated, ZeCommandListResetEvents, + ZeCopyCommandList, ExecutionFinishedEvent, WaitEvent, AllResetEvent, + CopyFinishedEvent, ComputeFinishedEvent, CommandBufferDesc, + IsInOrder, ImmediateAppendPath)); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -839,13 +855,15 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, } ur_result_t -urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { +urCommandBufferRetainExp(::ur_exp_command_buffer_handle_t CommandBufferOpque) { + auto CommandBuffer = v1_cast(CommandBufferOpque); CommandBuffer->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t -urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { +urCommandBufferReleaseExp(::ur_exp_command_buffer_handle_t CommandBufferOpque) { + auto CommandBuffer = v1_cast(CommandBufferOpque); if (!CommandBuffer->RefCount.release()) return UR_RESULT_SUCCESS; @@ -957,9 +975,10 @@ finalizeWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer) { return UR_RESULT_SUCCESS; } -ur_result_t -urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t CommandBuffer) { - UR_ASSERT(CommandBuffer, UR_RESULT_ERROR_INVALID_NULL_POINTER); +ur_result_t urCommandBufferFinalizeExp( + ::ur_exp_command_buffer_handle_t CommandBufferOpque) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + UR_ASSERT(CommandBufferOpque, UR_RESULT_ERROR_INVALID_NULL_POINTER); UR_ASSERT(!CommandBuffer->IsFinalized, UR_RESULT_ERROR_INVALID_OPERATION); // It is not allowed to append to command list from multiple threads. @@ -1000,7 +1019,8 @@ urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t CommandBuffer) { */ ur_result_t setKernelPendingArguments( ur_device_handle_t Device, - std::vector &PendingArguments, + std::vector + &PendingArguments, ze_kernel_handle_t ZeKernel) { // If there are any pending arguments set them now. for (auto &Arg : PendingArguments) { @@ -1023,18 +1043,29 @@ ur_result_t setKernelPendingArguments( return UR_RESULT_SUCCESS; } +kernel_command_handle * +CreateCommandHandle(::ur_exp_command_buffer_handle_t commandBuffer, + ::ur_kernel_handle_t kernel, uint64_t commandId, + uint32_t workDim, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *kernelAlternatives) { + return new kernel_command_handle(commandBuffer, kernel, commandId, workDim, + numKernelAlternatives, kernelAlternatives); +} + ur_result_t urCommandBufferAppendKernelLaunchExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_kernel_handle_t Kernel, - uint32_t WorkDim, const size_t *GlobalWorkOffset, - const size_t *GlobalWorkSize, const size_t *LocalWorkSize, - uint32_t NumKernelAlternatives, ur_kernel_handle_t *KernelAlternatives, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_kernel_handle_t KernelOpque, uint32_t WorkDim, + const size_t *GlobalWorkOffset, const size_t *GlobalWorkSize, + const size_t *LocalWorkSize, uint32_t NumKernelAlternatives, + ::ur_kernel_handle_t *KernelAlternatives, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *RetSyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t *Command) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t *Command) { + auto Kernel = v1_cast(KernelOpque); + auto CommandBuffer = v1_cast(CommandBufferOpque); UR_ASSERT(Kernel->Program, UR_RESULT_ERROR_INVALID_NULL_POINTER); // Command handles can only be obtained from updatable command-buffers @@ -1050,8 +1081,8 @@ ur_result_t urCommandBufferAppendKernelLaunchExp( UR_CALL(getZeKernel(Device->ZeDevice, Kernel, &ZeKernel)); if (GlobalWorkOffset != NULL) { - UR_CALL(setKernelGlobalOffset(CommandBuffer->Context, ZeKernel, WorkDim, - GlobalWorkOffset)); + UR_CALL(setKernelGlobalOffset(v1_cast(CommandBuffer->Context), ZeKernel, + WorkDim, GlobalWorkOffset)); } // If there are any pending arguments set them now. @@ -1070,13 +1101,13 @@ ur_result_t urCommandBufferAppendKernelLaunchExp( CommandBuffer->KernelsList.push_back(Kernel); for (size_t i = 0; i < NumKernelAlternatives; i++) { - CommandBuffer->KernelsList.push_back(KernelAlternatives[i]); + CommandBuffer->KernelsList.push_back(v1_cast(KernelAlternatives[i])); } - UR_CALL(ur::level_zero::urKernelRetain(Kernel)); + UR_CALL(ur::level_zero::v1::urKernelRetain(KernelOpque)); // Retain alternative kernels if provided for (size_t i = 0; i < NumKernelAlternatives; i++) { - UR_CALL(ur::level_zero::urKernelRetain(KernelAlternatives[i])); + UR_CALL(ur::level_zero::v1::urKernelRetain(KernelAlternatives[i])); } if (Command) { @@ -1090,10 +1121,12 @@ ur_result_t urCommandBufferAppendKernelLaunchExp( std::unique_ptr NewCommand; UR_CALL(createCommandHandleUnlocked( - CommandBuffer, ZeCommandList, Kernel, WorkDim, GlobalWorkSize, + CommandBufferOpque, ZeCommandList, KernelOpque, WorkDim, GlobalWorkSize, NumKernelAlternatives, KernelAlternatives, Platform, getZeKernelWrapped, - Device, NewCommand)); - *Command = NewCommand.get(); + CreateCommandHandle, Device, NewCommand)); + *Command = + common_cast(reinterpret_cast( + NewCommand.get())); CommandBuffer->CommandHandles.push_back(std::move(NewCommand)); } std::vector ZeEventList; @@ -1112,26 +1145,29 @@ ur_result_t urCommandBufferAppendKernelLaunchExp( } ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, - uint32_t workDim, const size_t *pGlobalWorkOffset, - const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, - uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, - uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_kernel_handle_t hKernelOpque, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternatives, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /* numEventsInWaitList */, - const ur_event_handle_t * /* phEventWaitList */, + const ::ur_event_handle_t * /* phEventWaitList */, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /* phEvent */, - ur_exp_command_buffer_command_handle_t *phCommand) { + ::ur_event_handle_t * /* phEvent */, + ::ur_exp_command_buffer_command_handle_t *phCommand) { - UR_ASSERT(hKernel, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(hKernel->Program, UR_RESULT_ERROR_INVALID_NULL_POINTER); + UR_ASSERT(hKernelOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + auto Kernel = v1_cast(hKernelOpque); + auto CmdBuf = v1_cast(hCommandBufferOpque); + UR_ASSERT(Kernel->Program, UR_RESULT_ERROR_INVALID_NULL_POINTER); // Command handles can only be obtained from updatable command-buffers - UR_ASSERT(!(phCommand && !hCommandBuffer->IsUpdatable), + UR_ASSERT(!(phCommand && !CmdBuf->IsUpdatable), UR_RESULT_ERROR_INVALID_OPERATION); - if (numArgs != hKernel->ZeKernelProperties->numKernelArgs) { + if (numArgs != Kernel->ZeKernelProperties->numKernelArgs) { setErrorMessage("Wrong number of kernel arguments", UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX, static_cast(ZE_RESULT_ERROR_INVALID_ARGUMENT)); @@ -1140,7 +1176,7 @@ ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( { std::scoped_lock Lock( - hKernel->Mutex, hKernel->Program->Mutex, hCommandBuffer->Mutex); + Kernel->Mutex, Kernel->Program->Mutex, CmdBuf->Mutex); for (uint32_t i = 0; i < numArgs; i++) { if (pArgs[i].index != i) { @@ -1152,29 +1188,30 @@ ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( switch (pArgs[i].type) { case UR_EXP_KERNEL_ARG_TYPE_LOCAL: - UR_CALL(urKernelSetArgValueHelper(hKernel, pArgs[i].index, - pArgs[i].size, nullptr)); + UR_CALL(urKernelSetArgValueHelper(Kernel, pArgs[i].index, pArgs[i].size, + nullptr)); break; case UR_EXP_KERNEL_ARG_TYPE_VALUE: - UR_CALL(urKernelSetArgValueHelper(hKernel, pArgs[i].index, - pArgs[i].size, pArgs[i].value.value)); + UR_CALL(urKernelSetArgValueHelper(Kernel, pArgs[i].index, pArgs[i].size, + pArgs[i].value.value)); break; case UR_EXP_KERNEL_ARG_TYPE_POINTER: - UR_CALL(urKernelSetArgValueHelper( - hKernel, pArgs[i].index, pArgs[i].size, &pArgs[i].value.pointer)); + UR_CALL(urKernelSetArgValueHelper(Kernel, pArgs[i].index, pArgs[i].size, + &pArgs[i].value.pointer)); break; case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: { ur_kernel_arg_mem_obj_properties_t Properties = { UR_STRUCTURE_TYPE_KERNEL_ARG_MEM_OBJ_PROPERTIES, nullptr, pArgs[i].value.memObjTuple.flags}; - UR_CALL(urKernelSetArgMemObjHelper(hKernel, pArgs[i].index, &Properties, - pArgs[i].value.memObjTuple.hMem)); + UR_CALL(urKernelSetArgMemObjHelper( + Kernel, pArgs[i].index, &Properties, + v1_cast(pArgs[i].value.memObjTuple.hMem))); break; } case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { - UR_CALL(urKernelSetArgValueHelper(hKernel, pArgs[i].index, - pArgs[i].size, - &pArgs[i].value.sampler->ZeSampler)); + UR_CALL(urKernelSetArgValueHelper( + Kernel, pArgs[i].index, pArgs[i].size, + &ur::level_zero::common_cast(pArgs[i].value.sampler)->ZeSampler)); break; } default: @@ -1183,30 +1220,32 @@ ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( } } - return ur::level_zero::urCommandBufferAppendKernelLaunchExp( - hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, - numSyncPointsInWaitList, pSyncPointWaitList, 0 /*numEventsInWaitList*/, - nullptr /*phEventWaitList*/, pSyncPoint, nullptr /*phEvent*/, phCommand); + return ur::level_zero::v1::urCommandBufferAppendKernelLaunchExp( + hCommandBufferOpque, hKernelOpque, workDim, pGlobalWorkOffset, + pGlobalWorkSize, pLocalWorkSize, numKernelAlternatives, + phKernelAlternatives, numSyncPointsInWaitList, pSyncPointWaitList, + 0 /*numEventsInWaitList*/, nullptr /*phEventWaitList*/, pSyncPoint, + nullptr /*phEvent*/, phCommand); } ur_result_t urCommandBufferAppendUSMMemcpyExp( - ur_exp_command_buffer_handle_t CommandBuffer, void *Dst, const void *Src, - size_t Size, uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, void *Dst, + const void *Src, size_t Size, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); - bool PreferCopyEngine = !IsDevicePointer(CommandBuffer->Context, Src) || - !IsDevicePointer(CommandBuffer->Context, Dst); + auto Context = CommandBuffer->Context; + bool PreferCopyEngine = + !IsDevicePointer(Context, Src) || !IsDevicePointer(Context, Dst); // For better performance, Copy Engines are not preferred given Shared // pointers on DG2. if (CommandBuffer->Device->isDG2() && - (IsSharedPointer(CommandBuffer->Context, Src) || - IsSharedPointer(CommandBuffer->Context, Dst))) { + (IsSharedPointer(Context, Src) || IsSharedPointer(Context, Dst))) { PreferCopyEngine = false; } PreferCopyEngine |= UseCopyEngineForD2DCopy; @@ -1217,15 +1256,16 @@ ur_result_t urCommandBufferAppendUSMMemcpyExp( } ur_result_t urCommandBufferAppendMemBufferCopyExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t SrcMem, - ur_mem_handle_t DstMem, size_t SrcOffset, size_t DstOffset, size_t Size, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t SrcMem, ::ur_mem_handle_t DstMem, size_t SrcOffset, + size_t DstOffset, size_t Size, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); auto SrcBuffer = ur_cast(SrcMem); auto DstBuffer = ur_cast(DstMem); @@ -1234,11 +1274,13 @@ ur_result_t urCommandBufferAppendMemBufferCopyExp( SrcLock, DstBuffer->Mutex); char *ZeHandleSrc; - UR_CALL(SrcBuffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - CommandBuffer->Device, nullptr, 0u)); + UR_CALL(SrcBuffer->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + CommandBuffer->Device, nullptr, 0u)); char *ZeHandleDst; - UR_CALL(DstBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - CommandBuffer->Device, nullptr, 0u)); + UR_CALL(DstBuffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + CommandBuffer->Device, nullptr, 0u)); bool PreferCopyEngine = (SrcBuffer->OnHost || DstBuffer->OnHost); @@ -1251,17 +1293,18 @@ ur_result_t urCommandBufferAppendMemBufferCopyExp( } ur_result_t urCommandBufferAppendMemBufferCopyRectExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t SrcMem, - ur_mem_handle_t DstMem, ur_rect_offset_t SrcOrigin, - ur_rect_offset_t DstOrigin, ur_rect_region_t Region, size_t SrcRowPitch, - size_t SrcSlicePitch, size_t DstRowPitch, size_t DstSlicePitch, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t SrcMem, ::ur_mem_handle_t DstMem, + ur_rect_offset_t SrcOrigin, ur_rect_offset_t DstOrigin, + ur_rect_region_t Region, size_t SrcRowPitch, size_t SrcSlicePitch, + size_t DstRowPitch, size_t DstSlicePitch, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); auto SrcBuffer = ur_cast(SrcMem); auto DstBuffer = ur_cast(DstMem); @@ -1270,11 +1313,13 @@ ur_result_t urCommandBufferAppendMemBufferCopyRectExp( SrcLock, DstBuffer->Mutex); char *ZeHandleSrc; - UR_CALL(SrcBuffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - CommandBuffer->Device, nullptr, 0u)); + UR_CALL(SrcBuffer->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + CommandBuffer->Device, nullptr, 0u)); char *ZeHandleDst; - UR_CALL(DstBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - CommandBuffer->Device, nullptr, 0u)); + UR_CALL(DstBuffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + CommandBuffer->Device, nullptr, 0u)); bool PreferCopyEngine = (SrcBuffer->OnHost || DstBuffer->OnHost); @@ -1288,19 +1333,22 @@ ur_result_t urCommandBufferAppendMemBufferCopyRectExp( } ur_result_t urCommandBufferAppendMemBufferWriteExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t Buffer, - size_t Offset, size_t Size, const void *Src, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t BufferOpque, size_t Offset, size_t Size, const void *Src, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto Buffer = v1_cast(BufferOpque); std::scoped_lock Lock(Buffer->Mutex); char *ZeHandleDst = nullptr; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, + UR_CALL(Buffer->getZeHandle(ZeHandleDst, + ur::level_zero::v1::ur_mem_handle_t_::write_only, CommandBuffer->Device, nullptr, 0u)); // Always prefer copy engine for writes bool PreferCopyEngine = true; @@ -1314,21 +1362,24 @@ ur_result_t urCommandBufferAppendMemBufferWriteExp( } ur_result_t urCommandBufferAppendMemBufferWriteRectExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t Buffer, - ur_rect_offset_t BufferOffset, ur_rect_offset_t HostOffset, - ur_rect_region_t Region, size_t BufferRowPitch, size_t BufferSlicePitch, - size_t HostRowPitch, size_t HostSlicePitch, void *Src, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t BufferOpque, ur_rect_offset_t BufferOffset, + ur_rect_offset_t HostOffset, ur_rect_region_t Region, size_t BufferRowPitch, + size_t BufferSlicePitch, size_t HostRowPitch, size_t HostSlicePitch, + void *Src, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto Buffer = v1_cast(BufferOpque); std::scoped_lock Lock(Buffer->Mutex); char *ZeHandleDst = nullptr; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, + UR_CALL(Buffer->getZeHandle(ZeHandleDst, + ur::level_zero::v1::ur_mem_handle_t_::write_only, CommandBuffer->Device, nullptr, 0u)); // Always prefer copy engine for writes @@ -1343,18 +1394,22 @@ ur_result_t urCommandBufferAppendMemBufferWriteRectExp( } ur_result_t urCommandBufferAppendMemBufferReadExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t Buffer, - size_t Offset, size_t Size, void *Dst, uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t BufferOpque, size_t Offset, size_t Size, void *Dst, + uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto Buffer = v1_cast(BufferOpque); std::scoped_lock SrcLock(Buffer->Mutex); char *ZeHandleSrc = nullptr; - UR_CALL(Buffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, + UR_CALL(Buffer->getZeHandle(ZeHandleSrc, + ur::level_zero::v1::ur_mem_handle_t_::read_only, CommandBuffer->Device, nullptr, 0u)); // Always prefer copy engine for reads @@ -1367,21 +1422,24 @@ ur_result_t urCommandBufferAppendMemBufferReadExp( } ur_result_t urCommandBufferAppendMemBufferReadRectExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t Buffer, - ur_rect_offset_t BufferOffset, ur_rect_offset_t HostOffset, - ur_rect_region_t Region, size_t BufferRowPitch, size_t BufferSlicePitch, - size_t HostRowPitch, size_t HostSlicePitch, void *Dst, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t BufferOpque, ur_rect_offset_t BufferOffset, + ur_rect_offset_t HostOffset, ur_rect_region_t Region, size_t BufferRowPitch, + size_t BufferSlicePitch, size_t HostRowPitch, size_t HostSlicePitch, + void *Dst, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto Buffer = v1_cast(BufferOpque); std::scoped_lock SrcLock(Buffer->Mutex); char *ZeHandleSrc; - UR_CALL(Buffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, + UR_CALL(Buffer->getZeHandle(ZeHandleSrc, + ur::level_zero::v1::ur_mem_handle_t_::read_only, CommandBuffer->Device, nullptr, 0u)); // Always prefer copy engine for reads @@ -1395,14 +1453,16 @@ ur_result_t urCommandBufferAppendMemBufferReadRectExp( } ur_result_t urCommandBufferAppendUSMPrefetchExp( - ur_exp_command_buffer_handle_t CommandBuffer, const void *Mem, size_t Size, - ur_usm_migration_flags_t Flags, uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, const void *Mem, + size_t Size, ur_usm_migration_flags_t Flags, + uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *RetSyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); std::vector ZeEventList; ze_event_handle_t ZeLaunchEvent = nullptr; @@ -1446,14 +1506,15 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp( } ur_result_t urCommandBufferAppendUSMAdviseExp( - ur_exp_command_buffer_handle_t CommandBuffer, const void *Mem, size_t Size, - ur_usm_advice_flags_t Advice, uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, const void *Mem, + size_t Size, ur_usm_advice_flags_t Advice, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *RetSyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); // A memory chunk can be advised with muliple memory advices // We therefore prefer if statements to switch cases to combine all potential // flags @@ -1509,22 +1570,25 @@ ur_result_t urCommandBufferAppendUSMAdviseExp( } ur_result_t urCommandBufferAppendMemBufferFillExp( - ur_exp_command_buffer_handle_t CommandBuffer, ur_mem_handle_t Buffer, - const void *Pattern, size_t PatternSize, size_t Offset, size_t Size, - uint32_t NumSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + ::ur_mem_handle_t BufferOpque, const void *Pattern, size_t PatternSize, + size_t Offset, size_t Size, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto Buffer = v1_cast(BufferOpque); std::scoped_lock Lock(Buffer->Mutex); char *ZeHandleDst = nullptr; - ur_buffer *UrBuffer = reinterpret_cast(Buffer); - UR_CALL(UrBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - CommandBuffer->Device, nullptr, 0u)); + ur_buffer *UrBuffer = static_cast(Buffer); + UR_CALL(UrBuffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + CommandBuffer->Device, nullptr, 0u)); return enqueueCommandBufferFillHelper( UR_COMMAND_MEM_BUFFER_FILL, CommandBuffer, ZeHandleDst + Offset, @@ -1534,15 +1598,16 @@ ur_result_t urCommandBufferAppendMemBufferFillExp( } ur_result_t urCommandBufferAppendUSMFillExp( - ur_exp_command_buffer_handle_t CommandBuffer, void *Ptr, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, void *Ptr, const void *Pattern, size_t PatternSize, size_t Size, uint32_t NumSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *SyncPointWaitList, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, ur_exp_command_buffer_sync_point_t *SyncPoint, - ur_event_handle_t * /*Event*/, - ur_exp_command_buffer_command_handle_t * /*Command*/) { + ::ur_event_handle_t * /*Event*/, + ::ur_exp_command_buffer_command_handle_t * /*Command*/) { + auto CommandBuffer = v1_cast(CommandBufferOpque); return enqueueCommandBufferFillHelper( UR_COMMAND_MEM_BUFFER_FILL, CommandBuffer, Ptr, @@ -1829,27 +1894,33 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer, return UR_RESULT_SUCCESS; } -ur_result_t urEnqueueCommandBufferExp( - ur_queue_handle_t UrQueue, ur_exp_command_buffer_handle_t CommandBuffer, - uint32_t NumEventsInWaitList, const ur_event_handle_t *EventWaitList, - ur_event_handle_t *Event) { +ur_result_t +urEnqueueCommandBufferExp(::ur_queue_handle_t UrQueueOpque, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + uint32_t NumEventsInWaitList, + const ::ur_event_handle_t *EventWaitList, + ::ur_event_handle_t *Event) { + auto UrQueue = v1_cast(UrQueueOpque); + auto CommandBuffer = v1_cast(CommandBufferOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto EventInternal = v1_cast(Event); std::scoped_lock Lock(UrQueue->Mutex); UR_CALL(waitForOngoingExecution(CommandBuffer)); - const bool IsInternal = (Event == nullptr); + const bool IsInternal = (EventInternal == nullptr); const bool DoProfiling = (UrQueue->Properties & UR_QUEUE_FLAG_PROFILING_ENABLE) && (!CommandBuffer->IsInOrderCmdList) && - (CommandBuffer->IsProfilingEnabled) && Event; + (CommandBuffer->IsProfilingEnabled) && EventInternal; ur_event_handle_t InternalEvent; - ur_event_handle_t *OutEvent = Event ? Event : &InternalEvent; + ur_event_handle_t *OutEvent = EventInternal ? EventInternal : &InternalEvent; ur_command_list_ptr_t ZeCommandListHelper{}; UR_CALL(UrQueue->Context->getAvailableCommandList( UrQueue, ZeCommandListHelper, false /*UseCopyEngine*/, - NumEventsInWaitList, EventWaitList, false /*AllowBatching*/, + NumEventsInWaitList, EventWaitListInternal, false /*AllowBatching*/, nullptr /*ForcedCmdQueue*/)); UR_CALL(createEventAndAssociateQueue( @@ -1858,12 +1929,12 @@ ur_result_t urEnqueueCommandBufferExp( if (CommandBuffer->UseImmediateAppendPath) { UR_CALL(enqueueImmediateAppendPath( - CommandBuffer, UrQueue, NumEventsInWaitList, EventWaitList, OutEvent, - ZeCommandListHelper, DoProfiling)); + CommandBuffer, UrQueue, NumEventsInWaitList, EventWaitListInternal, + OutEvent, ZeCommandListHelper, DoProfiling)); } else { UR_CALL(enqueueWaitEventPath(CommandBuffer, UrQueue, NumEventsInWaitList, - EventWaitList, OutEvent, ZeCommandListHelper, - DoProfiling)); + EventWaitListInternal, OutEvent, + ZeCommandListHelper, DoProfiling)); } return UR_RESULT_SUCCESS; @@ -1902,14 +1973,16 @@ ur_result_t updateCommandBuffer( } // namespace ur_result_t urCommandBufferUpdateKernelLaunchExp( - ur_exp_command_buffer_handle_t CommandBuffer, uint32_t numKernelUpdates, + ::ur_exp_command_buffer_handle_t CommandBufferOpque, + uint32_t numKernelUpdates, const ur_exp_command_buffer_update_kernel_launch_desc_t *CommandDesc) { + auto CommandBuffer = v1_cast(CommandBufferOpque); UR_ASSERT(CommandBuffer->IsUpdatable && CommandBuffer->IsFinalized, UR_RESULT_ERROR_INVALID_OPERATION); { std::scoped_lock Guard(CommandBuffer->Mutex); UR_CALL( - validateCommandDescUnlocked(CommandBuffer, CommandBuffer->Device, + validateCommandDescUnlocked(CommandBufferOpque, CommandBuffer->Device, CommandBuffer->Context->getPlatform() ->ZeDriverGlobalOffsetExtensionFound, numKernelUpdates, CommandDesc)); @@ -1923,23 +1996,24 @@ ur_result_t urCommandBufferUpdateKernelLaunchExp( } ur_result_t urCommandBufferUpdateSignalEventExp( - ur_exp_command_buffer_command_handle_t /*Command*/, - ur_event_handle_t * /*Event*/) { + ::ur_exp_command_buffer_command_handle_t /*Command*/, + ::ur_event_handle_t * /*Event*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ur_result_t urCommandBufferUpdateWaitEventsExp( - ur_exp_command_buffer_command_handle_t /*Command*/, + ::ur_exp_command_buffer_command_handle_t /*Command*/, uint32_t /*NumEventsInWaitList*/, - const ur_event_handle_t * /*EventWaitList*/) { + const ::ur_event_handle_t * /*EventWaitList*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ur_result_t -urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, +urCommandBufferGetInfoExp(::ur_exp_command_buffer_handle_t hCommandBufferOpque, ur_exp_command_buffer_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto hCommandBuffer = v1_cast(hCommandBufferOpque); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { @@ -1963,12 +2037,13 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, } ur_result_t urCommandBufferAppendNativeCommandExp( - ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - void *pData, ur_exp_command_buffer_handle_t, + void *pData, ::ur_exp_command_buffer_handle_t, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ur_exp_command_buffer_sync_point_t *pSyncPoint) { + auto hCommandBuffer = v1_cast(hCommandBufferOpque); // Use Compute command-list as we don't know the types of commands the // user will choose to append. ze_command_list_handle_t ZeCommandList = hCommandBuffer->ZeComputeCommandList; @@ -1995,12 +2070,13 @@ ur_result_t urCommandBufferAppendNativeCommandExp( return UR_RESULT_SUCCESS; } -ur_result_t -urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, - ur_native_handle_t *phNativeCommandBuffer) { +ur_result_t urCommandBufferGetNativeHandleExp( + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_native_handle_t *phNativeCommandBuffer) { + auto hCommandBuffer = v1_cast(hCommandBufferOpque); // Return Compute command-list as it is guaranteed to always exist ze_command_list_handle_t ZeCommandList = hCommandBuffer->ZeComputeCommandList; *phNativeCommandBuffer = reinterpret_cast(ZeCommandList); return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/command_buffer.hpp index 2bc2d3f195f9a..f7069c73843dd 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.hpp @@ -16,17 +16,20 @@ #include "common.hpp" +#include "common/command_buffer_command.hpp" #include "common/ur_ref_count.hpp" #include "context.hpp" #include "kernel.hpp" #include "queue.hpp" +namespace ur::level_zero::v1 { + struct command_buffer_profiling_t { ur_exp_command_buffer_sync_point_t NumEvents; ze_kernel_timestamp_result_t *Timestamps; }; -struct ur_exp_command_buffer_handle_t_ : public ur_object { +struct ur_exp_command_buffer_handle_t_ : public ur_object_t { ur_exp_command_buffer_handle_t_( ur_context_handle_t Context, ur_device_handle_t Device, ze_command_list_handle_t CommandList, @@ -147,8 +150,11 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { // command_buffer. std::vector KernelsList; // Track handle objects to free when command-buffer is destroyed. - std::vector> + std::vector< + std::unique_ptr> CommandHandles; ur::RefCount RefCount; }; + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index af5976fa8aa53..e7f502a386da4 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -14,248 +14,129 @@ #include #include #include +#include #include #include -#ifdef _WIN32 -#include "windows.h" -#else -#include -#include -#endif - -#include -#include -#include -#include -#include - -#include -#include - -#include "common/ur_ref_count.hpp" -#include "logger/ur_logger.hpp" +#include "common/helpers/shared_helpers.hpp" #include "ur_interface_loader.hpp" +#include -struct _ur_platform_handle_t; - -[[maybe_unused]] static bool checkL0LoaderTeardown() { - try { - if (!zelCheckIsLoaderInTearDown()) { - return true; - } - } catch (...) { - } - UR_LOG(DEBUG, - "ZE ---> checkL0LoaderTeardown: Loader is in teardown or is unstable"); - return false; -} +namespace ur::level_zero::v1 { -// Controls UR L0 calls tracing. -enum UrDebugLevel { - UR_L0_DEBUG_NONE = 0x0, - UR_L0_DEBUG_BASIC = 0x1, - UR_L0_DEBUG_VALIDATION = 0x2, - UR_L0_DEBUG_ALL = -1 +struct ur_object_t : ur::handle_base { + ur_shared_mutex Mutex; + bool OwnNativeHandle = false; }; -const int UrL0Debug = [] { - const char *ZeDebugMode = std::getenv("ZE_DEBUG"); - const char *UrL0DebugMode = std::getenv("UR_L0_DEBUG"); - uint32_t DebugMode = 0; - if (UrL0DebugMode) { - DebugMode = std::atoi(UrL0DebugMode); - } else if (ZeDebugMode) { - DebugMode = std::atoi(ZeDebugMode); - } - return DebugMode; -}(); - -const int UrL0LeaksDebug = [] { - const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG"); - if (!UrRet) - return 0; - return std::atoi(UrRet); -}(); - -const int UrL0VectorWidth = [] { - const char *UrRet = std::getenv("UR_L0_VECTOR_WIDTH_SIZE"); - if (!UrRet) - return 0; - return std::atoi(UrRet); -}(); - -// Enable for UR L0 Adapter to Init all L0 Drivers on the system with filtering -// in place for only currently used Drivers. -const int UrL0InitAllDrivers = [] { - const char *UrRet = std::getenv("UR_L0_INIT_ALL_DRIVERS"); - if (!UrRet) - return 0; - return std::atoi(UrRet); -}(); - -// Controls Level Zero calls serialization to w/a Level Zero driver being not MT -// ready. Recognized values (can be used as a bit mask): -enum { - UrL0SerializeNone = - 0, // no locking or blocking (except when SYCL RT requested blocking) - UrL0SerializeLock = 1, // locking around each UR_CALL - UrL0SerializeBlock = - 2, // blocking UR calls, where supported (usually in enqueue commands) +// Forward declarations for v1-only concrete handle types. +struct ur_context_handle_t_; +typedef struct ur_context_handle_t_ *ur_context_handle_t; +struct ur_event_handle_t_; +typedef struct ur_event_handle_t_ *ur_event_handle_t; +struct ur_queue_handle_t_; +typedef struct ur_queue_handle_t_ *ur_queue_handle_t; +struct ur_kernel_handle_t_; +typedef struct ur_kernel_handle_t_ *ur_kernel_handle_t; +struct ur_mem_handle_t_; +typedef struct ur_mem_handle_t_ *ur_mem_handle_t; +struct ur_usm_pool_handle_t_; +typedef struct ur_usm_pool_handle_t_ *ur_usm_pool_handle_t; +struct ur_exp_command_buffer_handle_t_; +typedef struct ur_exp_command_buffer_handle_t_ *ur_exp_command_buffer_handle_t; + +// Cast from an opaque UR handle to the v1 concrete type. The loader only ever +// reads offset 0 (ddi_table). +namespace detail { +// Maps an opaque handle typedef to its corresponding v1 internal struct. +template struct v1_handle_traits; +template <> struct v1_handle_traits<::ur_context_handle_t> { + using type = ur_context_handle_t_; }; - -static const uint32_t UrL0Serialize = [] { - const char *ZeSerializeMode = std::getenv("ZE_SERIALIZE"); - const char *UrL0SerializeMode = std::getenv("UR_L0_SERIALIZE"); - uint32_t SerializeModeValue = 0; - if (UrL0SerializeMode) { - SerializeModeValue = std::atoi(UrL0SerializeMode); - } else if (ZeSerializeMode) { - SerializeModeValue = std::atoi(ZeSerializeMode); - } - return SerializeModeValue; -}(); - -static const uint32_t UrL0QueueSyncNonBlocking = [] { - const char *UrL0QueueSyncNonBlocking = - std::getenv("UR_L0_QUEUE_SYNCHRONIZE_NON_BLOCKING"); - uint32_t L0QueueSyncLockingModeValue = 1; - if (UrL0QueueSyncNonBlocking) { - L0QueueSyncLockingModeValue = std::atoi(UrL0QueueSyncNonBlocking); - } - return L0QueueSyncLockingModeValue; -}(); - -// Controls whether the L0 Adapter creates signal events for commands on -// integrated gpu devices. -static const uint32_t UrL0OutOfOrderIntegratedSignalEvent = [] { - const char *UrL0OutOfOrderIntegratedSignalEventEnv = - std::getenv("UR_L0_OOQ_INTEGRATED_SIGNAL_EVENT"); - uint32_t UrL0OutOfOrderIntegratedSignalEventValue = 1; - if (UrL0OutOfOrderIntegratedSignalEventEnv) { - UrL0OutOfOrderIntegratedSignalEventValue = - std::atoi(UrL0OutOfOrderIntegratedSignalEventEnv); - } - return UrL0OutOfOrderIntegratedSignalEventValue; -}(); - -// This class encapsulates actions taken along with a call to Level Zero API. -class ZeCall { -private: - // The global mutex that is used for total serialization of Level Zero calls. - static std::mutex GlobalLock; - -public: - ZeCall() { - if ((UrL0Serialize & UrL0SerializeLock) != 0) { - GlobalLock.lock(); - } - } - ~ZeCall() { - if ((UrL0Serialize & UrL0SerializeLock) != 0) { - GlobalLock.unlock(); - } - } - - // The non-static version just calls static one. - ze_result_t doCall(ze_result_t ZeResult, const char *ZeName, - const char *ZeArgs, bool TraceError = true); +template <> struct v1_handle_traits<::ur_event_handle_t> { + using type = ur_event_handle_t_; }; - -// This function will ensure compatibility with both Linux and Windows for -// setting environment variables. -bool setEnvVar(const char *name, const char *value); - -// Returns the ze_structure_type_t to use in .stype of a structured descriptor. -// Intentionally not defined; will give an error if no proper specialization -template ze_structure_type_t getZeStructureType(); -template ze_structure_type_ext_t getZexStructureType(); -template zes_structure_type_t getZesStructureType(); - -// The helpers to properly default initialize Level-Zero descriptor and -// properties structures. -template struct ZeStruct : public T { - ZeStruct() : T{} { // zero initializes base struct - this->stype = getZeStructureType(); - this->pNext = nullptr; - } +template <> struct v1_handle_traits<::ur_queue_handle_t> { + using type = ur_queue_handle_t_; }; - -template struct ZexStruct : public T { - ZexStruct() : T{} { // zero initializes base struct - this->stype = getZexStructureType(); - this->pNext = nullptr; - } +template <> struct v1_handle_traits<::ur_kernel_handle_t> { + using type = ur_kernel_handle_t_; }; - -template struct ZesStruct : public T { - ZesStruct() : T{} { // zero initializes base struct - this->stype = getZesStructureType(); - this->pNext = nullptr; - } +template <> struct v1_handle_traits<::ur_mem_handle_t> { + using type = ur_mem_handle_t_; +}; +template <> struct v1_handle_traits<::ur_usm_pool_handle_t> { + using type = ur_usm_pool_handle_t_; +}; +template <> struct v1_handle_traits<::ur_exp_command_buffer_handle_t> { + using type = ur_exp_command_buffer_handle_t_; }; -// This function will ensure compatibility with both Linux and Windows for -// setting environment variables. -bool setEnvVar(const char *name, const char *value); - -// Map Level Zero runtime error code to UR error code. -ur_result_t ze2urResult(ze_result_t ZeResult); - -// Parse Level Zero error code and return the error string. -void zeParseError(ze_result_t ZeError, const char *&ErrorString); +template +using v1_internal_t = typename v1_handle_traits::type; -// Trace a call to Level-Zero RT -#define ZE2UR_CALL(ZeName, ZeArgs) \ - { \ - ze_result_t ZeResult = ZeName ZeArgs; \ - if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) \ - return ze2urResult(Result); \ - } - -// Trace a call to Level-Zero RT, throw on error -#define ZE2UR_CALL_THROWS(ZeName, ZeArgs) \ - { \ - ze_result_t ZeResult = ZeName ZeArgs; \ - if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) { \ - throw ze2urResult(Result); \ - } \ - } +// Reverse mapping: v1 internal struct -> opaque handle. +template struct v1_opaque_handle_for; +template <> struct v1_opaque_handle_for { + using type = ::ur_context_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_event_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_queue_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_kernel_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_mem_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_usm_pool_handle_t; +}; +template <> struct v1_opaque_handle_for { + using type = ::ur_exp_command_buffer_handle_t; +}; +} // namespace detail -// Perform traced call to L0 without checking for errors -#define ZE_CALL_NOCHECK(ZeName, ZeArgs) \ - ZeCall().doCall(ZeName ZeArgs, #ZeName, #ZeArgs, false) +// Opaque handle -> v1 internal pointer. +template +inline detail::v1_internal_t *v1_cast(Opaque h) { + return reinterpret_cast *>(h); +} -#define ZE_CALL_NOCHECK_NAME(ZeName, ZeArgs, callName) \ - ZeCall().doCall(ZeName ZeArgs, callName, #ZeArgs, false) +// Opaque handle array -> v1 internal pointer array. +template +inline detail::v1_internal_t **v1_cast(Opaque *ph) { + return reinterpret_cast **>(ph); +} -// Base class to store common data -struct ur_object : ur::handle_base { - ur_object() : handle_base() {} +// const Opaque handle array -> const v1 internal pointer array. +template +inline detail::v1_internal_t *const *v1_cast(const Opaque *ph) { + return reinterpret_cast *const *>(ph); +} - // This mutex protects accesses to all the non-const member variables. - // Exclusive access is required to modify any of these members. - // - // To get shared access to the object in a scope use std::shared_lock: - // std::shared_lock Lock(Obj->Mutex); - // To get exclusive access to the object in a scope use std::scoped_lock: - // std::scoped_lock Lock(Obj->Mutex); - // - // If several UR objects are accessed in a scope then each object's mutex must - // be locked. For example, to get write access to Obj1 and Obj2 and read - // access to Obj3 in a scope use the following approach: - // std::shared_lock Obj3Lock(Obj3->Mutex, std::defer_lock); - // std::scoped_lock LockAll(Obj1->Mutex, Obj2->Mutex, Obj3Lock); - ur_shared_mutex Mutex; +// V1 internal pointer -> opaque handle (reverse direction). +template +inline typename detail::v1_opaque_handle_for::type +v1_cast(Internal *p) { + return reinterpret_cast< + typename detail::v1_opaque_handle_for::type>(p); +} - // Indicates if we own the native handle or it came from interop that - // asked to not transfer the ownership to SYCL RT. - bool OwnNativeHandle = false; -}; +// V1 internal pointer array -> opaque handle array. +template +inline typename detail::v1_opaque_handle_for::type * +v1_cast(Internal **p) { + return reinterpret_cast< + typename detail::v1_opaque_handle_for::type *>(p); +} // Record for a memory allocation. This structure is used to keep information // for each memory allocation. -struct MemAllocRecord : ur_object { +struct MemAllocRecord : ur_object_t { MemAllocRecord(ur_context_handle_t Context, bool OwnZeMemHandle = true) : Context(Context) { OwnNativeHandle = OwnZeMemHandle; @@ -270,94 +151,4 @@ struct MemAllocRecord : ur_object { ur::RefCount RefCount; }; -extern usm::DisjointPoolAllConfigs DisjointPoolConfigInstance; -extern const bool UseUSMAllocator; - -// Controls support of the indirect access kernels and deferred memory release. -const bool IndirectAccessTrackingEnabled = [] { - char *UrRet = std::getenv("UR_L0_TRACK_INDIRECT_ACCESS_MEMORY"); - char *PiRet = std::getenv("SYCL_PI_LEVEL_ZERO_TRACK_INDIRECT_ACCESS_MEMORY"); - const bool RetVal = UrRet ? std::stoi(UrRet) : (PiRet ? std::stoi(PiRet) : 0); - return RetVal; -}(); - -extern const bool UseUSMAllocator; - -const bool ExposeCSliceInAffinityPartitioning = [] { - char *UrRet = std::getenv("UR_L0_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING"); - char *PiRet = - std::getenv("SYCL_PI_LEVEL_ZERO_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING"); - const char *Flag = UrRet ? UrRet : (PiRet ? PiRet : 0); - return Flag ? std::atoi(Flag) != 0 : false; -}(); - -// TODO: make it into a ur_device_handle_t class member -const std::pair -getRangeOfAllowedCopyEngines(const ur_device_handle_t &Device); - -class ZeDriverVersionStringExtension { - // Pointer to function for Intel Driver Version String - ze_result_t (*zeIntelGetDriverVersionStringPointer)( - ze_driver_handle_t hDriver, char *, size_t *) = nullptr; - -public: - // Whether platform supports Intel Driver Version String. - bool Supported; - - ZeDriverVersionStringExtension() : Supported{false} {} - - void setZeDriverVersionString(ur_platform_handle_t_ *Platform); - void getDriverVersionString(ze_driver_handle_t DriverHandle, - char *pDriverVersion, size_t *pVersionSize); -}; - -class ZeUSMImportExtension { - // Pointers to functions that import/release host memory into USM - ze_result_t (*zexDriverImportExternalPointer)(ze_driver_handle_t hDriver, - void *, size_t) = nullptr; - ze_result_t (*zexDriverReleaseImportedPointer)(ze_driver_handle_t, - void *) = nullptr; - -public: - // Whether platform supports Import/Release. - bool Supported; - - // Whether user has requested Import/Release for buffers. - bool Enabled; - - ZeUSMImportExtension() : Supported{false}, Enabled{false} {} - - void setZeUSMImport(ur_platform_handle_t_ *Platform); - void doZeUSMImport(ze_driver_handle_t DriverHandle, void *HostPtr, - size_t Size); - void doZeUSMRelease(ze_driver_handle_t DriverHandle, void *HostPtr); -}; - -// Helper wrapper for working with USM import extension in Level Zero. -extern ZeUSMImportExtension ZeUSMImport; - -// Some opencl extensions we know are supported by all Level Zero devices. -constexpr char ZE_SUPPORTED_EXTENSIONS[] = - "cl_khr_il_program cl_khr_subgroups cl_intel_subgroups " - "cl_intel_subgroups_short cl_intel_required_subgroup_size "; - -// Global variables for ZER_EXT_RESULT_ADAPTER_SPECIFIC_ERROR -constexpr size_t MaxMessageSize = 256; -extern thread_local int32_t ErrorMessageCode; -extern thread_local char ErrorMessage[MaxMessageSize]; -extern thread_local int32_t ErrorAdapterNativeCode; - -// Utility function for setting a message and warning -[[maybe_unused]] void setErrorMessage(const char *pMessage, int32_t ErrorCode, - int32_t AdapterErrorCode); - -// Returns the host memory page size in bytes. -inline size_t getHostPageSize() { -#ifdef _WIN32 - SYSTEM_INFO SystemInfo; - GetSystemInfo(&SystemInfo); - return static_cast(SystemInfo.dwPageSize); -#else - return static_cast(sysconf(_SC_PAGESIZE)); -#endif -} +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/common/adapter.cpp b/unified-runtime/source/adapters/level_zero/common/adapter.cpp new file mode 100644 index 0000000000000..e4462b00bd552 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/common/adapter.cpp @@ -0,0 +1,733 @@ +//===--------- adapter.cpp - Level Zero Adapter ---------------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "adapter.hpp" +#include "helpers/shared_helpers.hpp" +#include "platform.hpp" +#include + +// As windows order of unloading dlls is reversed from linux, windows will call +// umfTearDown before it could release umf objects in level_zero, so we call +// umfInit on urAdapterGet and umfAdapterTearDown to enforce the teardown of umf +// after umf objects are destructed. +#if defined(_WIN32) +#include +#endif + +ur::level_zero::ZeUSMImportExtension ur::level_zero::ZeUSMImport; + +// This is a temporary workaround on windows, where UR adapter is teardowned +// before the UR loader, which will result in access violation when we use print +// function as the overrided print function was already released with the UR +// adapter. +// TODO: Change adapters to use a common sink class in the loader instead of +// using thier own sink class that inherit from logger::Sink. +class ur_legacy_sink : public logger::Sink { +public: + ur_legacy_sink(std::string logger_name = "", bool skip_prefix = true) + : Sink(std::move(logger_name), skip_prefix) { + this->ostream = &std::cerr; + } + + virtual void print([[maybe_unused]] ur_logger_level_t level, + const std::string &msg) override { + fprintf(stderr, "%s", msg.c_str()); + } +}; + +namespace ur::level_zero { + +// The default value of the loader versions skiplist +// (SYCL_UR_L0_LOADER_SKIPLIST) +static const std::vector LoaderSkiplistDefault = {}; + +// GlobalAdapter used by both L0v1 and L0v2. Only one of the adapters is +// selected at the runtime. +ur_adapter_handle_t_ *GlobalAdapter = nullptr; +std::mutex GlobalAdapterMutex; + +// Find the corresponding ZesDevice Handle for a given ZeDevice +ur_result_t getZesDeviceHandle(ur_adapter_handle_t_ *adapter, + zes_uuid_t coreDeviceUuid, + zes_device_handle_t *ZesDevice, + uint32_t *SubDeviceId, ze_bool_t *SubDevice) { + uint32_t ZesDriverCount = 0; + std::vector ZesDrivers; + std::vector ZesDevices; + ze_result_t ZesResult = ZE_RESULT_ERROR_INVALID_ARGUMENT; + ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, (&ZesDriverCount, nullptr)); + ZesDrivers.resize(ZesDriverCount); + ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, + (&ZesDriverCount, ZesDrivers.data())); + for (uint32_t I = 0; I < ZesDriverCount; ++I) { + ZesResult = ZE_CALL_NOCHECK( + adapter->getDeviceByUUIdFunctionPtr, + (ZesDrivers[I], coreDeviceUuid, ZesDevice, SubDevice, SubDeviceId)); + if (ZesResult == ZE_RESULT_SUCCESS) { + return UR_RESULT_SUCCESS; + } + } + return UR_RESULT_ERROR_INVALID_ARGUMENT; +} + +ur_result_t checkDeviceIntelGPUIpVersionOrNewer(uint32_t ipVersion) { + uint32_t ZeDriverCount = 0; + ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, nullptr)); + if (ZeDriverCount == 0) { + return UR_RESULT_SUCCESS; + } + + std::vector ZeDrivers; + std::vector ZeDevices; + ZeDrivers.resize(ZeDriverCount); + + ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, ZeDrivers.data())); + for (uint32_t I = 0; I < ZeDriverCount; ++I) { + ze_device_properties_t device_properties{}; + ze_device_ip_version_ext_t ipVersionExt{}; + ipVersionExt.stype = ZE_STRUCTURE_TYPE_DEVICE_IP_VERSION_EXT; + ipVersionExt.pNext = nullptr; + device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + device_properties.pNext = &ipVersionExt; + uint32_t ZeDeviceCount = 0; + ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, nullptr)); + ZeDevices.resize(ZeDeviceCount); + ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, ZeDevices.data())); + // Check if this driver has GPU Devices that have this IP Version or newer. + for (uint32_t D = 0; D < ZeDeviceCount; ++D) { + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevices[D], &device_properties)); + if (device_properties.type == ZE_DEVICE_TYPE_GPU && + device_properties.vendorId == 0x8086) { + ze_device_ip_version_ext_t *ipVersionExt = + (ze_device_ip_version_ext_t *)device_properties.pNext; + if (ipVersionExt->ipVersion >= ipVersion) { + return UR_RESULT_SUCCESS; + } + } + } + } + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; +} + +/** + * @brief Initializes the platforms by querying Level Zero drivers and devices. + * + * This function initializes the platforms by querying the available Level Zero + * drivers and devices. It handles different behaviors based on the presence of + * drivers obtained through `zeDriverGet` and initialized drivers through + * `zeInitDrivers`. + * + * @param platforms A vector to store the initialized platform handles. + * @param ZesResult The result of a previous ZES (Level Zero System) operation. + * @return ur_result_t The result of the initialization process. + * + * The function performs the following steps: + * 1. Queries the number of Level Zero drivers using `zeDriverGet`. + * 2. If drivers are found, it retrieves their handles. + * 3. If no drivers are found in either `zeInitDrivers` or `zeDriverGet`, + * it logs a message and returns success. + * 4. If `zeInitDrivers` is supported by the global adapter, it retrieves + * their handles and properties. + * 5. It compares the drivers obtained from `zeDriverGet` and `zeInitDrivers`, + * adding unique drivers to the list. + * 6. If `zeInitDrivers` is not supported, it uses the drivers obtained + * from `zeDriverGet`. + * 7. For each driver, it queries the devices and checks if they are GPU + * devices. + * 8. If a GPU device is found, it initializes a platform for the driver and + * adds it to the platforms vector. + * 9. If ZES operations are successful, it populates the ZES/ZE device mapping + * for the devices into the platform. + * 10. The function handles exceptions and returns the appropriate result. + */ +ur_result_t initPlatforms(ur_adapter_handle_t_ *adapter, PlatformVec &platforms, + ze_result_t ZesResult) noexcept try { + std::vector ZeDrivers; + std::vector ZeDriverGetHandles; + std::vector ZeInitDriversHandles; + std::vector ZeDevices; + uint32_t ZeDriverCount = 0; + uint32_t ZeDriverGetCount = 0; + + auto ZeDriverGetResult = + ZE_CALL_NOCHECK(zeDriverGet, (&ZeDriverGetCount, nullptr)); + if (ZeDriverGetCount > 0 && ZeDriverGetResult == ZE_RESULT_SUCCESS) { + ZeDriverGetHandles.resize(ZeDriverGetCount); + ZE2UR_CALL(zeDriverGet, (&ZeDriverGetCount, ZeDriverGetHandles.data())); + } + if (ZeDriverGetCount == 0 && adapter->ZeInitDriversCount == 0) { + UR_LOG(ERR, "\nNo Valid L0 Drivers found.\n"); + return UR_RESULT_SUCCESS; + } + + if (adapter->InitDriversSupported) { + ZeInitDriversHandles.resize(adapter->ZeInitDriversCount); + ZeDrivers.resize(adapter->ZeInitDriversCount); + ZE2UR_CALL(adapter->initDriversFunctionPtr, + (&adapter->ZeInitDriversCount, ZeInitDriversHandles.data(), + &adapter->InitDriversDesc)); + ZeDrivers.assign(ZeInitDriversHandles.begin(), ZeInitDriversHandles.end()); + if (ZeDriverGetCount > 0 && adapter->ZeInitDriversCount > 0) { + for (uint32_t X = 0; X < adapter->ZeInitDriversCount; ++X) { + // zeDriverGet and zeInitDrivers can return the driver handles in + // reverse order based on driver ordering causing an issue if the + // drivers are expected to be in the same order. To resolve, this loop + // checks if the driver from zeDriverGet already exists in zeInitDrivers + // and only adds it if it is not found. + bool unMatchedDriverHandle = false; + ze_driver_handle_t driverGetHandle = nullptr; + for (uint32_t Y = 0; Y < ZeDriverGetCount; ++Y) { + unMatchedDriverHandle = true; + ZeStruct ZeDriverGetProperties; + ZeStruct ZeInitDriverProperties; + ZE2UR_CALL(zeDriverGetProperties, + (ZeDriverGetHandles[Y], &ZeDriverGetProperties)); + ZE2UR_CALL(zeDriverGetProperties, + (ZeInitDriversHandles[X], &ZeInitDriverProperties)); + driverGetHandle = ZeDriverGetHandles[Y]; + // If zeDriverGet driver is the same version as zeInitDriver driver, + // then do not add it again. + if (ZeDriverGetProperties.driverVersion == + ZeInitDriverProperties.driverVersion) { + UR_LOG(DEBUG, + "zeDriverHandle {} matched between zeDriverGet and " + "zeInitDrivers. Not adding duplicate driver to list", + ZeDriverGetHandles[Y]); + unMatchedDriverHandle = false; + break; + } + } + if (unMatchedDriverHandle) { + UR_LOG(DEBUG, + "zeDriverHandle {} not found in zeInitDrivers. Adding to " + "driver list.", + driverGetHandle); + ZeDrivers.push_back(driverGetHandle); + } + } + } + } else { + ZeDrivers.resize(ZeDriverGetCount); + ZeDrivers.assign(ZeDriverGetHandles.begin(), ZeDriverGetHandles.end()); + } + ZeDriverCount = ZeDrivers.size(); + UR_LOG(DEBUG, "{} L0 Drivers found.", ZeDriverCount); + for (uint32_t I = 0; I < ZeDriverCount; ++I) { + // Keep track of the first platform init for this Driver + bool DriverPlatformInit = false; + ze_device_properties_t device_properties{}; + device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + uint32_t ZeDeviceCount = 0; + ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, nullptr)); + ZeDevices.resize(ZeDeviceCount); + ZE2UR_CALL(zeDeviceGet, (ZeDrivers[I], &ZeDeviceCount, ZeDevices.data())); + auto platform = std::make_unique(ZeDrivers[I]); + // Check if this driver has GPU Devices + for (uint32_t D = 0; D < ZeDeviceCount; ++D) { + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevices[D], &device_properties)); + if (ZE_DEVICE_TYPE_GPU == device_properties.type) { + // Check if this driver's platform has already been init. + if (!DriverPlatformInit) { + // If this Driver is a GPU, save it as a usable platform. + ur_result_t Result; + UR_CALL_NOCHECK(Result = platform->initialize()); + + // Forget platform with L0 driver version listed in the skiplist + if (platform->IsDriverVersionSkipListed) { + break; + } + + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + + // Save a copy in the cache for future uses. + platforms.push_back(std::move(platform)); + // Mark this driver's platform as init to prevent additional platforms + // from being created per driver. + DriverPlatformInit = true; + } + if (ZesResult == ZE_RESULT_SUCCESS) { + // Populate the Zes/Ze device mapping for this Ze Device into the last + // added platform which represents the current driver being queried. + ur_zes_device_handle_data_t ZesDeviceData; + zes_uuid_t ZesUUID; + std::memcpy(&ZesUUID, &device_properties.uuid, sizeof(zes_uuid_t)); + if (getZesDeviceHandle(adapter, ZesUUID, &ZesDeviceData.ZesDevice, + &ZesDeviceData.SubDeviceId, + &ZesDeviceData.SubDevice) == + UR_RESULT_SUCCESS) { + platforms.back()->ZedeviceToZesDeviceMap.insert( + std::make_pair(ZeDevices[D], std::move(ZesDeviceData))); + } + } + } + } + } + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +ur_result_t adapterStateInit() { + +#if defined(_WIN32) + umfInit(); +#endif + + return UR_RESULT_SUCCESS; +} + +bool isBMGorNewer() { + auto urResult = checkDeviceIntelGPUIpVersionOrNewer(0x05004000); + if (urResult != UR_RESULT_SUCCESS && + urResult != UR_RESULT_ERROR_UNSUPPORTED_VERSION) { + UR_LOG(ERR, "Intel GPU IP Version check failed: {}\n", urResult); + throw urResult; + } + + return urResult == UR_RESULT_SUCCESS; +} + +void releaseStaticLoaderResourcesIfNeeded() { +#ifdef UR_STATIC_LEVEL_ZERO + // Given static linking of the L0 Loader, we must release the static loader + // resources if the adapter is not used. + zelLoaderContextTeardown(); +#endif +} + +std::pair shouldUseV1Adapter() { + auto specificAdapterVersionRequested = + ur_getenv("UR_LOADER_USE_LEVEL_ZERO_V2").has_value() || + ur_getenv("SYCL_UR_USE_LEVEL_ZERO_V2").has_value(); + + auto v2Requested = getenv_tobool("UR_LOADER_USE_LEVEL_ZERO_V2", false); + v2Requested |= getenv_tobool("SYCL_UR_USE_LEVEL_ZERO_V2", false); + + std::string reason = + specificAdapterVersionRequested + ? "Specific adapter version requested by UR_LOADER_USE_LEVEL_ZERO_V2 " + "or SYCL_UR_USE_LEVEL_ZERO_V2" + : "Using default adapter version based on device IP version"; + + if (v2Requested) { + return {false, reason}; + } + + if (!v2Requested && specificAdapterVersionRequested) { + // v1 specifically requested + return {true, reason}; + } + + // default: only enable for devices older than BMG + return {!isBMGorNewer(), reason}; +} + +std::pair shouldUseV2Adapter() { + auto [useV1, reason] = shouldUseV1Adapter(); + return {!useV1, reason}; +} + +/* +This constructor initializes the `ur_adapter_handle_t_` object and +sets up the environment for Level Zero (L0) initialization. +The behavior of the initialization process is influenced by two +environment variables: +`UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` and `UR_L0_ENABLE_ZESINIT_DEFAULT`. + +| Environment Variable | Value | Behavior | +|--------------------------------|-------|----------------------------| +| UR_L0_ENABLE_SYSMAN_ENV_DEFAULT| 1 | Enables the default SysMan | +| | | environment initialization | +| | | by setting | +| | | `ZES_ENABLE_SYSMAN` to "1".| +| | 0 | Disables the default SysMan| +| | | environment initialization.| +| | unset | Defaults to 1, enabling the| +| | | SysMan environment | +| | | initialization. | +| UR_L0_ENABLE_ZESINIT_DEFAULT | 1 | Enables the default SysMan | +| | | initialization by loading | +| | | SysMan-related functions | +| | | and calling `zesInit`. | +| | 0 | Disables the default SysMan| +| | | initialization with zesInit| +| | unset | Defaults to 0, disabling | +| | | the SysMan initialization | +| | | thru zesInit. | + +Behavior Summary: +- If `UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` is set to 1 or is unset, + `ZES_ENABLE_SYSMAN` is set to "1". +- If `UR_L0_ENABLE_ZESINIT_DEFAULT` is set to 1 and + `UR_L0_ENABLE_SYSMAN_ENV_DEFAULT` is not set to 1, + SysMan-related functions are loaded and `zesInit` is called. +- If `UR_L0_ENABLE_ZESINIT_DEFAULT` is set to 0 or is unset, + SysMan initialization is skipped. +*/ +ur_adapter_handle_t_::ur_adapter_handle_t_() + : logger(logger::get_logger("level_zero")), RefCount(0) { + auto ZeInitDriversResult = ZE_RESULT_ERROR_UNINITIALIZED; + auto ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; + auto ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; + +#ifdef UR_STATIC_LEVEL_ZERO + // Given static linking of the L0 Loader, we must delay the loader's + // destruction of its context until after the UR Adapter is destroyed. + zelSetDelayLoaderContextTeardown(); +#endif + + if (UrL0Debug & UR_L0_DEBUG_BASIC) { + logger.setLegacySink(std::make_unique()); + setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1"); + setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace"); + setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1"); + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + }; + + if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); + } + + if (UrL0LeaksDebug) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); + } + + uint32_t UserForcedSysManInit = 0; + // Check if the user has disabled the default L0 Env initialization. + const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { + const char *UrRet = std::getenv("UR_L0_ENABLE_SYSMAN_ENV_DEFAULT"); + if (!UrRet) + return 1; + UserForcedSysManInit &= 1; + return std::atoi(UrRet); + }(); + + // Dynamically load the new L0 apis separately. + // This must be done to avoid attempting to use symbols that do + // not exist in older loader runtimes. +#ifndef UR_STATIC_LEVEL_ZERO +#ifdef _WIN32 + processHandle = GetModuleHandle(NULL); +#else + processHandle = nullptr; +#endif +#endif + + // Setting these environment variables before running zeInit will enable + // the validation layer in the Level Zero loader. + if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); + } + + if (getenv("SYCL_ENABLE_PCI") != nullptr) { + UR_LOG(WARN, + "WARNING: SYCL_ENABLE_PCI is deprecated and no longer needed.\n"); + } + + // TODO: We can still safely recover if something goes wrong during the + // init. Implement handling segfault using sigaction. + + // We must only initialize the driver once, even if urPlatformGet() is + // called multiple times. Declaring the return value as "static" ensures + // it's only called once. + + // Set ZES_ENABLE_SYSMAN by default if the user has not set it. + if (UrSysManEnvInitEnabled) { + setEnvVar("ZES_ENABLE_SYSMAN", "1"); + } + + // Init with all flags set to enable for all driver types to be init in + // the application. + ze_init_flags_t L0InitFlags = ZE_INIT_FLAG_GPU_ONLY; + if (UrL0InitAllDrivers) { + L0InitFlags |= ZE_INIT_FLAG_VPU_ONLY; + } + UR_LOG(DEBUG, "zeInit with flags value of {}", static_cast(L0InitFlags)); + ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); + if (ZeInitResult != ZE_RESULT_SUCCESS) { + const char *ErrorString = "Unknown"; + zeParseError(ZeInitResult, ErrorString); + UR_LOG(ERR, "zeInit failed with {}", ErrorString); + } + + bool useInitDrivers = false; + zel_version_t loader_version = {}; + size_t num_components; + auto result = zelLoaderGetVersions(&num_components, nullptr); + if (result == ZE_RESULT_SUCCESS) { + zel_component_version_t *versions = + new zel_component_version_t[num_components]; + result = zelLoaderGetVersions(&num_components, versions); + if (result == ZE_RESULT_SUCCESS) { + for (size_t i = 0; i < num_components; ++i) { + if (strncmp(versions[i].component_name, "loader", strlen("loader")) == + 0) { + loader_version = versions[i].component_lib_version; + UR_LOG(DEBUG, "Level Zero Loader version: {}.{}.{}", + loader_version.major, loader_version.minor, + loader_version.patch); + break; + } + } + } + delete[] versions; + + auto LoaderSkiplist = getenv_to_vec("SYCL_UR_L0_LOADER_SKIPLIST"); + if (!LoaderSkiplist.has_value()) { + LoaderSkiplist = LoaderSkiplistDefault; + + if ((loader_version.major == 1 && loader_version.minor < 8)) { + UR_LOG(ERR, + "ERROR: Level Zero Loader version older than 1.8.0 is not " + "supported (current version: {}.{}.{}). Please update Intel GPU " + "compute drivers.", + loader_version.major, loader_version.minor, + loader_version.patch); + // Return 0 platforms as the loader is too old to be supported. + return; + } + } + + auto LoaderVersionString = std::to_string(loader_version.major) + "." + + std::to_string(loader_version.minor) + "." + + std::to_string(loader_version.patch); + for (const auto &version : LoaderSkiplist.value()) { + UR_LOG(DEBUG, "Checking loader version {} against skiplist entry {}", + LoaderVersionString, version); + if (version == LoaderVersionString) { + UR_LOG(WARN, + "Skipping Level Zero adapter due to the loader version: {} " + "matches a skiplist entry (set SYCL_UR_L0_LOADER_SKIPLIST to " + "override)", + version); + // Return 0 platforms + return; + } + } + + if (loader_version.major > 1 || + (loader_version.major == 1 && loader_version.minor > 19) || + (loader_version.major == 1 && loader_version.minor == 19 && + loader_version.patch >= 2)) { + useInitDrivers = true; + } + + if ((loader_version.major == 1 && loader_version.minor < 21) || + (loader_version.major == 1 && loader_version.minor == 21 && + loader_version.patch < 2)) { + UR_LOG(WARN, + "WARNING: Level Zero Loader version is older than 1.21.2. " + "Please update to the latest version for API logging support."); + } + } + + if (useInitDrivers) { +#ifdef UR_STATIC_LEVEL_ZERO + initDriversFunctionPtr = zeInitDrivers; +#else + initDriversFunctionPtr = + (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zeInitDrivers"); +#endif + if (initDriversFunctionPtr) { + UR_LOG(DEBUG, "zeInitDrivers with flags value of {}", + static_cast(InitDriversDesc.flags)); + ZeInitDriversResult = + ZE_CALL_NOCHECK(initDriversFunctionPtr, + (&ZeInitDriversCount, nullptr, &InitDriversDesc)); + if (ZeInitDriversResult == ZE_RESULT_SUCCESS) { + InitDriversSupported = true; + } else { + const char *ErrorString = "Unknown"; + zeParseError(ZeInitDriversResult, ErrorString); + UR_LOG(ERR, "zeInitDrivers failed with {}", ErrorString); + } + } + } + + if (ZeInitResult != ZE_RESULT_SUCCESS && + ZeInitDriversResult != ZE_RESULT_SUCCESS) { + // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. + UR_LOG(ERR, "Level Zero Uninitialized"); + return; + } + + PlatformVec platforms; + + // Check if the user has enabled the default L0 SysMan initialization. + const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { + const char *UrRet = std::getenv("UR_L0_ENABLE_ZESINIT_DEFAULT"); + if (!UrRet) + return 0; + UserForcedSysManInit &= 2; + return std::atoi(UrRet); + }(); + + bool ZesInitNeeded = UrSysmanZesinitEnable && !UrSysManEnvInitEnabled; + // Unless the user has forced the SysMan init, we will check the device + // version to see if the zesInit is needed. + if (UserForcedSysManInit == 0 && + checkDeviceIntelGPUIpVersionOrNewer(0x05004000) == UR_RESULT_SUCCESS) { + if (UrSysManEnvInitEnabled) { + setEnvVar("ZES_ENABLE_SYSMAN", "0"); + } + ZesInitNeeded = true; + } + if (ZesInitNeeded) { +#ifdef UR_STATIC_LEVEL_ZERO + getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp; + getSysManDriversFunctionPtr = zesDriverGet; + sysManInitFunctionPtr = zesInit; +#else + getDeviceByUUIdFunctionPtr = + (zes_pfnDriverGetDeviceByUuidExp_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zesDriverGetDeviceByUuidExp"); + getSysManDriversFunctionPtr = + (zes_pfnDriverGet_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zesDriverGet"); + sysManInitFunctionPtr = (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zesInit"); +#endif + } + if (getDeviceByUUIdFunctionPtr && getSysManDriversFunctionPtr && + sysManInitFunctionPtr) { + ze_init_flags_t L0ZesInitFlags = 0; + UR_LOG(DEBUG, "\nzesInit with flags value of {}\n", + static_cast(L0ZesInitFlags)); + ZesResult = ZE_CALL_NOCHECK(sysManInitFunctionPtr, (L0ZesInitFlags)); + } else { + ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; + } + + ur_result_t err = initPlatforms(this, platforms, ZesResult); + if (err == UR_RESULT_SUCCESS) { + UR_LOG(DEBUG, "Initialized {} platforms", platforms.size()); + Platforms = std::move(platforms); + } else { + UR_LOG(ERR, "Failed to initialize Platforms"); + throw err; + } +} + +ur_result_t adapterStateTeardown() { + // On Windows the adapter library is unloaded before UMF, so tear UMF + // down here. +#if defined(_WIN32) + umfTearDown(); +#endif + + return UR_RESULT_SUCCESS; +} + +#if defined(_WIN32) +void globalAdapterOnDemandCleanup() { + std::lock_guard Lock(GlobalAdapterMutex); + if (GlobalAdapter) { + delete GlobalAdapter; + GlobalAdapter = nullptr; + } +} +#endif + +ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapterOpque) { + if (!hAdapterOpque) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + auto hAdapter = common_cast(hAdapterOpque); + + // NOTE: This does not require guarding with a mutex; the instant the ref + // count hits zero, both Get and Retain are UB. + if (hAdapter->RefCount.release()) { + auto result = adapterStateTeardown(); +#ifdef UR_STATIC_LEVEL_ZERO + // Given static linking of the L0 Loader, we must delay the loader's + // destruction of its context until after the UR Adapter is destroyed. + zelLoaderContextTeardown(); +#endif + + if (hAdapter == GlobalAdapter) + GlobalAdapter = nullptr; + delete hAdapter; + + return result; + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapterOpque) { + if (!hAdapterOpque) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + common_cast(hAdapterOpque)->RefCount.retain(); + + return UR_RESULT_SUCCESS; +} + +ur_result_t urAdapterGetLastError( + /// [in] handle of the platform instance + ::ur_adapter_handle_t, + /// [out] pointer to a C string where the adapter specific error message + /// will be stored. + const char **Message, + /// [out] pointer to an integer where the adapter specific error code will + /// be stored. + int32_t *Error) { + *Message = ErrorMessage; + *Error = ErrorAdapterNativeCode; + + return UR_RESULT_SUCCESS; +} + +ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapterOpque, + ur_adapter_info_t PropName, size_t PropSize, + void *PropValue, size_t *PropSizeRet) { + auto hAdapter = common_cast(hAdapterOpque); + UrReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet); + + switch (PropName) { + case UR_ADAPTER_INFO_BACKEND: + return ReturnValue(UR_BACKEND_LEVEL_ZERO); + case UR_ADAPTER_INFO_REFERENCE_COUNT: + return ReturnValue(hAdapter->RefCount.getCount()); + case UR_ADAPTER_INFO_VERSION: + return ReturnValue(hAdapter->version); + default: + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urAdapterSetLoggerCallback( + ::ur_adapter_handle_t hAdapterOpque, ur_logger_callback_t pfnLoggerCallback, + void *pUserData, ur_logger_level_t level = UR_LOGGER_LEVEL_QUIET) { + + if (hAdapterOpque) { + auto hAdapter = common_cast(hAdapterOpque); + hAdapter->logger.setCallbackSink(pfnLoggerCallback, pUserData, level); + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapterOpque, + ur_logger_level_t level) { + + if (hAdapterOpque) { + common_cast(hAdapterOpque)->logger.setCallbackLevel(level); + } + + return UR_RESULT_SUCCESS; +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/adapter.hpp b/unified-runtime/source/adapters/level_zero/common/adapter.hpp similarity index 55% rename from unified-runtime/source/adapters/level_zero/adapter.hpp rename to unified-runtime/source/adapters/level_zero/common/adapter.hpp index f462222e7b4a4..919afd8aef26f 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.hpp +++ b/unified-runtime/source/adapters/level_zero/common/adapter.hpp @@ -1,4 +1,4 @@ -//===--------- adapters.hpp - Level Zero Adapter --------------------------===// +//===--------- adapter.hpp - Level Zero Adapter ---------------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -8,22 +8,33 @@ //===----------------------------------------------------------------------===// #pragma once +#include "api.hpp" #include "common/ur_ref_count.hpp" +#include "interfaces.hpp" #include "logger/ur_logger.hpp" -#include "ur_interface_loader.hpp" #include #include +#include +#include #include #include +#include +#include #include #include #include -using PlatformVec = std::vector>; +#include class ur_legacy_sink; +struct ur_platform_handle_t_; + +namespace ur::level_zero { -struct ur_adapter_handle_t_ : ur::handle_base { +using PlatformVec = std::vector>; + +// Concrete definition of the adapter handle shared by L0v1 and L0v2. +struct ur_adapter_handle_t_ : ur_object_t { ur_adapter_handle_t_(); zes_pfnDriverGetDeviceByUuidExp_t getDeviceByUUIdFunctionPtr = nullptr; @@ -40,7 +51,28 @@ struct ur_adapter_handle_t_ : ur::handle_base { logger::Logger &logger; HMODULE processHandle = nullptr; + // The adapter version returned via UR_ADAPTER_INFO_VERSION. + uint32_t version = 1; + ur::RefCount RefCount; }; +// External reference to GlobalAdapter used by both L0v1 and L0v2. Only one of +// the adapters is selected at the runtime. extern ur_adapter_handle_t_ *GlobalAdapter; +extern std::mutex GlobalAdapterMutex; + +std::pair shouldUseV1Adapter(); +std::pair shouldUseV2Adapter(); + +void releaseStaticLoaderResourcesIfNeeded(); + +ur_result_t adapterStateInit(); +ur_result_t initPlatforms(ur_adapter_handle_t_ *adapter, PlatformVec &platforms, + ze_result_t ZesResult) noexcept; + +#if defined(_WIN32) +void globalAdapterOnDemandCleanup(); +#endif + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/common/api.hpp b/unified-runtime/source/adapters/level_zero/common/api.hpp new file mode 100644 index 0000000000000..7f9e072a0bc16 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/common/api.hpp @@ -0,0 +1,295 @@ +//===--------- api.hpp - Level Zero Adapter ------------------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include + +// Declarations of every official UR entry point implemented in the common. + +namespace ur::level_zero { + +// Device +ur_result_t urDeviceGet(::ur_platform_handle_t hPlatformOpque, + ::ur_device_type_t DeviceType, uint32_t NumEntries, + ::ur_device_handle_t *phDevicesOpque, + uint32_t *pNumDevices); +ur_result_t urDeviceGetInfo(::ur_device_handle_t hDeviceOpque, + ::ur_device_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urDeviceRetain(::ur_device_handle_t hDeviceOpque); +ur_result_t urDeviceRelease(::ur_device_handle_t hDeviceOpque); +ur_result_t +urDevicePartition(::ur_device_handle_t hDeviceOpque, + const ::ur_device_partition_properties_t *pProperties, + uint32_t NumDevices, ::ur_device_handle_t *phSubDevicesOpque, + uint32_t *pNumDevicesRet); +ur_result_t urDeviceSelectBinary(::ur_device_handle_t hDeviceOpque, + const ::ur_device_binary_t *pBinaries, + uint32_t NumBinaries, + uint32_t *pSelectedBinary); +ur_result_t urDeviceGetNativeHandle(::ur_device_handle_t hDeviceOpque, + ::ur_native_handle_t *phNativeDevice); +ur_result_t urDeviceCreateWithNativeHandle( + ::ur_native_handle_t hNativeDevice, ::ur_adapter_handle_t hAdapterOpque, + const ::ur_device_native_properties_t *pProperties, + ::ur_device_handle_t *phDeviceOpque); +ur_result_t urDeviceGetGlobalTimestamps(::ur_device_handle_t hDeviceOpque, + uint64_t *pDeviceTimestamp, + uint64_t *pHostTimestamp); +ur_result_t urDeviceWaitExp(::ur_device_handle_t hDeviceOpque); + +// Platform +ur_result_t urPlatformGet(::ur_adapter_handle_t hAdapterOpque, + uint32_t NumEntries, + ::ur_platform_handle_t *phPlatformsOpque, + uint32_t *pNumPlatforms); +ur_result_t urPlatformGetInfo(::ur_platform_handle_t hPlatformOpque, + ur_platform_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urPlatformGetApiVersion(::ur_platform_handle_t hPlatformOpque, + ur_api_version_t *pVersion); +ur_result_t urPlatformGetNativeHandle(::ur_platform_handle_t hPlatformOpque, + ::ur_native_handle_t *phNativePlatform); +ur_result_t urPlatformCreateWithNativeHandle( + ::ur_native_handle_t hNativePlatform, ::ur_adapter_handle_t hAdapterOpque, + const ur_platform_native_properties_t *pProperties, + ::ur_platform_handle_t *phPlatformOpque); +ur_result_t urPlatformGetBackendOption(::ur_platform_handle_t hPlatformOpque, + const char *pFrontendOption, + const char **ppPlatformOption); + +// Adapter +ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapterOpque); +ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapterOpque); +ur_result_t urAdapterGetLastError(::ur_adapter_handle_t hAdapterOpque, + const char **ppMessage, int32_t *pError); +ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapterOpque, + ur_adapter_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urAdapterSetLoggerCallback(::ur_adapter_handle_t hAdapterOpque, + ur_logger_callback_t pfnLoggerCallback, + void *pUserData, + ur_logger_level_t level); +ur_result_t urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapterOpque, + ur_logger_level_t level); + +// Sampler +ur_result_t urSamplerCreate(::ur_context_handle_t ContextOpque, + const ur_sampler_desc_t *Props, + ::ur_sampler_handle_t *SamplerOpque); +ur_result_t urSamplerRetain(::ur_sampler_handle_t SamplerOpque); +ur_result_t urSamplerRelease(::ur_sampler_handle_t SamplerOpque); +ur_result_t urSamplerGetInfo(::ur_sampler_handle_t SamplerOpque, + ur_sampler_info_t PropName, size_t PropValueSize, + void *PropValue, size_t *PropSizeRet); +ur_result_t urSamplerGetNativeHandle(::ur_sampler_handle_t SamplerOpque, + ::ur_native_handle_t *NativeSampler); +ur_result_t urSamplerCreateWithNativeHandle( + ::ur_native_handle_t NativeSampler, ::ur_context_handle_t ContextOpque, + const ur_sampler_native_properties_t *Properties, + ::ur_sampler_handle_t *SamplerOpque); + +// Program +ur_result_t urProgramCreateWithIL(::ur_context_handle_t hContextOpque, + const void *pIL, size_t length, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgramOpque); +ur_result_t urProgramCreateWithBinary( + ::ur_context_handle_t hContextOpque, uint32_t numDevices, + ::ur_device_handle_t *phDevicesOpque, size_t *pLengths, + const uint8_t **ppBinaries, const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgramOpque); +ur_result_t urProgramBuild(::ur_context_handle_t hContextOpque, + ::ur_program_handle_t hProgramOpque, + const char *pOptions); +ur_result_t urProgramCompile(::ur_context_handle_t hContextOpque, + ::ur_program_handle_t hProgramOpque, + const char *pOptions); +ur_result_t urProgramLink(::ur_context_handle_t hContextOpque, uint32_t count, + const ::ur_program_handle_t *phProgramsOpque, + const char *pOptions, + ::ur_program_handle_t *phProgramOpque); +ur_result_t urProgramRetain(::ur_program_handle_t hProgramOpque); +ur_result_t urProgramRelease(::ur_program_handle_t hProgramOpque); +ur_result_t urProgramGetFunctionPointer(::ur_device_handle_t hDeviceOpque, + ::ur_program_handle_t hProgramOpque, + const char *pFunctionName, + void **ppFunctionPointer); +ur_result_t urProgramGetGlobalVariablePointer( + ::ur_device_handle_t hDeviceOpque, ::ur_program_handle_t hProgramOpque, + const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, + void **ppGlobalVariablePointerRet); +ur_result_t urProgramGetInfo(::ur_program_handle_t hProgramOpque, + ur_program_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urProgramGetBuildInfo(::ur_program_handle_t hProgramOpque, + ::ur_device_handle_t hDeviceOpque, + ur_program_build_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urProgramSetSpecializationConstants( + ::ur_program_handle_t hProgramOpque, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants); +ur_result_t urProgramGetNativeHandle(::ur_program_handle_t hProgramOpque, + ::ur_native_handle_t *phNativeProgram); +ur_result_t urProgramCreateWithNativeHandle( + ::ur_native_handle_t hNativeProgram, ::ur_context_handle_t hContextOpque, + const ur_program_native_properties_t *pProperties, + ::ur_program_handle_t *phProgramOpque); +ur_result_t +urProgramDynamicLinkExp(::ur_context_handle_t hContextOpque, uint32_t count, + const ::ur_program_handle_t *phProgramsOpque); +ur_result_t urProgramBuildExp(::ur_program_handle_t hProgramOpque, + uint32_t numDevices, + ::ur_device_handle_t *phDevicesOpque, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramCompileExp(::ur_program_handle_t hProgramOpque, + uint32_t numDevices, + ::ur_device_handle_t *phDevicesOpque, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramLinkExp(::ur_context_handle_t hContextOpque, + uint32_t numDevices, + ::ur_device_handle_t *phDevicesOpque, + ur_exp_program_flags_t flags, uint32_t count, + const ::ur_program_handle_t *phProgramsOpque, + const char *pOptions, + ::ur_program_handle_t *phProgramOpque); + +// Virtual memory +ur_result_t urVirtualMemGranularityGetInfo( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t allocationSize, ur_virtual_mem_granularity_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet); +ur_result_t urVirtualMemReserve(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size, + void **ppStart); +ur_result_t urVirtualMemFree(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size); +ur_result_t urVirtualMemMap(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size, + ::ur_physical_mem_handle_t hPhysicalMemOpque, + size_t offset, ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size); +ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size, + ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size, + ur_virtual_mem_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); + +// Memory export +ur_result_t urMemoryExportAllocExportableMemoryExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t +urMemoryExportFreeExportableMemoryExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + void *pMem); +ur_result_t urMemoryExportExportMemoryHandleExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet); + +// Bindless images / pitched USM +ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t poolOpque, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch); +ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesImageAllocateExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesImageFreeExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hImageMem); +ur_result_t urBindlessImagesUnsampledImageCreateExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_native_handle_t *phImage); +ur_result_t urBindlessImagesSampledImageCreateExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_sampler_desc_t *pSamplerDesc, + ur_exp_image_native_handle_t *phImage); +ur_result_t +urBindlessImagesImageGetInfoExp(::ur_context_handle_t hContextOpque, + ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesMipmapGetLevelExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hMem); +ur_result_t urBindlessImagesImportExternalMemoryExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t size, ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ::ur_exp_external_mem_handle_t *phExternalMemOpque); +ur_result_t urBindlessImagesMapExternalArrayExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_external_mem_handle_t hExternalMemOpque, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + uint64_t offset, uint64_t size, + ::ur_exp_external_mem_handle_t hExternalMemOpque, void **ppRetMem); +ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ::ur_exp_external_mem_handle_t hExternalMemOpque); +ur_result_t +urBindlessImagesFreeMappedLinearMemoryExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + void *pMem); +ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( + ::ur_device_handle_t hDeviceOpque, ur_exp_external_mem_type_t memHandleType, + ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ::ur_exp_external_semaphore_handle_t *phExternalSemaphoreOpque); +ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ::ur_exp_external_semaphore_handle_t hExternalSemaphoreOpque); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/command_buffer_command.cpp b/unified-runtime/source/adapters/level_zero/common/command_buffer_command.cpp similarity index 78% rename from unified-runtime/source/adapters/level_zero/command_buffer_command.cpp rename to unified-runtime/source/adapters/level_zero/common/command_buffer_command.cpp index c87245a05d0b7..c19938bcc4d61 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer_command.cpp +++ b/unified-runtime/source/adapters/level_zero/common/command_buffer_command.cpp @@ -8,10 +8,12 @@ //===----------------------------------------------------------------------===// #include "command_buffer_command.hpp" +#include "interfaces.hpp" #include "unified-runtime/ur_api.h" -#include "ur_interface_loader.hpp" -#include "ur_level_zero.hpp" +#include "unified-runtime/ur_ddi.h" + +namespace ur::level_zero { kernel_command_handle::kernel_command_handle( ur_exp_command_buffer_handle_t commandBuffer, ur_kernel_handle_t kernel, @@ -20,12 +22,13 @@ kernel_command_handle::kernel_command_handle( : ur_exp_command_buffer_command_handle_t_(commandBuffer, commandId), workDim(workDim), kernel(kernel) { // Add the default kernel to the list of valid kernels - ur::level_zero::urKernelRetain(kernel); + ddiTableOf(kernel)->Kernel.pfnRetain(kernel); validKernelHandles.insert(kernel); // Add alternative kernels if provided if (kernelAlternatives) { for (size_t i = 0; i < numKernelAlternatives; i++) { - ur::level_zero::urKernelRetain(kernelAlternatives[i]); + ddiTableOf(kernelAlternatives[i]) + ->Kernel.pfnRetain(kernelAlternatives[i]); validKernelHandles.insert(kernelAlternatives[i]); } } @@ -33,6 +36,8 @@ kernel_command_handle::kernel_command_handle( kernel_command_handle::~kernel_command_handle() { for (const ur_kernel_handle_t &kernelHandle : validKernelHandles) { - ur::level_zero::urKernelRelease(kernelHandle); + ddiTableOf(kernelHandle)->Kernel.pfnRelease(kernelHandle); } } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/command_buffer_command.hpp b/unified-runtime/source/adapters/level_zero/common/command_buffer_command.hpp similarity index 82% rename from unified-runtime/source/adapters/level_zero/command_buffer_command.hpp rename to unified-runtime/source/adapters/level_zero/common/command_buffer_command.hpp index 76cd9e2c04632..5256c5f3891da 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer_command.hpp +++ b/unified-runtime/source/adapters/level_zero/common/command_buffer_command.hpp @@ -7,14 +7,22 @@ // //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +#include "interfaces.hpp" +#include #include #include +#include -struct ur_exp_command_buffer_command_handle_t_ : public ur_object { +namespace ur::level_zero { + +struct ur_exp_command_buffer_command_handle_t_ : public ur_object_t { ur_exp_command_buffer_command_handle_t_( ur_exp_command_buffer_handle_t commandBuffer, uint64_t commandId) - : commandBuffer(commandBuffer), commandId(commandId) {} + : commandBuffer(commandBuffer), commandId(commandId) { + // Auto-populate ddi_table from the owning command buffer so the loader's + // intercept layer can dispatch this handle back to the correct adapter. + ddi_table = ddiTableOf(commandBuffer); + } virtual ~ur_exp_command_buffer_command_handle_t_() {} @@ -50,3 +58,5 @@ struct kernel_command_handle : public ur_exp_command_buffer_command_handle_t_ { // Storage for valid kernel alternatives for this command. std::unordered_set validKernelHandles; }; + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/common/device.cpp b/unified-runtime/source/adapters/level_zero/common/device.cpp new file mode 100644 index 0000000000000..4adf7d1611535 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/common/device.cpp @@ -0,0 +1,2387 @@ +//===--------- device.cpp - Level Zero Adapter ----------------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "device.hpp" +#include "adapter.hpp" +#include "logger/ur_logger.hpp" +#include "platform.hpp" +#include "ur_util.hpp" +#include +#include +#if defined(__linux__) +#include +#elif defined(_WIN32) +#include +#endif +#include +#include + +namespace ur::level_zero { + +// UR_L0_USE_COPY_ENGINE can be set to an integer value, or +// a pair of integer values of the form "lower_index:upper_index". +// Here, the indices point to copy engines in a list of all available copy +// engines. +// This functions returns this pair of indices. +// If the user specifies only a single integer, a value of 0 indicates that +// the copy engines will not be used at all. A value of 1 indicates that all +// available copy engines can be used. +const std::pair +getRangeOfAllowedCopyEngines(const ur_device_handle_t &Device) { + const char *UrRet = std::getenv("UR_L0_USE_COPY_ENGINE"); + const char *PiRet = std::getenv("SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE"); + static const char *EnvVar = UrRet ? UrRet : (PiRet ? PiRet : nullptr); + // If the environment variable is not set, no copy engines are used when + // immediate commandlists are being used. For standard commandlists all are + // used. + if (!EnvVar) { + if (Device->ImmCommandListUsed) + return std::pair(0, 0); // Only main copy engine will be used. + return std::pair(0, INT_MAX); // All copy engines will be used. + } + std::string CopyEngineRange = EnvVar; + // Environment variable can be a single integer or a pair of integers + // separated by ":" + auto pos = CopyEngineRange.find(":"); + if (pos == std::string::npos) { + bool UseCopyEngine = (std::stoi(CopyEngineRange) != 0); + if (UseCopyEngine) + return std::pair(0, INT_MAX); // All copy engines can be used. + return std::pair(-1, -1); // No copy engines will be used. + } + int LowerCopyEngineIndex = std::stoi(CopyEngineRange.substr(0, pos)); + int UpperCopyEngineIndex = std::stoi(CopyEngineRange.substr(pos + 1)); + if ((LowerCopyEngineIndex > UpperCopyEngineIndex) || + (LowerCopyEngineIndex < -1) || (UpperCopyEngineIndex < -1)) { + UR_LOG(ERR, "UR_L0_LEVEL_ZERO_USE_COPY_ENGINE: invalid value provided, " + "default set."); + LowerCopyEngineIndex = 0; + UpperCopyEngineIndex = INT_MAX; + } + return std::pair(LowerCopyEngineIndex, UpperCopyEngineIndex); +} + +ur_result_t urDeviceGet( + /// [in] handle of the platform instance + ::ur_platform_handle_t PlatformOpque, + /// [in] the type of the devices. + ur_device_type_t DeviceType, + /// [in] the number of devices to be added to phDevices. If phDevices in not + /// NULL then NumEntries should be greater than zero, otherwise + /// ::UR_RESULT_ERROR_INVALID_SIZE, will be returned. + uint32_t NumEntries, + /// [out][optional][range(0, NumEntries)] array of handle of devices. If + /// NumEntries is less than the number of devices available, then + /// platform shall only retrieve that number of devices. + ::ur_device_handle_t *DevicesOpque, + /// [out][optional] pointer to the number of devices. pNumDevices will be + /// updated with the total number of devices available. + uint32_t *NumDevices) { + auto Platform = common_cast(PlatformOpque); + auto Devices = common_cast(DevicesOpque); + + auto Res = Platform->populateDeviceCacheIfNeeded(); + if (Res != UR_RESULT_SUCCESS) { + return Res; + } + + // Filter available devices based on input DeviceType. + std::vector MatchedDevices; + std::shared_lock Lock(Platform->URDevicesCacheMutex); + // We need to filter out composite devices when + // ZE_FLAT_DEVICE_HIERARCHY=COMBINED. We can know if we are in combined + // mode depending on the return value of zeDeviceGetRootDevice: + // - If COMPOSITE, L0 returns cards as devices. Since we filter out + // subdevices early, zeDeviceGetRootDevice must return nullptr, because we + // only query for root-devices and they don't have any device higher up in + // the hierarchy. + // - If FLAT, according to L0 spec, zeDeviceGetRootDevice always returns + // nullptr in this mode. + // - If COMBINED, L0 returns tiles as devices, and zeDeviceGetRootdevice + // returns the card containing a given tile. + + // Track best discrete and integrated GPU candidates (device, max compute + // units) + std::pair GPUDeviceDiscrete = {nullptr, 0}; + std::pair GPUDeviceIntegrated = {nullptr, 0}; + bool DeviceDefaultGPU = false; + + bool isCombinedMode = + std::any_of(Platform->URDevicesCache.begin(), + Platform->URDevicesCache.end(), [](const auto &D) { + if (D->isSubDevice()) + return false; + ze_device_handle_t RootDev = nullptr; + // Query Root Device for root-devices. + // We cannot use ZE2UR_CALL because under some circumstances + // this call may return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, + // and ZE2UR_CALL will abort because it's not + // UR_RESULT_SUCCESS. Instead, we use ZE_CALL_NOCHECK and we + // check manually that the result is either + // ZE_RESULT_SUCCESS or ZE_RESULT_ERROR_UNSUPPORTED_FEATURE. + auto errc = ZE_CALL_NOCHECK(zeDeviceGetRootDevice, + (D->ZeDevice, &RootDev)); + return (errc == ZE_RESULT_SUCCESS && RootDev != nullptr); + }); + for (auto &D : Platform->URDevicesCache) { + // Only ever return root-devices from urDeviceGet, but the + // devices cache also keeps sub-devices. + if (D->isSubDevice()) + continue; + + bool Matched = false; + switch (DeviceType) { + case UR_DEVICE_TYPE_ALL: + Matched = true; + break; + case UR_DEVICE_TYPE_GPU: + Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU); + break; + case UR_DEVICE_TYPE_DEFAULT: + Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU); + DeviceDefaultGPU = true; + break; + case UR_DEVICE_TYPE_CPU: + Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_CPU); + break; + case UR_DEVICE_TYPE_FPGA: + Matched = D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_FPGA; + break; + case UR_DEVICE_TYPE_MCA: + Matched = D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_MCA; + break; + default: + Matched = false; + UR_LOG(WARN, "Unknown device type"); + break; + } + + if (Matched) { + bool isComposite = + isCombinedMode && (D->ZeDeviceProperties->flags & + ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) == 0; + if (!isComposite) { + // In case of DeviceType is DEFAULT, pick only the most powerful GPU + // device. + if (DeviceDefaultGPU) { + uint32_t maxComputeUnits = 0; + ur_result_t UrRet = ur::level_zero::urDeviceGetInfo( + common_cast(D.get()), UR_DEVICE_INFO_MAX_COMPUTE_UNITS, + sizeof(maxComputeUnits), &maxComputeUnits, nullptr); + maxComputeUnits = (UrRet == UR_RESULT_SUCCESS) ? maxComputeUnits : 0; + auto &BestGpu = + D->isIntegrated() ? GPUDeviceIntegrated : GPUDeviceDiscrete; + if (!BestGpu.first || maxComputeUnits > BestGpu.second) + BestGpu = std::make_pair(D.get(), maxComputeUnits); + } else { + MatchedDevices.push_back(D.get()); + } + } + } + } + + // Handle GPU/DEFAULT device selection outside the loop + if (DeviceDefaultGPU) { + // Prefer discrete GPU over integrated GPU + if (GPUDeviceDiscrete.first) { + MatchedDevices = {GPUDeviceDiscrete.first}; + } else if (GPUDeviceIntegrated.first) { + MatchedDevices = {GPUDeviceIntegrated.first}; + } + } + + uint32_t ZeDeviceCount = MatchedDevices.size(); + + auto N = (std::min)(ZeDeviceCount, NumEntries); + if (Devices) + std::copy_n(MatchedDevices.begin(), N, Devices); + + if (NumDevices) + *NumDevices = ZeDeviceCount; + + return UR_RESULT_SUCCESS; +} + +uint64_t calculateGlobalMemSize(ur_device_handle_t Device) { + // Cache GlobalMemSize + Device->ZeGlobalMemSize.Compute = + [Device](struct ze_global_memsize &GlobalMemSize) { + for (const auto &ZeDeviceMemoryExtProperty : + Device->ZeDeviceMemoryProperties->second) { + GlobalMemSize.value += ZeDeviceMemoryExtProperty.physicalSize; + } + if (GlobalMemSize.value == 0) { + for (const auto &ZeDeviceMemoryProperty : + Device->ZeDeviceMemoryProperties->first) { + GlobalMemSize.value += ZeDeviceMemoryProperty.totalSize; + } + } + }; + return Device->ZeGlobalMemSize.get().value; +} + +static bool +supportsDeviceUsableMemSizeExtension(ur_platform_handle_t Platform) { +#ifdef ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME + constexpr const char *ExtensionName = + ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME; + constexpr uint32_t MinVersion = ZE_MAKE_VERSION(1, 0); + + auto Extension = Platform->zeDriverExtensionMap.find(ExtensionName); + return Extension != Platform->zeDriverExtensionMap.end() && + Extension->second >= MinVersion; +#else + std::ignore = Platform; + return false; +#endif +} + +static std::optional +getDeviceUsableMemSizeFromCore(ur_device_handle_t Device) { +#ifdef ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME + if (!supportsDeviceUsableMemSizeExtension(Device->Platform)) { + return std::nullopt; + } + + ZeStruct DeviceProperties; + ZeStruct UsableMemProperties; + DeviceProperties.pNext = &UsableMemProperties; + + auto ZeResult = ZE_CALL_NOCHECK(zeDeviceGetProperties, + (Device->ZeDevice, &DeviceProperties)); + if (ZeResult != ZE_RESULT_SUCCESS) { + return std::nullopt; + } + + return UsableMemProperties.currUsableMemSize; +#else + std::ignore = Device; + return std::nullopt; +#endif +} + +// Return the Sysman device handle and correpsonding data for the given UR +// device. +static std::tuple +getZesDeviceData(ur_device_handle_t Device) { + bool SysManEnv = getenv_tobool("ZES_ENABLE_SYSMAN", false); + if ((Device->Platform->ZedeviceToZesDeviceMap.size() == 0) && !SysManEnv) { + UR_LOG(ERR, "SysMan support is unavailable on this system. Please " + "check your level zero driver installation."); + return {nullptr, {}, UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION}; + } + + zes_device_handle_t ZesDevice = Device->ZeDevice; + ur_zes_device_handle_data_t ZesDeviceData = {}; + // If legacy sysman is enabled thru the environment variable, then zesInit + // will fail, but sysman is still usable so go the legacy route. + if (!SysManEnv) { + auto It = Device->Platform->ZedeviceToZesDeviceMap.find(Device->ZeDevice); + if (It == Device->Platform->ZedeviceToZesDeviceMap.end()) { + // no matching device + return {nullptr, {}, UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION}; + } else { + ZesDeviceData = + Device->Platform->ZedeviceToZesDeviceMap[Device->ZeDevice]; + ZesDevice = ZesDeviceData.ZesDevice; + } + } else { + ZesDeviceData.SubDevice = Device->isSubDevice(); + ZesDeviceData.SubDeviceId = Device->ZeDeviceProperties->subdeviceId; + } + + return {ZesDevice, ZesDeviceData, UR_RESULT_SUCCESS}; +} + +ur_result_t urDeviceGetInfo( + /// [in] handle of the device instance + ::ur_device_handle_t DeviceOpque, + /// [in] type of the info to retrieve + ur_device_info_t ParamName, + /// [in] the number of bytes pointed to by ParamValue. + size_t propSize, + /// [out][optional] array of bytes holding the info. If propSize is not + /// equal to or greater than the real number of bytes needed to return the + /// info then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pDeviceInfo is not used. + void *ParamValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// infoType. + size_t *pSize) { + auto Device = common_cast(DeviceOpque); + UrReturnHelper ReturnValue(propSize, ParamValue, pSize); + + ze_device_handle_t ZeDevice = Device->ZeDevice; + + switch ((int)ParamName) { + case UR_DEVICE_INFO_TYPE: { + switch (Device->ZeDeviceProperties->type) { + case ZE_DEVICE_TYPE_GPU: + return ReturnValue(UR_DEVICE_TYPE_GPU); + case ZE_DEVICE_TYPE_CPU: + return ReturnValue(UR_DEVICE_TYPE_CPU); + case ZE_DEVICE_TYPE_FPGA: + return ReturnValue(UR_DEVICE_TYPE_FPGA); + default: + UR_LOG(ERR, "This device type is not supported"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } + case UR_DEVICE_INFO_PARENT_DEVICE: + return ReturnValue(Device->RootDevice); + case UR_DEVICE_INFO_PLATFORM: + return ReturnValue(Device->Platform); + case UR_DEVICE_INFO_VENDOR_ID: + return ReturnValue(uint32_t{Device->ZeDeviceProperties->vendorId}); + case UR_DEVICE_INFO_UUID: { + // Intel extension for device UUID. This returns the UUID as + // std::array. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + const auto &UUID = Device->ZeDeviceProperties->uuid.id; + return ReturnValue(UUID, sizeof(UUID)); + } + case UR_DEVICE_INFO_MAX_MEMORY_BANDWIDTH: { + // ZeDeviceMemoryProperties should be set already by initialization + if (Device->ZeDeviceMemoryProperties->second.empty()) + return ReturnValue(uint64_t{0}); + + uint32_t maxBandwidth = 0; + for (const auto &extProp : Device->ZeDeviceMemoryProperties->second) { + // Only consider bandwidth if the unit is BYTES_PER_NANOSEC + if (extProp.bandwidthUnit == ZE_BANDWIDTH_UNIT_BYTES_PER_NANOSEC) { + maxBandwidth = std::max( + {maxBandwidth, extProp.readBandwidth, extProp.writeBandwidth}); + } + } + // Convert to Bytes/sec from Bytes/nanosec + return ReturnValue(static_cast(maxBandwidth * 1e9)); + } + case UR_DEVICE_INFO_ATOMIC_64: + return ReturnValue( + static_cast(Device->ZeDeviceModuleProperties->flags & + ZE_DEVICE_MODULE_FLAG_INT64_ATOMICS)); + case UR_DEVICE_INFO_EXTENSIONS: { + // Convention adopted from OpenCL: + // "Returns a space separated list of extension names (the extension + // names themselves do not contain any spaces) supported by the device." + // + // TODO: Use proper mechanism to get this information from Level Zero after + // it is added to Level Zero. + // Hardcoding the few we know are supported by the current hardware. + // + // + std::string SupportedExtensions; + + // cl_khr_il_program - OpenCL 2.0 KHR extension for SPIR-V support. Core + // feature in >OpenCL 2.1 + // cl_khr_subgroups - Extension adds support for implementation-controlled + // subgroups. + // cl_intel_subgroups - Extension adds subgroup features, defined by Intel. + // cl_intel_subgroups_short - Extension adds subgroup functions described in + // the cl_intel_subgroups extension to support 16-bit integer data types + // for performance. + // cl_intel_required_subgroup_size - Extension to allow programmers to + // optionally specify the required subgroup size for a kernel function. + // cl_khr_fp16 - Optional half floating-point support. + // cl_khr_fp64 - Support for double floating-point precision. + // cl_khr_int64_base_atomics, cl_khr_int64_extended_atomics - Optional + // extensions that implement atomic operations on 64-bit signed and + // unsigned integers to locations in __global and __local memory. + // cl_khr_3d_image_writes - Extension to enable writes to 3D image memory + // objects. + // + // Hardcoding some extensions we know are supported by all Level Zero + // devices. + SupportedExtensions += (ZE_SUPPORTED_EXTENSIONS); + if (Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16) + SupportedExtensions += ("cl_khr_fp16 "); + if (Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP64) + SupportedExtensions += ("cl_khr_fp64 "); + if (Device->ZeDeviceModuleProperties->flags & + ZE_DEVICE_MODULE_FLAG_INT64_ATOMICS) + // int64AtomicsSupported indicates support for both. + SupportedExtensions += + ("cl_khr_int64_base_atomics cl_khr_int64_extended_atomics "); + if (Device->ZeDeviceImageProperties->maxImageDims3D > 0) + // Supports reading and writing of images. + SupportedExtensions += ("cl_khr_3d_image_writes "); + + if (Device->Platform->zeDriverExtensionMap.count( + ZE_BFLOAT16_CONVERSIONS_EXT_NAME)) + SupportedExtensions += ("cl_intel_bfloat16_conversions "); + else if ((Device->ZeDeviceProperties->deviceId & 0xfff) == 0x201 || + (Device->ZeDeviceProperties->deviceId & 0xff0) == 0xbd0) + SupportedExtensions += ("cl_intel_bfloat16_conversions "); + + return ReturnValue(SupportedExtensions.c_str()); + } + case UR_DEVICE_INFO_NAME: + return ReturnValue(Device->ZeDeviceProperties->name); + // zeModuleCreate allows using root device module for sub-devices: + // > The application must only use the module for the device, or its + // > sub-devices, which was provided during creation. + case UR_DEVICE_INFO_BUILD_ON_SUBDEVICE: + return ReturnValue(ur_bool_t{0}); + case UR_DEVICE_INFO_COMPILER_AVAILABLE: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_LINKER_AVAILABLE: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_MAX_COMPUTE_UNITS: { + uint32_t MaxComputeUnits = + Device->ZeDeviceProperties->numEUsPerSubslice * + Device->ZeDeviceProperties->numSubslicesPerSlice * + Device->ZeDeviceProperties->numSlices; + + bool RepresentsCSlice = + Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] + .ZeIndex >= 0; + if (RepresentsCSlice) + MaxComputeUnits /= Device->RootDevice->SubDevices.size(); + + return ReturnValue(uint32_t{MaxComputeUnits}); + } + case UR_DEVICE_INFO_NUM_COMPUTE_UNITS: { + uint32_t NumComputeUnits = + Device->ZeDeviceProperties->numSubslicesPerSlice * + Device->ZeDeviceProperties->numSlices; + return ReturnValue(uint32_t{NumComputeUnits}); + } + case UR_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS: + // Level Zero spec defines only three dimensions + return ReturnValue(uint32_t{3}); + case UR_DEVICE_INFO_MAX_WORK_GROUP_SIZE: + return ReturnValue( + uint64_t{Device->ZeDeviceComputeProperties->maxTotalGroupSize}); + case UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES: { + struct { + size_t Arr[3]; + } MaxGroupSize = {{Device->ZeDeviceComputeProperties->maxGroupSizeX, + Device->ZeDeviceComputeProperties->maxGroupSizeY, + Device->ZeDeviceComputeProperties->maxGroupSizeZ}}; + return ReturnValue(MaxGroupSize); + } + case UR_DEVICE_INFO_MAX_WORK_GROUPS_3D: { + struct { + size_t Arr[3]; + } MaxGroupCounts = {{Device->ZeDeviceComputeProperties->maxGroupCountX, + Device->ZeDeviceComputeProperties->maxGroupCountY, + Device->ZeDeviceComputeProperties->maxGroupCountZ}}; + return ReturnValue(MaxGroupCounts); + } + case UR_DEVICE_INFO_MAX_WORK_GROUPS: { + // Multiply the max group counts in each dimension to get the total max + // number of work groups. Prevent overflow. + return ReturnValue(multiplyWithOverflowCheck( + Device->ZeDeviceComputeProperties->maxGroupCountX, + Device->ZeDeviceComputeProperties->maxGroupCountY, + Device->ZeDeviceComputeProperties->maxGroupCountZ)); + } + case UR_DEVICE_INFO_MAX_CLOCK_FREQUENCY: + return ReturnValue(uint32_t{Device->ZeDeviceProperties->coreClockRate}); + case UR_DEVICE_INFO_ADDRESS_BITS: { + // TODO: To confirm with spec. + return ReturnValue(uint32_t{64}); + } + case UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE: + // if the user wishes to allocate large allocations on a system that usually + // does not allow that allocation size, then we return the max global mem + // size as the limit. + if (Device->useRelaxedAllocationLimits()) { + return ReturnValue(uint64_t{calculateGlobalMemSize(Device)}); + } else { + return ReturnValue(uint64_t{Device->ZeDeviceProperties->maxMemAllocSize}); + } + case UR_DEVICE_INFO_GLOBAL_MEM_SIZE: { + // Support to read physicalSize depends on kernel, + // so fallback into reading totalSize if physicalSize + // is not available. + uint64_t GlobalMemSize = calculateGlobalMemSize(Device); + return ReturnValue(uint64_t{GlobalMemSize}); + } + case UR_DEVICE_INFO_LOCAL_MEM_SIZE: + return ReturnValue( + uint64_t{Device->ZeDeviceComputeProperties->maxSharedLocalMemory}); + case UR_DEVICE_INFO_IMAGE_SUPPORT: + return ReturnValue(Device->ZeDeviceImageProperties->maxImageDims1D > 0); + case UR_DEVICE_INFO_HOST_UNIFIED_MEMORY: + return ReturnValue( + static_cast((Device->ZeDeviceProperties->flags & + ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) != 0)); + case UR_DEVICE_INFO_AVAILABLE: + return ReturnValue(static_cast(ZeDevice ? true : false)); + case UR_DEVICE_INFO_VENDOR: + // TODO: Level-Zero does not return vendor's name at the moment + // only the ID. + return ReturnValue("Intel(R) Corporation"); + case UR_DEVICE_INFO_DRIVER_VERSION: + case UR_DEVICE_INFO_BACKEND_RUNTIME_VERSION: + return ReturnValue(Device->Platform->ZeDriverVersion.c_str()); + case UR_DEVICE_INFO_VERSION: { + // from compute-runtime/shared/source/helpers/hw_ip_version.h + typedef struct { + uint32_t revision : 6; + uint32_t reserved : 8; + uint32_t release : 8; + uint32_t architecture : 10; + } version_components_t; + typedef struct { + union { + uint32_t value; + version_components_t components; + }; + } ipVersion_t; + ipVersion_t IpVersion; + IpVersion.value = Device->ZeDeviceIpVersionExt->ipVersion; + std::stringstream S; + S << IpVersion.components.architecture << "." + << IpVersion.components.release << "." << IpVersion.components.revision; + return ReturnValue(S.str().c_str()); + } + case UR_DEVICE_INFO_PARTITION_MAX_SUB_DEVICES: { + auto Res = Device->Platform->populateDeviceCacheIfNeeded(); + if (Res != UR_RESULT_SUCCESS) { + return Res; + } + return ReturnValue((uint32_t)Device->SubDevices.size()); + } + case UR_DEVICE_INFO_REFERENCE_COUNT: + return ReturnValue(uint32_t{Device->RefCount.getCount()}); + case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: { + // SYCL spec says: if this SYCL device cannot be partitioned into at least + // two sub devices then the returned vector must be empty. + auto Res = Device->Platform->populateDeviceCacheIfNeeded(); + if (Res != UR_RESULT_SUCCESS) { + return Res; + } + + uint32_t ZeSubDeviceCount = Device->SubDevices.size(); + if (pSize && ZeSubDeviceCount < 2) { + *pSize = 0; + return UR_RESULT_SUCCESS; + } + bool PartitionedByCSlice = Device->SubDevices[0]->isCCS(); + + auto ReturnHelper = [&](auto... Partitions) { + struct { + ur_device_partition_t Arr[sizeof...(Partitions)]; + } PartitionProperties = {{Partitions...}}; + return ReturnValue(PartitionProperties); + }; + + if (ExposeCSliceInAffinityPartitioning) { + if (PartitionedByCSlice) + return ReturnHelper(UR_DEVICE_PARTITION_BY_CSLICE, + UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); + + else + return ReturnHelper(UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); + } else { + return ReturnHelper(PartitionedByCSlice + ? UR_DEVICE_PARTITION_BY_CSLICE + : UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); + } + break; + } + case UR_DEVICE_INFO_PARTITION_AFFINITY_DOMAIN: + return ReturnValue(ur_device_affinity_domain_flag_t( + UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA | + UR_DEVICE_AFFINITY_DOMAIN_FLAG_NEXT_PARTITIONABLE)); + case UR_DEVICE_INFO_PARTITION_TYPE: { + // For root-device there is no partitioning to report. + if (Device->SubDeviceCreationProperty == std::nullopt || + !Device->isSubDevice()) { + if (pSize) + *pSize = 0; + return UR_RESULT_SUCCESS; + } + + if (Device->isCCS()) { + ur_device_partition_property_t cslice{}; + cslice.type = UR_DEVICE_PARTITION_BY_CSLICE; + + return ReturnValue(cslice); + } + + return ReturnValue(*Device->SubDeviceCreationProperty); + } + // Everything under here is not supported yet + case UR_DEVICE_INFO_PREFERRED_INTEROP_USER_SYNC: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_PRINTF_BUFFER_SIZE: + return ReturnValue( + size_t{Device->ZeDeviceModuleProperties->printfBufferSize}); + case UR_DEVICE_INFO_PROFILE: + return ReturnValue("FULL_PROFILE"); + case UR_DEVICE_INFO_BUILT_IN_KERNELS: + // TODO: To find out correct value + return ReturnValue(""); + case UR_DEVICE_INFO_LOW_POWER_EVENTS_SUPPORT_EXP: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_QUEUE_PROPERTIES: + return ReturnValue( + ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE | + UR_QUEUE_FLAG_PROFILING_ENABLE)); + case UR_DEVICE_INFO_EXECUTION_CAPABILITIES: + return ReturnValue(ur_device_exec_capability_flag_t{ + UR_DEVICE_EXEC_CAPABILITY_FLAG_NATIVE_KERNEL}); + case UR_DEVICE_INFO_ENDIAN_LITTLE: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_ERROR_CORRECTION_SUPPORT: + return ReturnValue(static_cast( + Device->ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_ECC)); + case UR_DEVICE_INFO_PROFILING_TIMER_RESOLUTION: + return ReturnValue(static_cast(Device->getTimerResolution())); + case UR_DEVICE_INFO_LOCAL_MEM_TYPE: + return ReturnValue(UR_DEVICE_LOCAL_MEM_TYPE_LOCAL); + case UR_DEVICE_INFO_MAX_CONSTANT_ARGS: + return ReturnValue(uint32_t{64}); + case UR_DEVICE_INFO_MAX_CONSTANT_BUFFER_SIZE: + return ReturnValue( + uint64_t{Device->ZeDeviceImageProperties->maxImageBufferSize}); + case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_TYPE: + return ReturnValue(UR_DEVICE_MEM_CACHE_TYPE_READ_WRITE_CACHE); + case UR_DEVICE_INFO_GLOBAL_MEM_CACHELINE_SIZE: + if (Device->Platform->zeDriverExtensionMap.count( + ZE_CACHELINE_SIZE_EXT_NAME)) { + return ReturnValue(uint32_t{static_cast( + Device->ZeDeviceCacheLinePropertiesExt->cacheLineSize)}); + } else { + return ReturnValue(uint32_t{1}); + } + case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_SIZE: + return ReturnValue(uint64_t{Device->ZeDeviceCacheProperties->cacheSize}); + case UR_DEVICE_INFO_IP_VERSION: + return ReturnValue(uint32_t{Device->ZeDeviceIpVersionExt->ipVersion}); + case UR_DEVICE_INFO_MAX_PARAMETER_SIZE: + return ReturnValue( + size_t{Device->ZeDeviceModuleProperties->maxArgumentsSize}); + case UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN: + // SYCL/OpenCL spec is vague on what this means exactly, but seems to + // be for "alignment requirement (in bits) for sub-buffer offsets." + // An OpenCL implementation returns 8*128, but Level Zero can do just 8, + // meaning unaligned access for values of types larger than 8 bits. + return ReturnValue(uint32_t{8}); + case UR_DEVICE_INFO_MAX_SAMPLERS: + return ReturnValue(uint32_t{Device->ZeDeviceImageProperties->maxSamplers}); + case UR_DEVICE_INFO_MAX_READ_IMAGE_ARGS: + return ReturnValue( + uint32_t{Device->ZeDeviceImageProperties->maxReadImageArgs}); + case UR_DEVICE_INFO_MAX_WRITE_IMAGE_ARGS: + return ReturnValue( + uint32_t{Device->ZeDeviceImageProperties->maxWriteImageArgs}); + case UR_DEVICE_INFO_SINGLE_FP_CONFIG: { + ur_device_fp_capability_flags_t SingleFPValue = 0; + ze_device_fp_flags_t ZeSingleFPCapabilities = + Device->ZeDeviceModuleProperties->fp32flags; + if (ZE_DEVICE_FP_FLAG_DENORM & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; + } + if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; + } + if (ZE_DEVICE_FP_FLAG_FMA & ZeSingleFPCapabilities) { + SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; + } + if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeSingleFPCapabilities) { + SingleFPValue |= + UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; + } + return ReturnValue(SingleFPValue); + } + case UR_DEVICE_INFO_HALF_FP_CONFIG: { + ur_device_fp_capability_flags_t HalfFPValue = 0; + ze_device_fp_flags_t ZeHalfFPCapabilities = + Device->ZeDeviceModuleProperties->fp16flags; + if (ZE_DEVICE_FP_FLAG_DENORM & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; + } + if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; + } + if (ZE_DEVICE_FP_FLAG_FMA & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; + } + if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeHalfFPCapabilities) { + HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; + } + return ReturnValue(HalfFPValue); + } + case UR_DEVICE_INFO_DOUBLE_FP_CONFIG: { + ur_device_fp_capability_flags_t DoubleFPValue = 0; + ze_device_fp_flags_t ZeDoubleFPCapabilities = + Device->ZeDeviceModuleProperties->fp64flags; + if (ZE_DEVICE_FP_FLAG_DENORM & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; + } + if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; + } + if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; + } + if (ZE_DEVICE_FP_FLAG_FMA & ZeDoubleFPCapabilities) { + DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; + } + if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeDoubleFPCapabilities) { + DoubleFPValue |= + UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; + } + return ReturnValue(DoubleFPValue); + } + case UR_DEVICE_INFO_IMAGE2D_MAX_WIDTH: + return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims2D}); + case UR_DEVICE_INFO_IMAGE2D_MAX_HEIGHT: + return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims2D}); + case UR_DEVICE_INFO_IMAGE3D_MAX_WIDTH: + return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); + case UR_DEVICE_INFO_IMAGE3D_MAX_HEIGHT: + return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); + case UR_DEVICE_INFO_IMAGE3D_MAX_DEPTH: + return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); + case UR_DEVICE_INFO_IMAGE_MAX_BUFFER_SIZE: + return ReturnValue( + size_t{Device->ZeDeviceImageProperties->maxImageBufferSize}); + case UR_DEVICE_INFO_IMAGE_MAX_ARRAY_SIZE: + return ReturnValue( + size_t{Device->ZeDeviceImageProperties->maxImageArraySlices}); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_char); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_CHAR: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_char); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_short); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_SHORT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_short); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_int); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_INT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_int); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG: + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG_LONG: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_long); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG: + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG_LONG: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_long); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_float); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_FLOAT: + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_float); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE: + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_DOUBLE: + // Must return 0 for *vector_width_double* if the device does not have fp64. + if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP64)) + return ReturnValue(uint32_t{0}); + return ReturnValue(uint32_t{1}); + case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF: + // Must return 0 for *vector_width_half* if the device does not have fp16. + if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16)) + return ReturnValue(uint32_t{0}); + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_half); + case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_HALF: + // Must return 0 for *vector_width_half* if the device does not have fp16. + if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16)) + return ReturnValue(uint32_t{0}); + return ReturnValue( + Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_half); + case UR_DEVICE_INFO_MAX_NUM_SUB_GROUPS: { + // Max_num_sub_Groups = maxTotalGroupSize/min(set of subGroupSizes); + uint32_t MinSubGroupSize = + Device->ZeDeviceComputeProperties->subGroupSizes[0]; + for (uint32_t I = 1; + I < Device->ZeDeviceComputeProperties->numSubGroupSizes; I++) { + if (MinSubGroupSize > Device->ZeDeviceComputeProperties->subGroupSizes[I]) + MinSubGroupSize = Device->ZeDeviceComputeProperties->subGroupSizes[I]; + } + return ReturnValue(Device->ZeDeviceComputeProperties->maxTotalGroupSize / + MinSubGroupSize); + } + case UR_DEVICE_INFO_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS: { + // TODO: Not supported yet. Needs to be updated after support is added. + return ReturnValue(static_cast(false)); + } + case UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL: { + return ReturnValue(Device->ZeDeviceComputeProperties->subGroupSizes, + Device->ZeDeviceComputeProperties->numSubGroupSizes); + } + case UR_DEVICE_INFO_IL_VERSION: { + // Set to a space separated list of IL version strings of the form + // _.. + // "SPIR-V" is a required IL prefix when cl_khr_il_progam extension is + // reported. + uint32_t SpirvVersion = + Device->ZeDeviceModuleProperties->spirvVersionSupported; + uint32_t SpirvVersionMajor = ZE_MAJOR_VERSION(SpirvVersion); + uint32_t SpirvVersionMinor = ZE_MINOR_VERSION(SpirvVersion); + + char SpirvVersionString[50]; + int Len = sprintf(SpirvVersionString, "SPIR-V_%d.%d ", SpirvVersionMajor, + SpirvVersionMinor); + // returned string to contain only len number of characters. + std::string ILVersion(SpirvVersionString, Len); + return ReturnValue(ILVersion.c_str()); + } + case UR_DEVICE_INFO_USM_HOST_SUPPORT: + case UR_DEVICE_INFO_USM_DEVICE_SUPPORT: + case UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT: + case UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT: + case UR_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT: { + auto MapCaps = [](const ze_memory_access_cap_flags_t &ZeCapabilities) { + ur_device_usm_access_capability_flags_t Capabilities = 0; + if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_RW) + Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ACCESS; + if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_ATOMIC) + Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ATOMIC_ACCESS; + if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT) + Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_CONCURRENT_ACCESS; + if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT_ATOMIC) + Capabilities |= + UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ATOMIC_CONCURRENT_ACCESS; + return Capabilities; + }; + auto &Props = Device->ZeDeviceMemoryAccessProperties; + switch (ParamName) { + case UR_DEVICE_INFO_USM_HOST_SUPPORT: + return ReturnValue(MapCaps(Props->hostAllocCapabilities)); + case UR_DEVICE_INFO_USM_DEVICE_SUPPORT: + return ReturnValue(MapCaps(Props->deviceAllocCapabilities)); + case UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT: + return ReturnValue(MapCaps(Props->sharedSingleDeviceAllocCapabilities)); + case UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT: + return ReturnValue(MapCaps(Props->sharedCrossDeviceAllocCapabilities)); + case UR_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT: + return ReturnValue(MapCaps(Props->sharedSystemAllocCapabilities)); + default: + die("urDeviceGetInfo: unexpected ParamName."); + } + } + + // intel extensions for GPU information + case UR_DEVICE_INFO_DEVICE_ID: + return ReturnValue(uint32_t{Device->ZeDeviceProperties->deviceId}); + case UR_DEVICE_INFO_PCI_ADDRESS: { + ze_pci_address_ext_t PciAddr{}; + ZeStruct ZeDevicePciProperties; + ZeDevicePciProperties.address = PciAddr; + ZE2UR_CALL(zeDevicePciGetPropertiesExt, (ZeDevice, &ZeDevicePciProperties)); + constexpr size_t AddressBufferSize = 13; + char AddressBuffer[AddressBufferSize]; + std::snprintf(AddressBuffer, AddressBufferSize, "%04x:%02x:%02x.%01x", + ZeDevicePciProperties.address.domain, + ZeDevicePciProperties.address.bus, + ZeDevicePciProperties.address.device, + ZeDevicePciProperties.address.function); + return ReturnValue(AddressBuffer); + } + + case UR_DEVICE_INFO_GLOBAL_MEM_FREE: { + if (!ParamValue && pSize) { + if (supportsDeviceUsableMemSizeExtension(Device->Platform)) { + return ReturnValue(uint64_t{0}); + } + + auto [ZesDevice, ZesDeviceData, Result] = getZesDeviceData(Device); + (void)ZesDevice; + (void)ZesDeviceData; + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + + return ReturnValue(uint64_t{0}); + } + + // Calculate the global memory size as the max limit that can be reported as + // "free" memory for the user to allocate. + uint64_t GlobalMemSize = calculateGlobalMemSize(Device); + // Only report device memory which zeMemAllocDevice can allocate from. + // Currently this is only the one enumerated with ordinal 0. + uint64_t FreeMemory = 0; + uint32_t MemCount = 0; + + if (auto CoreUsableMemSize = getDeviceUsableMemSizeFromCore(Device)) { + return ReturnValue(std::min(GlobalMemSize, *CoreUsableMemSize)); + } + + auto [ZesDevice, ZesDeviceData, Result] = getZesDeviceData(Device); + if (Result != UR_RESULT_SUCCESS) + return Result; + + ZE2UR_CALL(zesDeviceEnumMemoryModules, (ZesDevice, &MemCount, nullptr)); + if (MemCount != 0) { + std::vector ZesMemHandles(MemCount); + ZE2UR_CALL(zesDeviceEnumMemoryModules, + (ZesDevice, &MemCount, ZesMemHandles.data())); + for (auto &ZesMemHandle : ZesMemHandles) { + ZesStruct ZesMemProperties; + ZE2UR_CALL(zesMemoryGetProperties, (ZesMemHandle, &ZesMemProperties)); + // For root-device report memory from all memory modules since that + // is what totally available in the default implicit scaling mode. + // For sub-devices only report memory local to them. + if (ZesDeviceData.SubDeviceId == ZesMemProperties.subdeviceId || + !ZesDeviceData.SubDevice) { + ZesStruct ZesMemState; + ZE2UR_CALL(zesMemoryGetState, (ZesMemHandle, &ZesMemState)); + FreeMemory += ZesMemState.free; + } + } + } + if (MemCount > 0) { + return ReturnValue(std::min(GlobalMemSize, FreeMemory)); + } else { + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + } + case UR_DEVICE_INFO_MEMORY_CLOCK_RATE: { + // If there are not any memory modules then return 0. + if (Device->ZeDeviceMemoryProperties->first.empty()) + return ReturnValue(uint32_t{0}); + + // If there are multiple memory modules on the device then we have to report + // the value of the slowest memory. + auto Comp = [](const ze_device_memory_properties_t &A, + const ze_device_memory_properties_t &B) -> bool { + return A.maxClockRate < B.maxClockRate; + }; + auto MinIt = + std::min_element(Device->ZeDeviceMemoryProperties->first.begin(), + Device->ZeDeviceMemoryProperties->first.end(), Comp); + return ReturnValue(uint32_t{MinIt->maxClockRate}); + } + case UR_DEVICE_INFO_MEMORY_BUS_WIDTH: { + // If there are not any memory modules then return 0. + if (Device->ZeDeviceMemoryProperties->first.empty()) + return ReturnValue(uint32_t{0}); + + // If there are multiple memory modules on the device then we have to report + // the value of the slowest memory. + auto Comp = [](const ze_device_memory_properties_t &A, + const ze_device_memory_properties_t &B) -> bool { + return A.maxBusWidth < B.maxBusWidth; + }; + auto MinIt = + std::min_element(Device->ZeDeviceMemoryProperties->first.begin(), + Device->ZeDeviceMemoryProperties->first.end(), Comp); + return ReturnValue(uint32_t{MinIt->maxBusWidth}); + } + case UR_DEVICE_INFO_MAX_COMPUTE_QUEUE_INDICES: { + if (Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] + .ZeIndex >= 0) + // Sub-sub-device represents a particular compute index already. + return ReturnValue(int32_t{1}); + + auto ZeDeviceNumIndices = + Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] + .ZeProperties.numQueues; + return ReturnValue(int32_t(ZeDeviceNumIndices)); + } break; + case UR_DEVICE_INFO_GPU_EU_COUNT: { + if (Device->Platform->ZeDriverEuCountExtensionFound) { + ze_device_properties_t DeviceProp = {}; + DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_eu_count_ext_t EuCountDesc = {}; + EuCountDesc.stype = ZE_STRUCTURE_TYPE_EU_COUNT_EXT; + DeviceProp.pNext = (void *)&EuCountDesc; + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); + if (EuCountDesc.numTotalEUs > 0) { + return ReturnValue(uint32_t{EuCountDesc.numTotalEUs}); + } + } + + uint32_t count = Device->ZeDeviceProperties->numEUsPerSubslice * + Device->ZeDeviceProperties->numSubslicesPerSlice * + Device->ZeDeviceProperties->numSlices; + return ReturnValue(uint32_t{count}); + } + case UR_DEVICE_INFO_GPU_EU_SLICES: { + return ReturnValue(uint32_t{Device->ZeDeviceProperties->numSlices}); + } + case UR_DEVICE_INFO_GPU_EU_SIMD_WIDTH: + return ReturnValue( + uint32_t{Device->ZeDeviceProperties->physicalEUSimdWidth}); + case UR_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE: + return ReturnValue( + uint32_t{Device->ZeDeviceProperties->numSubslicesPerSlice}); + case UR_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE: + return ReturnValue(uint32_t{Device->ZeDeviceProperties->numEUsPerSubslice}); + case UR_DEVICE_INFO_GPU_HW_THREADS_PER_EU: + return ReturnValue(uint32_t{Device->ZeDeviceProperties->numThreadsPerEU}); + case UR_DEVICE_INFO_ATOMIC_MEMORY_SCOPE_CAPABILITIES: { + // There are no explicit restrictions in L0 programming guide, so assume all + // are supported + ur_memory_scope_capability_flags_t result = + UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_ITEM | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_SUB_GROUP | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_DEVICE | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_SYSTEM; + + return ReturnValue(result); + } + case UR_DEVICE_INFO_ATOMIC_FENCE_ORDER_CAPABILITIES: { + // There are no explicit restrictions in L0 programming guide, so assume all + // are supported + ur_memory_order_capability_flags_t result = + UR_MEMORY_ORDER_CAPABILITY_FLAG_RELAXED | + UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQUIRE | + UR_MEMORY_ORDER_CAPABILITY_FLAG_RELEASE | + UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQ_REL | + UR_MEMORY_ORDER_CAPABILITY_FLAG_SEQ_CST; + + return ReturnValue(result); + } + case UR_DEVICE_INFO_ATOMIC_FENCE_SCOPE_CAPABILITIES: { + // There are no explicit restrictions in L0 programming guide, so assume all + // are supported + ur_memory_scope_capability_flags_t result = + UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_ITEM | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_SUB_GROUP | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_DEVICE | + UR_MEMORY_SCOPE_CAPABILITY_FLAG_SYSTEM; + + return ReturnValue(result); + } + + case UR_DEVICE_INFO_ATOMIC_MEMORY_ORDER_CAPABILITIES: { + ur_memory_order_capability_flags_t capabilities = + UR_MEMORY_ORDER_CAPABILITY_FLAG_RELAXED | + UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQUIRE | + UR_MEMORY_ORDER_CAPABILITY_FLAG_RELEASE | + UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQ_REL | + UR_MEMORY_ORDER_CAPABILITY_FLAG_SEQ_CST; + return ReturnValue(capabilities); + } + case UR_DEVICE_INFO_MEM_CHANNEL_SUPPORT: + return ReturnValue(ur_bool_t{false}); + case UR_DEVICE_INFO_IMAGE_SRGB: + return ReturnValue(ur_bool_t{false}); + + case UR_DEVICE_INFO_QUEUE_ON_DEVICE_PROPERTIES: + case UR_DEVICE_INFO_QUEUE_ON_HOST_PROPERTIES: { + ur_queue_flags_t queue_flags = 0; + return ReturnValue(queue_flags); + } + case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS: { + return ReturnValue(static_cast( + 0)); //__read_write attribute currently undefinde in opencl + } + case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: { + return ReturnValue(static_cast(true)); + } + case UR_DEVICE_INFO_TIMESTAMP_RECORDING_SUPPORT_EXP: { + return ReturnValue(static_cast(true)); + } + case UR_DEVICE_INFO_ENQUEUE_NATIVE_COMMAND_SUPPORT_EXP: { + // L0 doesn't support enqueueing native work through the urNativeEnqueueExp + return ReturnValue(static_cast(false)); + } + + case UR_DEVICE_INFO_ESIMD_SUPPORT: { + // ESIMD is only supported by Intel GPUs. + ur_bool_t result = Device->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU && + Device->ZeDeviceProperties->vendorId == 0x8086; + return ReturnValue(result); + } + + case UR_DEVICE_INFO_COMPONENT_DEVICES: { + ze_device_handle_t DevHandle = Device->ZeDevice; + uint32_t SubDeviceCount = 0; + // First call to get SubDeviceCount. + ZE2UR_CALL(zeDeviceGetSubDevices, (DevHandle, &SubDeviceCount, nullptr)); + if (SubDeviceCount == 0) + return ReturnValue(std::nullopt); + + std::vector SubDevs(SubDeviceCount); + // Second call to get the actual list of devices. + ZE2UR_CALL(zeDeviceGetSubDevices, + (DevHandle, &SubDeviceCount, SubDevs.data())); + + size_t SubDeviceCount_s{SubDeviceCount}; + auto ResSize = + std::min(SubDeviceCount_s, propSize / sizeof(ur_device_handle_t)); + std::vector Res; + for (const auto &d : SubDevs) { + // We can only reach this code if ZE_FLAT_DEVICE_HIERARCHY != FLAT, + // because in flat mode we directly get tiles, and those don't have any + // further divisions, so zeDeviceGetSubDevices always will return an empty + // list. Thus, there's only two options left: (a) composite mode, and (b) + // combined mode. In (b), zeDeviceGet returns tiles as devices, and those + // are presented as root devices (i.e. isSubDevice() returns false). In + // contrast, in (a), zeDeviceGet returns cards as devices, so tiles are + // not root devices (i.e. isSubDevice() returns true). Since we only reach + // this code if there are tiles returned by zeDeviceGetSubDevices, we + // can know if we are in (a) or (b) by checking if a tile is root device + // or not. + ur_device_handle_t URDev = Device->Platform->getDeviceFromNativeHandle(d); + if (URDev->isSubDevice()) { + // We are in COMPOSITE mode, return an empty list. + if (pSize) { + *pSize = 0; + } + return UR_RESULT_SUCCESS; + } + + Res.push_back(URDev); + } + if (pSize) + *pSize = SubDeviceCount * sizeof(ur_device_handle_t); + if (ParamValue) { + return ReturnValue(Res.data(), ResSize); + } + return UR_RESULT_SUCCESS; + } + case UR_DEVICE_INFO_COMPOSITE_DEVICE: { + ur_device_handle_t UrRootDev = nullptr; + ze_device_handle_t DevHandle = Device->ZeDevice; + ze_device_handle_t RootDev; + // Query Root Device. + auto errc = ZE_CALL_NOCHECK(zeDeviceGetRootDevice, (DevHandle, &RootDev)); + UrRootDev = Device->Platform->getDeviceFromNativeHandle(RootDev); + if (errc != ZE_RESULT_SUCCESS && + errc != ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) + return ze2urResult(errc); + return ReturnValue(UrRootDev); + } + case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { + bool Bfloat16ConversionSupport = + (Device->Platform->zeDriverExtensionMap.count( + ZE_BFLOAT16_CONVERSIONS_EXT_NAME)) || + ((Device->ZeDeviceProperties->deviceId & 0xfff) == 0x201 || + (Device->ZeDeviceProperties->deviceId & 0xff0) == 0xbd0); + return ReturnValue(Bfloat16ConversionSupport); + } + case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP: { + const auto ZeMutableCommandFlags = + Device->ZeDeviceMutableCmdListsProperties->mutableCommandFlags; + + auto supportsFlags = [&](ze_mutable_command_exp_flags_t RequiredFlags) { + return (ZeMutableCommandFlags & RequiredFlags) == RequiredFlags; + }; + + ur_device_command_buffer_update_capability_flags_t UpdateCapabilities = 0; + if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_ARGUMENTS)) { + UpdateCapabilities |= + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; + } + /* These capabilities are bundled together because, when the user updates + * the global work-size, the implementation might have to generate a new + * local work-size. This would require both mutable command flags to be set + * even though only the global work-size was explicitly updated. */ + if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_COUNT | + ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_SIZE)) { + UpdateCapabilities |= + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE | + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; + } + if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_GLOBAL_OFFSET)) { + UpdateCapabilities |= + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET; + } + if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_INSTRUCTION)) { + UpdateCapabilities |= + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE; + } + return ReturnValue(UpdateCapabilities); + } + case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP: + return ReturnValue(false); + case UR_DEVICE_INFO_COMMAND_BUFFER_SUBGRAPH_SUPPORT_EXP: + return ReturnValue(false); + case UR_DEVICE_INFO_BINDLESS_IMAGES_SUPPORT_EXP: { + return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported); + } + case UR_DEVICE_INFO_BINDLESS_IMAGES_SHARED_USM_SUPPORT_EXP: { + // On L0 bindless images can not be backed by shared (managed) USM. + return ReturnValue(false); + } + case UR_DEVICE_INFO_BINDLESS_IMAGES_1D_USM_SUPPORT_EXP: { + return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported && + Device->ZeDeviceImageProperties->maxImageDims1D > 0); + } + case UR_DEVICE_INFO_BINDLESS_IMAGES_2D_USM_SUPPORT_EXP: { + return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported && + Device->ZeDeviceImageProperties->maxImageDims2D > 0); + } + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_WIDTH_EXP: { + return ReturnValue( + Device->ZeDevicePitchedAllocProperties->AllocProps.maxImageLinearWidth); + } + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_HEIGHT_EXP: { + return ReturnValue(Device->ZeDevicePitchedAllocProperties->AllocProps + .maxImageLinearHeight); + } + case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP: { + return ReturnValue(static_cast( + Device->ZeDevicePitchedAllocProperties->PitchInfo.pitchAlign)); + } + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_PITCH_EXP: { + return ReturnValue( + Device->ZeDevicePitchedAllocProperties->PitchInfo.maxSupportedPitch); + } + case UR_DEVICE_INFO_MIPMAP_SUPPORT_EXP: { + // L0 does not support mipmaps. + return ReturnValue(false); + } + case UR_DEVICE_INFO_MIPMAP_ANISOTROPY_SUPPORT_EXP: { + // L0 does not support anisotropic filtering. + return ReturnValue(false); + } + case UR_DEVICE_INFO_MIPMAP_MAX_ANISOTROPY_EXP: + UR_LOG(ERR, "Unsupported ParamName in urGetDeviceInfo"); + UR_LOG(ERR, "ParamName=%{}(0x{})", ParamName, logger::toHex(ParamName)); + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + case UR_DEVICE_INFO_MIPMAP_LEVEL_REFERENCE_SUPPORT_EXP: { + // L0 does not support creation of images from individual mipmap levels. + return ReturnValue(false); + } + case UR_DEVICE_INFO_EXTERNAL_MEMORY_IMPORT_SUPPORT_EXP: { + // L0 supports importing external memory. + return ReturnValue(true); + } + case UR_DEVICE_INFO_EXTERNAL_SEMAPHORE_IMPORT_SUPPORT_EXP: { + return ReturnValue(Device->Platform->ZeExternalSemaphoreExt.Supported); + } + case UR_DEVICE_INFO_CUBEMAP_SUPPORT_EXP: { + // L0 does not support cubemaps. + return ReturnValue(false); + } + case UR_DEVICE_INFO_CUBEMAP_SEAMLESS_FILTERING_SUPPORT_EXP: { + // L0 does not support cubemap seamless filtering. + return ReturnValue(false); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_1D_USM_SUPPORT_EXP: { + // L0 does support fetching 1D USM sampled image data. + return ReturnValue(true); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_1D_SUPPORT_EXP: { + // L0 does support fetching 1D non-USM sampled image data. + return ReturnValue(true); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_2D_USM_SUPPORT_EXP: { + // L0 does support fetching 2D USM sampled image data. + return ReturnValue(true); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_2D_SUPPORT_EXP: { + // L0 does support fetching 2D non-USM sampled image data. + return ReturnValue(true); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_3D_SUPPORT_EXP: { + // L0 does support fetching 3D non-USM sampled image data. + return ReturnValue(true); + } + case UR_DEVICE_INFO_IMAGE_ARRAY_SUPPORT_EXP: { + // L0 does support image arrays + return ReturnValue(true); + } + case UR_DEVICE_INFO_BINDLESS_UNIQUE_ADDRESSING_PER_DIM_SUPPORT_EXP: { + // L0 does not support unique addressing per dimension + return ReturnValue(false); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLE_1D_USM_SUPPORT_EXP: { + // L0 does not support sampling 1D USM sampled image data. + return ReturnValue(false); + } + case UR_DEVICE_INFO_BINDLESS_SAMPLE_2D_USM_SUPPORT_EXP: { + // L0 does not support sampling 2D USM sampled image data. + return ReturnValue(false); + } + case UR_DEVICE_INFO_BINDLESS_IMAGES_GATHER_SUPPORT_EXP: { + // L0 doesn't support sampled image gather. + return ReturnValue(static_cast(false)); + } + case UR_DEVICE_INFO_PROGRAM_SET_SPECIALIZATION_CONSTANTS: + return ReturnValue(true); + case UR_DEVICE_INFO_KERNEL_SET_SPECIALIZATION_CONSTANTS: + return ReturnValue(false); + case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT: + return ReturnValue(true); + case UR_DEVICE_INFO_USM_POOL_SUPPORT: + return ReturnValue(true); + case UR_DEVICE_INFO_2D_BLOCK_ARRAY_CAPABILITIES_EXP: { +#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME + const auto ZeDeviceBlockArrayFlags = + Device->ZeDeviceBlockArrayProperties->flags; + + auto supportsFlags = + [&](ze_intel_device_block_array_exp_flags_t RequiredFlags) { + return (ZeDeviceBlockArrayFlags & RequiredFlags) == RequiredFlags; + }; + + ur_exp_device_2d_block_array_capability_flags_t BlockArrayCapabilities = 0; + if (supportsFlags(ZE_INTEL_DEVICE_EXP_FLAG_2D_BLOCK_LOAD)) { + BlockArrayCapabilities |= + UR_EXP_DEVICE_2D_BLOCK_ARRAY_CAPABILITY_FLAG_LOAD; + } + if (supportsFlags(ZE_INTEL_DEVICE_EXP_FLAG_2D_BLOCK_STORE)) { + BlockArrayCapabilities |= + UR_EXP_DEVICE_2D_BLOCK_ARRAY_CAPABILITY_FLAG_STORE; + } + return ReturnValue(BlockArrayCapabilities); +#else + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; +#endif + } + case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: +#ifdef _WIN32 + // TODO: Remove when IPC memory works in UMF on Windows. + return ReturnValue(false); +#else + return ReturnValue(true); +#endif + case UR_DEVICE_INFO_ASYNC_BARRIER: + return ReturnValue(false); + case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORT: + return ReturnValue(false); + case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_USE_NATIVE_ASSERT: + return ReturnValue(true); + case UR_DEVICE_INFO_USM_P2P_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_MULTI_DEVICE_COMPILE_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_DEVICE_WAIT_SUPPORT_EXP: { + auto Supported = Device->Platform->ZeDeviceSynchronizeSupported; + return ReturnValue(Supported); + } + case UR_DEVICE_INFO_DYNAMIC_LINK_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_CURRENT_CLOCK_THROTTLE_REASONS: { + ur_device_throttle_reasons_flags_t ThrottleReasons = 0; + if (!ParamValue) { + // If ParamValue is nullptr, then we are only interested in the size of + // the value. + return ReturnValue(ThrottleReasons); + } + [[maybe_unused]] auto [ZesDevice, Ignored, Result] = + getZesDeviceData(Device); + if (Result != UR_RESULT_SUCCESS) + return Result; + uint32_t FreqCount = 0; + ZE2UR_CALL(zesDeviceEnumFrequencyDomains, (ZesDevice, &FreqCount, nullptr)); + if (FreqCount != 0) { + std::vector ZesFreqHandles(FreqCount); + ZE2UR_CALL(zesDeviceEnumFrequencyDomains, + (ZesDevice, &FreqCount, ZesFreqHandles.data())); + for (auto &ZesFreqHandle : ZesFreqHandles) { + ZesStruct FreqProperties; + ZE2UR_CALL(zesFrequencyGetProperties, (ZesFreqHandle, &FreqProperties)); + if (FreqProperties.type != ZES_FREQ_DOMAIN_GPU) { + continue; + } + zes_freq_state_t State; + zesFrequencyGetState(ZesFreqHandle, &State); + constexpr zes_freq_throttle_reason_flags_t ZeThrottleFlags[] = { + ZES_FREQ_THROTTLE_REASON_FLAG_AVE_PWR_CAP, + ZES_FREQ_THROTTLE_REASON_FLAG_CURRENT_LIMIT, + ZES_FREQ_THROTTLE_REASON_FLAG_THERMAL_LIMIT, + ZES_FREQ_THROTTLE_REASON_FLAG_PSU_ALERT, + ZES_FREQ_THROTTLE_REASON_FLAG_SW_RANGE, + ZES_FREQ_THROTTLE_REASON_FLAG_HW_RANGE}; + + constexpr ur_device_throttle_reasons_flags_t UrThrottleFlags[] = { + UR_DEVICE_THROTTLE_REASONS_FLAG_POWER_CAP, + UR_DEVICE_THROTTLE_REASONS_FLAG_CURRENT_LIMIT, + UR_DEVICE_THROTTLE_REASONS_FLAG_THERMAL_LIMIT, + UR_DEVICE_THROTTLE_REASONS_FLAG_PSU_ALERT, + UR_DEVICE_THROTTLE_REASONS_FLAG_SW_RANGE, + UR_DEVICE_THROTTLE_REASONS_FLAG_HW_RANGE}; + + for (size_t i = 0; + i < sizeof(ZeThrottleFlags) / sizeof(ZeThrottleFlags[0]); ++i) { + if (State.throttleReasons & ZeThrottleFlags[i]) { + ThrottleReasons |= UrThrottleFlags[i]; + State.throttleReasons &= ~ZeThrottleFlags[i]; + } + } + + if (State.throttleReasons) { + ThrottleReasons |= UR_DEVICE_THROTTLE_REASONS_FLAG_OTHER; + } + } + } + return ReturnValue(ThrottleReasons); + } + case UR_DEVICE_INFO_FAN_SPEED: { + [[maybe_unused]] auto [ZesDevice, Ignored, Result] = + getZesDeviceData(Device); + if (Result != UR_RESULT_SUCCESS) + return Result; + + uint32_t FanCount = 0; + ZE2UR_CALL(zesDeviceEnumFans, (ZesDevice, &FanCount, nullptr)); + // If there are no fans, then report speed query as unsupported. + if (FanCount == 0) + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + + if (!ParamValue) { + // If ParamValue is nullptr, then we are only interested in the size of + // the value. + return ReturnValue(int32_t{0}); + } + + std::vector ZeFanHandles(FanCount); + ZE2UR_CALL(zesDeviceEnumFans, (ZesDevice, &FanCount, ZeFanHandles.data())); + int32_t Speed = -1; + for (auto Fan : ZeFanHandles) { + int32_t CurSpeed; + ZE2UR_CALL(zesFanGetState, (Fan, ZES_FAN_SPEED_UNITS_PERCENT, &CurSpeed)); + Speed = std::max(Speed, CurSpeed); + } + return ReturnValue(Speed); + } + case UR_DEVICE_INFO_MIN_POWER_LIMIT: + case UR_DEVICE_INFO_MAX_POWER_LIMIT: { + [[maybe_unused]] auto [ZesDevice, Ignored, Result] = + getZesDeviceData(Device); + if (Result != UR_RESULT_SUCCESS) + return Result; + + zes_pwr_handle_t ZesPwrHandle = nullptr; + auto DomainResult = zesDeviceGetCardPowerDomain(ZesDevice, &ZesPwrHandle); + if (DomainResult == ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) { + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } else if (DomainResult != ZE_RESULT_SUCCESS) { + return ze2urResult(DomainResult); + } + + if (!ParamValue) { + // If ParamValue is nullptr, then we are only interested in the size of + // the value. + // Do this after calling getCardPowerDomain so that UNSUPPORTED is + // returned correctly if required + return ReturnValue(int32_t{0}); + } + + ZesStruct PowerProperties; + ZE2UR_CALL(zesPowerGetProperties, (ZesPwrHandle, &PowerProperties)); + + if (ParamName == UR_DEVICE_INFO_MIN_POWER_LIMIT) { + return ReturnValue(int32_t{PowerProperties.minLimit}); + } else { + return ReturnValue(int32_t{PowerProperties.maxLimit}); + } + } + case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: + return ReturnValue(UR_KERNEL_LAUNCH_PROPERTIES_FLAG_COOPERATIVE); + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_LUID: { + // LUID is only available on Windows. + // Intel extension for device LUID. This returns the LUID as + // std::array. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + if (Device->Platform->ZeLUIDSupported) { + ze_device_properties_t DeviceProp = {}; + DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_device_luid_ext_properties_t LuidDesc = {}; + LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; + DeviceProp.pNext = (void *)&LuidDesc; + + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); + + const auto &LUID = LuidDesc.luid.id; + return ReturnValue(LUID, sizeof(LUID)); + } else { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + } + case UR_DEVICE_INFO_NODE_MASK: { + // Device node mask is only available on Windows. + // Intel extension for device node mask. This returns the node mask as + // uint32_t. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + + // Node mask is provided through the L0 LUID extension so support for this + // extension must be checked. + if (Device->Platform->ZeLUIDSupported) { + ze_device_properties_t DeviceProp = {}; + DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_device_luid_ext_properties_t LuidDesc = {}; + LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; + DeviceProp.pNext = (void *)&LuidDesc; + + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); + + return ReturnValue(LuidDesc.nodeMask); + } else { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + } + case UR_DEVICE_INFO_CLOCK_SUB_GROUP_SUPPORT_EXP: { + // IGC supports this since v.2.16.0 + return ReturnValue( + Device->Platform->isDriverVersionNewerOrSimilar(1, 6, 34666)); + } + case UR_DEVICE_INFO_CLOCK_WORK_GROUP_SUPPORT_EXP: + case UR_DEVICE_INFO_CLOCK_DEVICE_SUPPORT_EXP: + // Currently GPUs only support sub-group clock. + return ReturnValue(false); + case UR_DEVICE_INFO_IS_INTEGRATED_GPU: + return ReturnValue(static_cast(Device->isIntegrated() != 0)); + case UR_DEVICE_INFO_XE_STACK_COUNT: + return ReturnValue(uint32_t{Device->ZeXEDeviceProperties->numXeStacks}); + case UR_DEVICE_INFO_XE_REGIONS_PER_STACK: + return ReturnValue( + uint32_t{Device->ZeXEDeviceProperties->numXeRegionsPerStack}); + case UR_DEVICE_INFO_XE_CLUSTERS_PER_REGION: + return ReturnValue( + uint32_t{Device->ZeXEDeviceProperties->numXeClustersPerRegion}); + case UR_DEVICE_INFO_XE_CORES_PER_CLUSTER: + return ReturnValue( + uint32_t{Device->ZeXEDeviceProperties->numXeCorePerCluster}); + case UR_DEVICE_INFO_EUS_PER_XE_CORE: + return ReturnValue( + uint32_t{Device->ZeXEDeviceProperties->numExecutionEnginesPerXeCore}); + case UR_DEVICE_INFO_MAX_LANES_PER_HW_THREAD: + return ReturnValue( + uint32_t{Device->ZeXEDeviceProperties->maxNumLanesPerHwThread}); + default: + UR_LOG(ERR, "Unsupported ParamName in urGetDeviceInfo"); + UR_LOG(ERR, "ParamNameParamName={}(0x{})", ParamName, + logger::toHex(ParamName)); + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + + return UR_RESULT_SUCCESS; +} + +bool CopyEngineRequested(const ur_device_handle_t &Device) { + int LowerCopyQueueIndex = getRangeOfAllowedCopyEngines(Device).first; + int UpperCopyQueueIndex = getRangeOfAllowedCopyEngines(Device).second; + return ((LowerCopyQueueIndex != -1) || (UpperCopyQueueIndex != -1)); +} + +ur_result_t urDevicePartition( + /// [in] handle of the device to partition. + ::ur_device_handle_t hDeviceOpque, + /// [in] Device partition properties. + const ur_device_partition_properties_t *Properties, + /// [in] the number of sub-devices. + uint32_t NumDevices, + /// [out][optional][range(0, NumDevices)] array of handle of devices. If + /// NumDevices is less than the number of sub-devices available, then + /// the function shall only retrieve that number of sub-devices. + ::ur_device_handle_t *OutDevicesOpque, + /// [out][optional] pointer to the number of sub-devices the device can be + /// partitioned into according to the partitioning property. + uint32_t *NumDevicesRet) { + auto Device = common_cast(hDeviceOpque); + ur_device_handle_t *OutDevices = common_cast(OutDevicesOpque); + // Other partitioning ways are not supported by Level Zero + UR_ASSERT(Properties->PropCount == 1, UR_RESULT_ERROR_INVALID_VALUE); + if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN) { + if ((Properties->pProperties->value.affinity_domain != + UR_DEVICE_AFFINITY_DOMAIN_FLAG_NEXT_PARTITIONABLE && + Properties->pProperties->value.affinity_domain != + UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA)) { + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_CSLICE) { + if (Properties->pProperties->value.affinity_domain != 0) { + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else { + return UR_RESULT_ERROR_INVALID_VALUE; + } + + // Devices cache is normally created in piDevicesGet but still make + // sure that cache is populated. + // + auto Res = Device->Platform->populateDeviceCacheIfNeeded(); + if (Res != UR_RESULT_SUCCESS) { + return Res; + } + + auto EffectiveNumDevices = [&]() -> decltype(Device->SubDevices.size()) { + if (Device->SubDevices.size() == 0) + return 0; + + // Sub-Sub-Devices are partitioned by CSlices, not by affinity domain. + // However, if + // UR_L0_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING overrides that + // still expose CSlices in partitioning by affinity domain for compatibility + // reasons. + if (Properties->pProperties->type == + UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN && + !ExposeCSliceInAffinityPartitioning) { + if (Device->isSubDevice()) { + return 0; + } + } + if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_CSLICE) { + // Not a CSlice-based partitioning. + if (!Device->SubDevices[0]->isCCS()) { + return 0; + } + } + + return Device->SubDevices.size(); + }(); + + // TODO: Consider support for partitioning to <= total sub-devices. + // Currently supported partitioning (by affinity domain/numa) would always + // partition to all sub-devices. + // + if (NumDevices != 0) + UR_ASSERT(NumDevices == EffectiveNumDevices, UR_RESULT_ERROR_INVALID_VALUE); + + for (uint32_t I = 0; I < NumDevices; I++) { + auto prop = Properties->pProperties[0]; + if (prop.type == UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN) { + // In case the value is NEXT_PARTITIONABLE, we need to change it to the + // chosen domain. This will always be NUMA since that's the only domain + // supported by level zero. + prop.value.affinity_domain = UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA; + } + Device->SubDevices[I]->SubDeviceCreationProperty = prop; + + OutDevices[I] = Device->SubDevices[I]; + // reusing the same pi_device needs to increment the reference count + ur::level_zero::urDeviceRetain(common_cast(OutDevices[I])); + } + + if (NumDevicesRet) { + *NumDevicesRet = EffectiveNumDevices; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceSelectBinary( + /// [in] handle of the device to select binary for. + [[maybe_unused]] ::ur_device_handle_t DeviceOpque, + /// [in] the array of binaries to select from. + const ur_device_binary_t *Binaries, + /// [in] the number of binaries passed in ppBinaries. Must greater than or + /// equal to zero otherwise ::UR_RESULT_ERROR_INVALID_VALUE is returned. + uint32_t NumBinaries, + /// [out] the index of the selected binary in the input array of + /// binaries. If a suitable binary was not found the function returns + /// ${X}_INVALID_BINARY. + uint32_t *SelectedBinary) { + // TODO: this is a bare-bones implementation for choosing a device image + // that would be compatible with the targeted device. An AOT-compiled + // image is preferred over SPIR-V for known devices (i.e. Intel devices) + // The implementation makes no effort to differentiate between multiple images + // for the given device, and simply picks the first one compatible. + // + // Real implementation will use the same mechanism OpenCL ICD dispatcher + // uses. Something like: + // PI_VALIDATE_HANDLE_RETURN_HANDLE(ctx, PI_ERROR_INVALID_CONTEXT); + // return context->dispatch->piextDeviceSelectIR( + // ctx, images, num_images, selected_image); + // where context->dispatch is set to the dispatch table provided by PI + // plugin for platform/device the ctx was created for. + + // Look for GEN binary, which we known can only be handled by Level-Zero now. + const char *BinaryTarget = + UR_DEVICE_BINARY_TARGET_SPIRV64_GEN; // UR_DEVICE_BINARY_TARGET_SPIRV64_GEN; + + uint32_t *SelectedBinaryInd = SelectedBinary; + + // Find the appropriate device image + // The order of elements is important, as it defines the priority: + std::vector FallbackTargets = {UR_DEVICE_BINARY_TARGET_SPIRV64}; + + constexpr uint32_t InvalidInd = std::numeric_limits::max(); + uint32_t FallbackInd = InvalidInd; + uint32_t FallbackPriority = InvalidInd; + + for (uint32_t i = 0; i < NumBinaries; ++i) { + if (strcmp(Binaries[i].pDeviceTargetSpec, BinaryTarget) == 0) { + *SelectedBinaryInd = i; + return UR_RESULT_SUCCESS; + } + for (uint32_t j = 0; j < FallbackTargets.size(); ++j) { + // We have a fall-back with the same or higher priority already + // no need to check the rest + if (FallbackPriority <= j) + break; + + if (strcmp(Binaries[i].pDeviceTargetSpec, FallbackTargets[j]) == 0) { + FallbackInd = i; + FallbackPriority = j; + break; + } + } + } + // We didn't find a primary target, try the highest-priority fall-back + if ((*SelectedBinaryInd = FallbackInd) != InvalidInd) + return UR_RESULT_SUCCESS; + + // No image can be loaded for the given device + return UR_RESULT_ERROR_INVALID_BINARY; +} + +ur_result_t urDeviceGetNativeHandle( + /// [in] handle of the device. + ::ur_device_handle_t DeviceOpque, + /// [out] a pointer to the native handle of the device. + ::ur_native_handle_t *NativeDevice) { + *NativeDevice = + reinterpret_cast(common_cast(DeviceOpque)->ZeDevice); + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceCreateWithNativeHandle( + /// [in] the native handle of the device. + ::ur_native_handle_t NativeDevice, + /// [in] handle of the platform instance + ::ur_adapter_handle_t AdapterOpque, + /// [in][optional] pointer to native device properties struct. + [[maybe_unused]] const ur_device_native_properties_t *Properties, + /// [out] pointer to the handle of the device object created. + ::ur_device_handle_t *DeviceOpque) { + auto ZeDevice = ur_cast(NativeDevice); + + // The SYCL spec requires that the set of devices must remain fixed for the + // duration of the application's execution. We assume that we found all of the + // Level Zero devices when we initialized the platforms/devices cache, so the + // "NativeHandle" must already be in the cache. If it is not, this must not be + // a valid Level Zero device. + + ur_device_handle_t Dev = nullptr; + for (const auto &p : common_cast(AdapterOpque)->Platforms) { + Dev = p->getDeviceFromNativeHandle(ZeDevice); + } + + if (Dev == nullptr) + return UR_RESULT_ERROR_INVALID_VALUE; + + *DeviceOpque = common_cast(Dev); + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceGetGlobalTimestamps( + /// [in] handle of the device instance + ::ur_device_handle_t DeviceOpque, + /// [out][optional] pointer to the Device's global timestamp that correlates + /// with the Host's global timestamp value + uint64_t *DeviceTimestamp, + /// [out][optional] pointer to the Host's global timestamp that correlates + /// with the Device's global timestamp value + uint64_t *HostTimestamp) { + auto Device = common_cast(DeviceOpque); + if (!DeviceTimestamp && HostTimestamp) { + // If only HostTimestamp is requested, we need to avoid making a call to + // zeDeviceGetGlobalTimestamps which has higher latency. This is a + // workaround for the fact that Level Zero does not provide a way to get the + // host timestamp directly. It is known that current implementation of L0 + // runtime uses CLOCK_MONOTONIC_RAW on Linux and QueryPerformanceCounter on + // Windows. +#if defined(__linux__) + timespec Monotonic; + if (clock_gettime(CLOCK_MONOTONIC_RAW, &Monotonic) != 0) { + UR_LOG(ERR, "Failed to get CLOCK_MONOTONIC time"); + return UR_RESULT_ERROR_UNINITIALIZED; + } + *HostTimestamp = static_cast(Monotonic.tv_sec) * 1'000'000'000 + + static_cast(Monotonic.tv_nsec); + return UR_RESULT_SUCCESS; +#elif defined(_WIN32) + // Use QueryPerformanceCounter on Windows + uint64_t Counter; + if (!QueryPerformanceCounter((LARGE_INTEGER *)&Counter)) { + UR_LOG(ERR, "Failed to get performance counter"); + return UR_RESULT_ERROR_UNINITIALIZED; + } + LARGE_INTEGER Frequency; + if (!QueryPerformanceFrequency(&Frequency)) { + UR_LOG(ERR, "Failed to get performance frequency"); + return UR_RESULT_ERROR_UNINITIALIZED; + } + *HostTimestamp = static_cast( + (static_cast(Counter) * 1'000'000'000 / Frequency.QuadPart)); + return UR_RESULT_SUCCESS; +#endif + } + + const double ZeTimerResolution = Device->getTimerResolution(); + const uint64_t TimestampMaxCount = Device->getTimestampMask(); + uint64_t DeviceClockCount, Dummy; + + ZE2UR_CALL(zeDeviceGetGlobalTimestamps, + (Device->ZeDevice, + HostTimestamp == nullptr ? &Dummy : HostTimestamp, + &DeviceClockCount)); + + if (DeviceTimestamp != nullptr) { + *DeviceTimestamp = + (DeviceClockCount & TimestampMaxCount) * ZeTimerResolution; + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceRetain(::ur_device_handle_t hDeviceOpque) { + auto hDevice = common_cast(hDeviceOpque); + // The root-device ref-count remains unchanged (always 1). + if (hDevice->isSubDevice()) { + hDevice->RefCount.retain(); + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceRelease(::ur_device_handle_t hDeviceOpque) { + auto hDevice = common_cast(hDeviceOpque); + // Root devices are destroyed during the piTearDown process. + if (hDevice->isSubDevice()) { + if (hDevice->RefCount.release()) { + delete hDevice; + } + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urDeviceWaitExp(::ur_device_handle_t hDeviceOpque) { + auto hDevice = common_cast(hDeviceOpque); + ZE2UR_CALL(zeDeviceSynchronize, (hDevice->ZeDevice)); + return UR_RESULT_SUCCESS; +} + +/** + * @brief Determines the mode of immediate command lists to be used. + * + * This function checks environment variables and device properties to decide + * the mode of immediate command lists. The mode can be influenced by the + * following environment variables: + * - `UR_L0_USE_IMMEDIATE_COMMANDLISTS` + * - `SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS` + * + * If neither environment variable is set, the function defaults to using the + * device's properties to determine the mode. + * + * @return The mode of immediate command lists, which can be one of the + * following: + * - `NotUsed`: Immediate command lists are not used. + * - `PerQueue`: Immediate command lists are used per queue. + * - `PerThreadPerQueue`: Immediate command lists are used per thread per queue. + * + * The decision process is as follows: + * 1. If the environment variables are not set, the function checks if the + * device is Intel DG2 or newer and if the driver version is supported. If both + * conditions are met, or if the device is PVC, it returns `PerQueue`. + * Otherwise, it returns `NotUsed`. + * 2. If the environment variable is set, it returns the corresponding mode: + * - `0`: `NotUsed` + * - `1`: `PerQueue` + * - `2`: `PerThreadPerQueue` + * - Any other value: `NotUsed` + */ +ur_device_handle_t_::ImmCmdlistMode +ur_device_handle_t_::useImmediateCommandLists() { + // If immediate commandlist setting is not explicitly set, then use the device + // default. + // TODO: confirm this is good once make_queue revert is added + static const int ImmediateCommandlistsSetting = [] { + const char *UrRet = std::getenv("UR_L0_USE_IMMEDIATE_COMMANDLISTS"); + const char *PiRet = + std::getenv("SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS"); + const char *ImmediateCommandlistsSettingStr = + UrRet ? UrRet : (PiRet ? PiRet : nullptr); + if (!ImmediateCommandlistsSettingStr) + return -1; + return std::atoi(ImmediateCommandlistsSettingStr); + }(); + + if (ImmediateCommandlistsSetting == -1) { + bool isDG2OrNewer = this->isIntelDG2OrNewer(); + bool isDG2SupportedDriver = + this->Platform->isDriverVersionNewerOrSimilar(1, 5, 30820); + bool isIntelMTLDevice = this->isIntelMTL(); + bool isIntelARLDevice = this->isIntelARL(); + // Disable immediate command lists for DG2 devices on Windows due to driver + // limitations. + bool isLinux = true; +#ifdef _WIN32 + isLinux = false; +#endif + // Disable immediate command lists for Intel MTL/ARL devices on Linux by + // default due to driver limitations. + if ((isIntelMTLDevice || isIntelARLDevice) && isLinux) { + return NotUsed; + } + if ((isDG2SupportedDriver && isDG2OrNewer && isLinux) || isPVC() || + isNewerThanIntelDG2()) { + return PerQueue; + } else { + return NotUsed; + } + } + + UR_LOG(INFO, "NOTE: L0 Immediate CommandList Setting: {}", + ImmediateCommandlistsSetting); + + switch (ImmediateCommandlistsSetting) { + case 0: + return NotUsed; + case 1: + return PerQueue; + case 2: + return PerThreadPerQueue; + default: + return NotUsed; + } +} + +bool ur_device_handle_t_::useRelaxedAllocationLimits() { + static const bool EnableRelaxedAllocationLimits = [] { + auto UrRet = ur_getenv("UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS"); + const bool RetVal = UrRet ? std::stoi(*UrRet) : 0; + return RetVal; + }(); + + return EnableRelaxedAllocationLimits; +} + +bool ur_device_handle_t_::useDriverCounterBasedEvents() { + // Use counter-based events implementation from L0 driver. + + static const bool DriverCounterBasedEventsEnabled = [] { + const char *UrRet = std::getenv("UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS"); + if (!UrRet) { + return true; + } + return std::atoi(UrRet) != 0; + }(); + + return DriverCounterBasedEventsEnabled; +} + +ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal, + int SubSubDeviceIndex) { + // Maintain various device properties cache. + // Note that we just describe here how to compute the data. + // The real initialization is upon first access. + // + auto ZeDevice = this->ZeDevice; + ZeDeviceProperties.Compute = [ZeDevice](ze_device_properties_t &Properties) { + ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &Properties)); + }; + + ZeDeviceComputeProperties.Compute = + [ZeDevice](ze_device_compute_properties_t &Properties) { + ZE_CALL_NOCHECK(zeDeviceGetComputeProperties, (ZeDevice, &Properties)); + }; + + ZeDeviceIpVersionExt.Compute = + [ZeDevice](ze_device_ip_version_ext_t &Properties) { + ze_device_properties_t P; + P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + P.pNext = (void *)&Properties; + ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); + }; + + ZeDeviceImageProperties.Compute = + [ZeDevice](ze_device_image_properties_t &Properties) { + ZE_CALL_NOCHECK(zeDeviceGetImageProperties, (ZeDevice, &Properties)); + }; + + ZeDeviceModuleProperties.Compute = + [ZeDevice](ze_device_module_properties_t &Properties) { + ZE_CALL_NOCHECK(zeDeviceGetModuleProperties, (ZeDevice, &Properties)); + }; + + ZeDeviceMemoryProperties.Compute = + [ZeDevice]( + std::pair>, + std::vector>> + &Properties) { + uint32_t Count = 0; + ZE_CALL_NOCHECK(zeDeviceGetMemoryProperties, + (ZeDevice, &Count, nullptr)); + + auto &PropertiesVector = Properties.first; + auto &PropertiesExtVector = Properties.second; + + PropertiesVector.resize(Count); + PropertiesExtVector.resize(Count); + // Request for extended memory properties be read in + for (uint32_t I = 0; I < Count; ++I) + PropertiesVector[I].pNext = (void *)&PropertiesExtVector[I]; + + ZE_CALL_NOCHECK(zeDeviceGetMemoryProperties, + (ZeDevice, &Count, PropertiesVector.data())); + }; + + ZeDeviceMemoryAccessProperties.Compute = + [ZeDevice](ze_device_memory_access_properties_t &Properties) { + ZE_CALL_NOCHECK(zeDeviceGetMemoryAccessProperties, + (ZeDevice, &Properties)); + }; + + ZeDeviceCacheProperties.Compute = + [ZeDevice](ze_device_cache_properties_t &Properties) { + // TODO: Since v1.0 there can be multiple cache properties. + // For now remember the first one, if any. + uint32_t Count = 0; + ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, + (ZeDevice, &Count, nullptr)); + if (Count > 0) + Count = 1; + ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, + (ZeDevice, &Count, &Properties)); + }; + + if (this->Platform->zeDriverExtensionMap.count(ZE_CACHELINE_SIZE_EXT_NAME)) { + ZeDeviceCacheLinePropertiesExt.Compute = + [ZeDevice](ze_device_cache_line_size_ext_t &Properties) { + // TODO: Since v1.0 there can be multiple cache properties. + // For now remember the first one, if any. + uint32_t Count = 0; + ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, + (ZeDevice, &Count, nullptr)); + if (Count > 0) + Count = 1; + ze_device_cache_properties_t P; + P.stype = ZE_STRUCTURE_TYPE_DEVICE_CACHE_PROPERTIES; + P.pNext = &Properties; + ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, (ZeDevice, &Count, &P)); + if (Properties.cacheLineSize == 0) { + // If cache line size is not set, use the default value. + Properties.cacheLineSize = + 1; // Default cache line size property value. + } + }; + } + + ZeDeviceMutableCmdListsProperties.Compute = + [ZeDevice]( + ZeStruct &Properties) { + ze_device_properties_t P; + P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + P.pNext = &Properties; + ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); + }; + +#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME + ZeDeviceBlockArrayProperties.Compute = + [ZeDevice]( + ZexStruct &Properties) { + ze_device_properties_t P; + P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + P.pNext = &Properties; + ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); + }; +#endif // ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME + + auto UrPlatform = this->Platform; + ZeDeviceVectorWidthPropertiesExt.Compute = + [ZeDevice, UrPlatform]( + ZeStruct &Properties) { + // Set default vector width properties + Properties.preferred_vector_width_char = 16u; + Properties.preferred_vector_width_short = 8u; + Properties.preferred_vector_width_int = 4u; + Properties.preferred_vector_width_long = 1u; + Properties.preferred_vector_width_float = 1u; + Properties.preferred_vector_width_half = 8u; + Properties.native_vector_width_char = 16u; + Properties.native_vector_width_short = 8u; + Properties.native_vector_width_int = 4u; + Properties.native_vector_width_long = 1u; + Properties.native_vector_width_float = 1u; + Properties.native_vector_width_half = 8u; + + if (UrPlatform->zeDriverExtensionMap.count( + ZE_DEVICE_VECTOR_SIZES_EXT_NAME)) { + uint32_t Count = 0; + ZE_CALL_NOCHECK(zeDeviceGetVectorWidthPropertiesExt, + (ZeDevice, &Count, nullptr)); + + std::vector> + PropertiesVector; + PropertiesVector.resize(Count); + + ZeStruct + MaxVectorWidthProperties; + + ZE_CALL_NOCHECK(zeDeviceGetVectorWidthPropertiesExt, + (ZeDevice, &Count, PropertiesVector.data())); + if (!PropertiesVector.empty()) { + // Find the largest vector_width_size property + uint32_t max_vector_width_size = 0; + for (const auto &prop : PropertiesVector) { + if (!max_vector_width_size) { + max_vector_width_size = prop.vector_width_size; + MaxVectorWidthProperties = prop; + } else if (prop.vector_width_size > max_vector_width_size) { + max_vector_width_size = prop.vector_width_size; + MaxVectorWidthProperties = prop; + } + } + Properties = MaxVectorWidthProperties; + // If the environment variable is set, use the specified vector + // width if it exists + if (UrL0VectorWidth) { + for (const auto &prop : PropertiesVector) { + if (prop.vector_width_size == + static_cast(UrL0VectorWidth)) { + Properties = prop; + break; + } + } + } + } + } + }; + + ZeXEDeviceProperties.Compute = + [ZeDevice](ze_intel_xe_device_exp_properties_t &Properties) { + // Set up default values of 0 + Properties.numXeStacks = 0; + Properties.numXeRegionsPerStack = 0; + Properties.numXeClustersPerRegion = 0; + Properties.numXeCorePerCluster = 0; + Properties.numExecutionEnginesPerXeCore = 0; + Properties.maxNumHwThreadsPerExecutionEngine = 0; + Properties.maxNumLanesPerHwThread = 0; + + ze_device_properties_t P; + P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + P.pNext = (void *)&Properties; + ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); + }; + + ZeDevicePitchedAllocProperties.Compute = + [ZeDevice](ZeDevicePitchedAllocInfo &Properties) { + ze_device_image_properties_t ImageProps = {}; + ImageProps.stype = ZE_STRUCTURE_TYPE_DEVICE_IMAGE_PROPERTIES; + ImageProps.pNext = &Properties.AllocProps; + Properties.AllocProps.pNext = &Properties.PitchInfo; + ZE_CALL_NOCHECK(zeDeviceGetImageProperties, (ZeDevice, &ImageProps)); + }; + + ImmCommandListUsed = this->useImmediateCommandLists(); + + uint32_t numQueueGroups = 0; + ZE2UR_CALL(zeDeviceGetCommandQueueGroupProperties, + (ZeDevice, &numQueueGroups, nullptr)); + if (numQueueGroups == 0) { + return UR_RESULT_ERROR_UNKNOWN; + } + UR_LOG_LEGACY(INFO, + logger::LegacyMessage("NOTE: Number of queue groups = {}"), + "Number of queue groups = {}", numQueueGroups); + + std::vector> + QueueGroupProperties(numQueueGroups); + ZE2UR_CALL(zeDeviceGetCommandQueueGroupProperties, + (ZeDevice, &numQueueGroups, QueueGroupProperties.data())); + + // Initialize ordinal and compute queue group properties + for (uint32_t i = 0; i < numQueueGroups; i++) { + if (QueueGroupProperties[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) { + QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal = + i; + QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] + .ZeProperties = QueueGroupProperties[i]; + break; + } + } + + // Reinitialize a sub-sub-device with its own ordinal, index. + // Our sub-sub-device representation is currently [Level-Zero sub-device + // handle + Level-Zero compute group/engine index]. Only the specified + // index queue will be used to submit work to the sub-sub-device. + if (SubSubDeviceOrdinal >= 0) { + QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal = + SubSubDeviceOrdinal; + QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeIndex = + SubSubDeviceIndex; + } else { // Proceed with initialization for root and sub-device + // How is it possible that there are no "compute" capabilities? + if (QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal < + 0) { + return UR_RESULT_ERROR_UNKNOWN; + } + + if (ur::level_zero::CopyEngineRequested((ur_device_handle_t)this)) { + for (uint32_t i = 0; i < numQueueGroups; i++) { + if (((QueueGroupProperties[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) == 0) && + (QueueGroupProperties[i].flags & + ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY)) { + if (QueueGroupProperties[i].numQueues == 1) { + QueueGroup[queue_group_info_t::MainCopy].ZeOrdinal = i; + QueueGroup[queue_group_info_t::MainCopy].ZeProperties = + QueueGroupProperties[i]; + } else { + QueueGroup[queue_group_info_t::LinkCopy].ZeOrdinal = i; + QueueGroup[queue_group_info_t::LinkCopy].ZeProperties = + QueueGroupProperties[i]; + break; + } + } + } + if (QueueGroup[queue_group_info_t::MainCopy].ZeOrdinal < 0) + UR_LOG_LEGACY(INFO, + logger::LegacyMessage( + "NOTE: main blitter/copy engine is not available"), + "main blitter/copy engine is not available") + else + UR_LOG_LEGACY(INFO, + logger::LegacyMessage( + "NOTE: main blitter/copy engine is available"), + "main blitter/copy engine is available") + + if (QueueGroup[queue_group_info_t::LinkCopy].ZeOrdinal < 0) + UR_LOG_LEGACY(INFO, + logger::LegacyMessage( + "NOTE: link blitter/copy engines are not available"), + "link blitter/copy engines are not available") + else + UR_LOG_LEGACY(INFO, + logger::LegacyMessage( + "NOTE: link blitter/copy engines are available"), + "link blitter/copy engines are available") + } + } + + return UR_RESULT_SUCCESS; +} + +void ZeDriverVersionStringExtension::setZeDriverVersionString( + ur_platform_handle_t_ *Platform) { + // Check if Intel Driver Version String is available. If yes, save the API + // pointer. The pointer will be used when reading the Driver Version for + // users. + ze_driver_handle_t DriverHandle = Platform->ZeDriver; + if (auto extension = Platform->zeDriverExtensionMap.find( + "ZE_intel_get_driver_version_string"); + extension != Platform->zeDriverExtensionMap.end()) { + if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress, + (DriverHandle, "zeIntelGetDriverVersionString", + reinterpret_cast( + &zeIntelGetDriverVersionStringPointer))) == 0) { + // Intel Driver Version String is Supported by this Driver. + Supported = true; + } + } +} + +void ZeDriverVersionStringExtension::getDriverVersionString( + ze_driver_handle_t DriverHandle, char *pDriverVersion, + size_t *pVersionSize) { + ZE_CALL_NOCHECK(zeIntelGetDriverVersionStringPointer, + (DriverHandle, pDriverVersion, pVersionSize)); +} + +void ZeUSMImportExtension::setZeUSMImport(ur_platform_handle_t_ *Platform) { + // Check if USM hostptr import feature is available. If yes, save the API + // pointers. The pointers will be used for both import/release of SYCL buffer + // host ptr and the SYCL experimental APIs, prepare_for_device_copy and + // release_from_device_copy. + ze_driver_handle_t DriverHandle = Platform->ZeDriver; + if (ZE_CALL_NOCHECK( + zeDriverGetExtensionFunctionAddress, + (DriverHandle, "zexDriverImportExternalPointer", + reinterpret_cast(&zexDriverImportExternalPointer))) == 0) { + ZE_CALL_NOCHECK( + zeDriverGetExtensionFunctionAddress, + (DriverHandle, "zexDriverReleaseImportedPointer", + reinterpret_cast(&zexDriverReleaseImportedPointer))); + // Hostptr import/release is supported by this platform. + Supported = true; + + // Check if env var SYCL_USM_HOSTPTR_IMPORT has been set requesting + // host ptr import during buffer creation. + const char *USMHostPtrImportStr = std::getenv("SYCL_USM_HOSTPTR_IMPORT"); + if (!USMHostPtrImportStr || std::atoi(USMHostPtrImportStr) == 0) + return; + + // Hostptr import/release is turned on because it has been requested + // by the env var, and this platform supports the APIs. + Enabled = true; + // Hostptr import is only possible if piMemBufferCreate receives a + // hostptr as an argument. The SYCL runtime passes a host ptr + // only when SYCL_HOST_UNIFIED_MEMORY is enabled. Therefore we turn it on. + setEnvVar("SYCL_HOST_UNIFIED_MEMORY", "1"); + } +} +void ZeUSMImportExtension::doZeUSMImport(ze_driver_handle_t DriverHandle, + void *HostPtr, size_t Size) { + ZE_CALL_NOCHECK(zexDriverImportExternalPointer, + (DriverHandle, HostPtr, Size)); +} +void ZeUSMImportExtension::doZeUSMRelease(ze_driver_handle_t DriverHandle, + void *HostPtr) { + ZE_CALL_NOCHECK(zexDriverReleaseImportedPointer, (DriverHandle, HostPtr)); +} + +bool ur_context_common_t::isValidDevice(ur_device_handle_t hDevice) const { + while (hDevice) { + if (std::find(hDevices.begin(), hDevices.end(), hDevice) != hDevices.end()) + return true; + hDevice = hDevice->RootDevice; + } + return false; +} + +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_ const &device_handle) { + if (device_handle.Id.has_value()) { + return os << device_handle.Id.value(); + } + return os << "NONE"; +} + +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_::PeerStatus peer_status) { + switch (peer_status) { + case ur_device_handle_t_::PeerStatus::DISABLED: + return os << "DISABLED"; + case ur_device_handle_t_::PeerStatus::ENABLED: + return os << "ENABLED"; + case ur_device_handle_t_::PeerStatus::NO_CONNECTION: + return os << "NO_CONNECTION"; + } + return os << "UNKNOWN"; +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/common/device.hpp similarity index 96% rename from unified-runtime/source/adapters/level_zero/device.hpp rename to unified-runtime/source/adapters/level_zero/common/device.hpp index a557bf927730f..13800f9ed53e6 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/common/device.hpp @@ -1,4 +1,4 @@ -//===--------- device.hpp - Level Zero Adapter ----------------------------===// +//===--------- device.hpp - Level Zero Adapter ---------------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -14,18 +14,27 @@ #include #include #include +#include #include #include #include -#include "adapters/level_zero/platform.hpp" -#include "common.hpp" +#include "api.hpp" #include "common/ur_ref_count.hpp" +#include "helpers/shared_helpers.hpp" +#include "interfaces.hpp" #include #include #include #include +namespace ur::level_zero { + +typedef size_t DeviceId; + +struct ur_device_handle_t_; +typedef struct ur_device_handle_t_ *ur_device_handle_t; + enum EventsScope { // All events are created host-visible. AllHostVisible, @@ -56,13 +65,12 @@ struct ur_ze_external_memory_data { size_t size; }; -struct ur_device_handle_t_ : ur_object { +struct ur_device_handle_t_ : ur::level_zero::ur_object_t { ur_device_handle_t_(ze_device_handle_t Device, ur_platform_handle_t Plt, ur_device_handle_t ParentDevice = nullptr) : ZeDevice{Device}, Platform{Plt}, RootDevice{ParentDevice}, ZeDeviceProperties{}, ZeDeviceComputeProperties{}, Id(std::nullopt) { - // NOTE: one must additionally call initialize() to complete - // UR device creation. + ddi_table = ddiTableOf(Plt); } // The helper structure that keeps info about a command queue groups of the @@ -313,3 +321,5 @@ inline std::vector CollectDevicesForUsmPoolCreation( return DevicesAndSubDevices; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/enqueued_pool.cpp b/unified-runtime/source/adapters/level_zero/common/enqueued_pool.cpp similarity index 93% rename from unified-runtime/source/adapters/level_zero/enqueued_pool.cpp rename to unified-runtime/source/adapters/level_zero/common/enqueued_pool.cpp index cc4f0afe6b7bb..21317d466e94f 100644 --- a/unified-runtime/source/adapters/level_zero/enqueued_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/common/enqueued_pool.cpp @@ -8,9 +8,12 @@ //===----------------------------------------------------------------------===// #include "enqueued_pool.hpp" +#include "../usm.hpp" #include +namespace ur::level_zero { + EnqueuedPool::~EnqueuedPool() { cleanup(); } std::optional EnqueuedPool::getBestFit(size_t Size, @@ -30,7 +33,7 @@ std::optional EnqueuedPool::getBestFit(size_t Size, return std::nullopt; } -void EnqueuedPool::insert(void *Ptr, size_t Size, ur_event_handle_t Event, +void EnqueuedPool::insert(void *Ptr, size_t Size, ::ur_event_handle_t Event, void *Queue) { auto Lock = std::lock_guard(Mutex); @@ -78,3 +81,5 @@ bool EnqueuedPool::cleanupForQueue(void *Queue) { return FreedAllocations; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/enqueued_pool.hpp b/unified-runtime/source/adapters/level_zero/common/enqueued_pool.hpp similarity index 74% rename from unified-runtime/source/adapters/level_zero/enqueued_pool.hpp rename to unified-runtime/source/adapters/level_zero/common/enqueued_pool.hpp index 7236cfadc9b8a..ceb78039aa365 100644 --- a/unified-runtime/source/adapters/level_zero/enqueued_pool.hpp +++ b/unified-runtime/source/adapters/level_zero/common/enqueued_pool.hpp @@ -8,26 +8,36 @@ //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +// Pulls in the L0 umf2urResult customization point (defines +// UMF_GET_PROVIDER_NATIVE_ERROR) and itself. Must precede any +// other include that reaches . +#include "helpers/shared_helpers.hpp" #include "unified-runtime/ur_api.h" -#include "ur_pool_manager.hpp" +#include +#include #include -#include + +#include + +#include "ur_pool_manager.hpp" +#include + +namespace ur::level_zero { class EnqueuedPool { public: struct Allocation { void *Ptr; size_t Size; - ur_event_handle_t Event; + ::ur_event_handle_t Event; // Queue handle, used as an identifier for the associated queue. // This can either be a `ur_queue_handle_t` or a pointer to a v2 queue // object. void *Queue; }; - using event_release_callback_t = ur_result_t (*)(ur_event_handle_t); + using event_release_callback_t = ur_result_t (*)(::ur_event_handle_t); using memory_free_callback_t = std::function; EnqueuedPool(event_release_callback_t EventReleaseFn, @@ -37,7 +47,7 @@ class EnqueuedPool { ~EnqueuedPool(); std::optional getBestFit(size_t Size, void *Queue); - void insert(void *Ptr, size_t Size, ur_event_handle_t Event, void *Queue); + void insert(void *Ptr, size_t Size, ::ur_event_handle_t Event, void *Queue); bool cleanup(); bool cleanupForQueue(void *Queue); @@ -60,3 +70,5 @@ class EnqueuedPool { event_release_callback_t EventReleaseFn; memory_free_callback_t MemFreeFn; }; + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.cpp b/unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.cpp similarity index 96% rename from unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.cpp rename to unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.cpp index 52e897b3fa7d5..9706efa549d62 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.cpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.cpp @@ -10,14 +10,11 @@ #include "kernel_helpers.hpp" #include "logger/ur_logger.hpp" -#include "../common.hpp" #include "../device.hpp" +#include "../interfaces.hpp" +#include "../platform.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "../v2/context.hpp" -#else -#include "../context.hpp" -#endif +namespace ur::level_zero { ur_result_t getSuggestedLocalWorkSize(ur_device_handle_t hDevice, ze_kernel_handle_t hZeKernel, @@ -68,7 +65,9 @@ ur_result_t getSuggestedLocalWorkSize(ur_device_handle_t hDevice, ur_result_t setKernelGlobalOffset(ur_context_handle_t Context, ze_kernel_handle_t Kernel, uint32_t WorkDim, const size_t *GlobalWorkOffset) { - if (!Context->getPlatform()->ZeDriverGlobalOffsetExtensionFound) { + if (!common_cast(Context) + ->getPlatform() + ->ZeDriverGlobalOffsetExtensionFound) { UR_LOG(DEBUG, "No global offset extension found on this driver"); return UR_RESULT_ERROR_INVALID_VALUE; } @@ -170,3 +169,5 @@ ur_result_t setArgValueOnZeKernel(ze_kernel_handle_t hZeKernel, return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE; return ze2urResult(ZeResult); } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.hpp b/unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.hpp similarity index 97% rename from unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.hpp rename to unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.hpp index a1bb26ed70245..de84162ff024c 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/kernel_helpers.hpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/kernel_helpers.hpp @@ -7,9 +7,12 @@ // //===----------------------------------------------------------------------===// +#include "../device.hpp" #include #include +namespace ur::level_zero { + /** * Calculates a work group size for the kernel based on the GlobalWorkSize or * the LocalWorkSize if provided. @@ -82,3 +85,5 @@ inline void postSubmit(ze_kernel_handle_t hZeKernel, ur_result_t setArgValueOnZeKernel(ze_kernel_handle_t hZeKernel, uint32_t argIndex, size_t argSize, const void *pArgValue); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/memory_helpers.cpp b/unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.cpp similarity index 98% rename from unified-runtime/source/adapters/level_zero/helpers/memory_helpers.cpp rename to unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.cpp index 51a75bbd5fc75..2ac58f9591d3f 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/memory_helpers.cpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.cpp @@ -8,7 +8,8 @@ //===----------------------------------------------------------------------===// #include "memory_helpers.hpp" -#include "../common.hpp" + +namespace ur::level_zero { ur_result_t getMemoryAttrs(ze_context_handle_t hContext, void *ptr, @@ -77,3 +78,5 @@ ze_region_params ur2zeRegionParams(ur_rect_offset_t SrcOrigin, return ze_region_params{ZeDstRegion, DstPitch, DstSlicePitch, ZeSrcRegion, SrcPitch, SrcSlicePitch}; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/memory_helpers.hpp b/unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.hpp similarity index 95% rename from unified-runtime/source/adapters/level_zero/helpers/memory_helpers.hpp rename to unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.hpp index 9fe24ed0a8e4e..ea8121acbcac6 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/memory_helpers.hpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/memory_helpers.hpp @@ -13,7 +13,9 @@ #include -#include "../common.hpp" +#include "shared_helpers.hpp" + +namespace ur::level_zero { // If USM Import feature is enabled and hostptr is supplied, // import the hostptr if not already imported into USM. @@ -44,3 +46,5 @@ ze_region_params ur2zeRegionParams(ur_rect_offset_t SrcOrigin, ur_rect_region_t Region, size_t SrcRowPitch, size_t DstRowPitch, size_t SrcSlicePitch, size_t DstSlicePitch); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.cpp b/unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.cpp similarity index 95% rename from unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.cpp rename to unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.cpp index d0fbd2b6cdcd1..220205ef36769 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.cpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.cpp @@ -9,9 +9,12 @@ #include "mutable_helpers.hpp" #include "../device.hpp" -#include "../ur_interface_loader.hpp" -#include "../ur_level_zero.hpp" +#include "adapters/level_zero/common/command_buffer_command.hpp" #include "kernel_helpers.hpp" +#include "unified-runtime/ur_api.h" +#include + +namespace ur::level_zero { using desc_storage_t = std::vector>, @@ -315,10 +318,13 @@ ur_result_t updateCommandBufferUnlocked( // several L0 update descriptor structs. for (uint32_t i = 0; i < NumKernelUpdates; i++) { const auto &CommandDesc = CommandDescs[i]; - auto Command = static_cast(CommandDesc.hCommand); + auto Command = + static_cast(common_cast(CommandDesc.hCommand)); - std::scoped_lock Guard( - Command->Mutex, Command->kernel->Mutex); + auto &KernelMutex = + reinterpret_cast(Command->kernel)->Mutex; + std::scoped_lock Guard(Command->Mutex, + KernelMutex); ur_kernel_handle_t NewKernel = CommandDesc.hNewKernel; if (NewKernel && Command->kernel != NewKernel) { @@ -364,7 +370,8 @@ ur_result_t validateCommandDescUnlocked( for (size_t i = 0; i < CommandDescSize; i++) { const auto &CommandDesc = CommandDescs[i]; - auto Command = static_cast(CommandDesc.hCommand); + auto Command = + static_cast(common_cast(CommandDesc.hCommand)); UR_ASSERT(CommandBuffer == Command->commandBuffer, UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP); @@ -444,6 +451,9 @@ ur_result_t createCommandHandleUnlocked( ur_platform_handle_t Platform, ur_result_t (*GetZeKernel)(ur_kernel_handle_t, ze_kernel_handle_t &, ur_device_handle_t), + kernel_command_handle *(*CreateCommandHandle)( + ur_exp_command_buffer_handle_t, ur_kernel_handle_t, uint64_t, uint32_t, + uint32_t, ur_kernel_handle_t *), ur_device_handle_t Device, std::unique_ptr &Command) { @@ -505,10 +515,9 @@ ur_result_t createCommandHandleUnlocked( } try { - Command = std::make_unique( - CommandBuffer, Kernel, CommandId, WorkDim, NumKernelAlternatives, - KernelAlternatives); - + Command = std::unique_ptr( + CreateCommandHandle(CommandBuffer, Kernel, CommandId, WorkDim, + NumKernelAlternatives, KernelAlternatives)); Command->setGlobalWorkSize(GlobalWorkSize); } catch (const std::bad_alloc &) { @@ -519,3 +528,5 @@ ur_result_t createCommandHandleUnlocked( return UR_RESULT_SUCCESS; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.hpp b/unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.hpp similarity index 87% rename from unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.hpp rename to unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.hpp index 23b1691f5bf87..24ec5e3b5579b 100644 --- a/unified-runtime/source/adapters/level_zero/helpers/mutable_helpers.hpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/mutable_helpers.hpp @@ -14,10 +14,13 @@ #include #include "../command_buffer_command.hpp" -#include "../common.hpp" +#include "../device.hpp" +#include "../platform.hpp" #include "logger/ur_logger.hpp" #include +namespace ur::level_zero { + using device_ptr_storage_t = std::vector>; ur_result_t updateCommandBufferUnlocked( @@ -45,4 +48,9 @@ ur_result_t createCommandHandleUnlocked( ur_platform_handle_t Platform, ur_result_t (*getZeKernel)(ur_kernel_handle_t, ze_kernel_handle_t &, ur_device_handle_t), - ur_device_handle_t Device, std::unique_ptr &Command); \ No newline at end of file + kernel_command_handle *(*CreateCommandHandle)( + ur_exp_command_buffer_handle_t, ur_kernel_handle_t, uint64_t, uint32_t, + uint32_t, ur_kernel_handle_t *), + ur_device_handle_t Device, std::unique_ptr &Command); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/common.cpp b/unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.cpp similarity index 98% rename from unified-runtime/source/adapters/level_zero/common.cpp rename to unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.cpp index e298a9f2eab1b..8e452447c2cad 100644 --- a/unified-runtime/source/adapters/level_zero/common.cpp +++ b/unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.cpp @@ -7,10 +7,15 @@ // //===----------------------------------------------------------------------===// -#include "common.hpp" +#include "shared_helpers.hpp" #include "logger/ur_logger.hpp" -#include "usm.hpp" -#include +#include +#include +#include + +std::mutex ur::level_zero::ZeCall::GlobalLock; + +namespace ur::level_zero { ur_result_t ze2urResult(ze_result_t ZeResult) { if (ZeResult == ZE_RESULT_SUCCESS) @@ -396,3 +401,5 @@ thread_local int32_t ErrorAdapterNativeCode; ErrorMessageCode = ErrorCode; ErrorAdapterNativeCode = AdapterErrorCode; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.hpp b/unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.hpp new file mode 100644 index 0000000000000..447719f9a08ce --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/common/helpers/shared_helpers.hpp @@ -0,0 +1,345 @@ +//===--------- common.hpp - Level Zero Adapter ----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include "windows.h" +#else +#include +#include +#endif + +#include +#include +#include +#include +#include + +#include +#include + +#include "../interfaces.hpp" +#include "common/ur_ref_count.hpp" +#include "logger/ur_logger.hpp" + +namespace ur::level_zero { + +[[maybe_unused]] static bool checkL0LoaderTeardown() { + try { + if (!zelCheckIsLoaderInTearDown()) { + return true; + } + } catch (...) { + } + UR_LOG(DEBUG, + "ZE ---> checkL0LoaderTeardown: Loader is in teardown or is unstable"); + return false; +} + +// Controls UR L0 calls tracing. +enum UrDebugLevel { + UR_L0_DEBUG_NONE = 0x0, + UR_L0_DEBUG_BASIC = 0x1, + UR_L0_DEBUG_VALIDATION = 0x2, + UR_L0_DEBUG_ALL = -1 +}; + +const int UrL0Debug = [] { + const char *ZeDebugMode = std::getenv("ZE_DEBUG"); + const char *UrL0DebugMode = std::getenv("UR_L0_DEBUG"); + uint32_t DebugMode = 0; + if (UrL0DebugMode) { + DebugMode = std::atoi(UrL0DebugMode); + } else if (ZeDebugMode) { + DebugMode = std::atoi(ZeDebugMode); + } + return DebugMode; +}(); + +const int UrL0LeaksDebug = [] { + const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG"); + if (!UrRet) + return 0; + return std::atoi(UrRet); +}(); + +const int UrL0VectorWidth = [] { + const char *UrRet = std::getenv("UR_L0_VECTOR_WIDTH_SIZE"); + if (!UrRet) + return 0; + return std::atoi(UrRet); +}(); + +// Enable for UR L0 Adapter to Init all L0 Drivers on the system with filtering +// in place for only currently used Drivers. +const int UrL0InitAllDrivers = [] { + const char *UrRet = std::getenv("UR_L0_INIT_ALL_DRIVERS"); + if (!UrRet) + return 0; + return std::atoi(UrRet); +}(); + +// Controls Level Zero calls serialization to w/a Level Zero driver being not MT +// ready. Recognized values (can be used as a bit mask): +enum { + UrL0SerializeNone = + 0, // no locking or blocking (except when SYCL RT requested blocking) + UrL0SerializeLock = 1, // locking around each UR_CALL + UrL0SerializeBlock = + 2, // blocking UR calls, where supported (usually in enqueue commands) +}; + +static const uint32_t UrL0Serialize = [] { + const char *ZeSerializeMode = std::getenv("ZE_SERIALIZE"); + const char *UrL0SerializeMode = std::getenv("UR_L0_SERIALIZE"); + uint32_t SerializeModeValue = 0; + if (UrL0SerializeMode) { + SerializeModeValue = std::atoi(UrL0SerializeMode); + } else if (ZeSerializeMode) { + SerializeModeValue = std::atoi(ZeSerializeMode); + } + return SerializeModeValue; +}(); + +static const uint32_t UrL0QueueSyncNonBlocking = [] { + const char *UrL0QueueSyncNonBlocking = + std::getenv("UR_L0_QUEUE_SYNCHRONIZE_NON_BLOCKING"); + uint32_t L0QueueSyncLockingModeValue = 1; + if (UrL0QueueSyncNonBlocking) { + L0QueueSyncLockingModeValue = std::atoi(UrL0QueueSyncNonBlocking); + } + return L0QueueSyncLockingModeValue; +}(); + +// Controls whether the L0 Adapter creates signal events for commands on +// integrated gpu devices. +static const uint32_t UrL0OutOfOrderIntegratedSignalEvent = [] { + const char *UrL0OutOfOrderIntegratedSignalEventEnv = + std::getenv("UR_L0_OOQ_INTEGRATED_SIGNAL_EVENT"); + uint32_t UrL0OutOfOrderIntegratedSignalEventValue = 1; + if (UrL0OutOfOrderIntegratedSignalEventEnv) { + UrL0OutOfOrderIntegratedSignalEventValue = + std::atoi(UrL0OutOfOrderIntegratedSignalEventEnv); + } + return UrL0OutOfOrderIntegratedSignalEventValue; +}(); + +// This class encapsulates actions taken along with a call to Level Zero API. +class ZeCall { +private: + // The global mutex that is used for total serialization of Level Zero calls. + static std::mutex GlobalLock; + +public: + ZeCall() { + if ((UrL0Serialize & UrL0SerializeLock) != 0) { + GlobalLock.lock(); + } + } + ~ZeCall() { + if ((UrL0Serialize & UrL0SerializeLock) != 0) { + GlobalLock.unlock(); + } + } + + // The non-static version just calls static one. + ze_result_t doCall(ze_result_t ZeResult, const char *ZeName, + const char *ZeArgs, bool TraceError = true); +}; + +// This function will ensure compatibility with both Linux and Windows for +// setting environment variables. +bool setEnvVar(const char *name, const char *value); + +// Returns the ze_structure_type_t to use in .stype of a structured descriptor. +// Intentionally not defined; will give an error if no proper specialization +template ze_structure_type_t getZeStructureType(); +template ze_structure_type_ext_t getZexStructureType(); +template zes_structure_type_t getZesStructureType(); + +// The helpers to properly default initialize Level-Zero descriptor and +// properties structures. +template struct ZeStruct : public T { + ZeStruct() : T{} { // zero initializes base struct + this->stype = getZeStructureType(); + this->pNext = nullptr; + } +}; + +template struct ZexStruct : public T { + ZexStruct() : T{} { // zero initializes base struct + this->stype = getZexStructureType(); + this->pNext = nullptr; + } +}; + +template struct ZesStruct : public T { + ZesStruct() : T{} { // zero initializes base struct + this->stype = getZesStructureType(); + this->pNext = nullptr; + } +}; + +// Map Level Zero runtime error code to UR error code. +ur_result_t ze2urResult(ze_result_t ZeResult); + +// Parse Level Zero error code and return the error string. +void zeParseError(ze_result_t ZeError, const char *&ErrorString); + +// Trace a call to Level-Zero RT. +#define ZE2UR_CALL(ZeName, ZeArgs) \ + { \ + ze_result_t ZeResult = ZeName ZeArgs; \ + if (auto Result = ::ur::level_zero::ZeCall().doCall(ZeResult, #ZeName, \ + #ZeArgs, true)) \ + return ::ur::level_zero::ze2urResult(Result); \ + } + +// Trace a call to Level-Zero RT, throw on error +#define ZE2UR_CALL_THROWS(ZeName, ZeArgs) \ + { \ + ze_result_t ZeResult = ZeName ZeArgs; \ + if (auto Result = ::ur::level_zero::ZeCall().doCall(ZeResult, #ZeName, \ + #ZeArgs, true)) { \ + throw ::ur::level_zero::ze2urResult(Result); \ + } \ + } + +// Perform traced call to L0 without checking for errors +#define ZE_CALL_NOCHECK(ZeName, ZeArgs) \ + ::ur::level_zero::ZeCall().doCall(ZeName ZeArgs, #ZeName, #ZeArgs, false) + +#define ZE_CALL_NOCHECK_NAME(ZeName, ZeArgs, callName) \ + ::ur::level_zero::ZeCall().doCall(ZeName ZeArgs, callName, #ZeArgs, false) + +extern const bool UseUSMAllocator; + +// Controls support of the indirect access kernels and deferred memory release. +const bool IndirectAccessTrackingEnabled = [] { + char *UrRet = std::getenv("UR_L0_TRACK_INDIRECT_ACCESS_MEMORY"); + char *PiRet = std::getenv("SYCL_PI_LEVEL_ZERO_TRACK_INDIRECT_ACCESS_MEMORY"); + const bool RetVal = UrRet ? std::stoi(UrRet) : (PiRet ? std::stoi(PiRet) : 0); + return RetVal; +}(); + +const bool ExposeCSliceInAffinityPartitioning = [] { + char *UrRet = std::getenv("UR_L0_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING"); + char *PiRet = + std::getenv("SYCL_PI_LEVEL_ZERO_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING"); + const char *Flag = UrRet ? UrRet : (PiRet ? PiRet : 0); + return Flag ? std::atoi(Flag) != 0 : false; +}(); + +// TODO: make it into a ur_device_handle_t class member +const std::pair +getRangeOfAllowedCopyEngines(const ur_device_handle_t &Device); + +class ZeDriverVersionStringExtension { + // Pointer to function for Intel Driver Version String + ze_result_t (*zeIntelGetDriverVersionStringPointer)( + ze_driver_handle_t hDriver, char *, size_t *) = nullptr; + +public: + // Whether platform supports Intel Driver Version String. + bool Supported; + + ZeDriverVersionStringExtension() : Supported{false} {} + + void setZeDriverVersionString(ur_platform_handle_t_ *Platform); + void getDriverVersionString(ze_driver_handle_t DriverHandle, + char *pDriverVersion, size_t *pVersionSize); +}; + +class ZeUSMImportExtension { + // Pointers to functions that import/release host memory into USM + ze_result_t (*zexDriverImportExternalPointer)(ze_driver_handle_t hDriver, + void *, size_t) = nullptr; + ze_result_t (*zexDriverReleaseImportedPointer)(ze_driver_handle_t, + void *) = nullptr; + +public: + // Whether platform supports Import/Release. + bool Supported; + + // Whether user has requested Import/Release for buffers. + bool Enabled; + + ZeUSMImportExtension() : Supported{false}, Enabled{false} {} + + void setZeUSMImport(ur_platform_handle_t_ *Platform); + void doZeUSMImport(ze_driver_handle_t DriverHandle, void *HostPtr, + size_t Size); + void doZeUSMRelease(ze_driver_handle_t DriverHandle, void *HostPtr); +}; + +// Helper wrapper for working with USM import extension in Level Zero. +extern ZeUSMImportExtension ZeUSMImport; + +// Some opencl extensions we know are supported by all Level Zero devices. +constexpr char ZE_SUPPORTED_EXTENSIONS[] = + "cl_khr_il_program cl_khr_subgroups cl_intel_subgroups " + "cl_intel_subgroups_short cl_intel_required_subgroup_size "; + +// Global variables for ZER_EXT_RESULT_ADAPTER_SPECIFIC_ERROR +constexpr size_t MaxMessageSize = 256; +extern thread_local int32_t ErrorMessageCode; +extern thread_local char ErrorMessage[MaxMessageSize]; +extern thread_local int32_t ErrorAdapterNativeCode; + +// Utility function for setting a message and warning +[[maybe_unused]] void setErrorMessage(const char *pMessage, int32_t ErrorCode, + int32_t AdapterErrorCode); + +// Returns the host memory page size in bytes. +inline size_t getHostPageSize() { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return static_cast(SystemInfo.dwPageSize); +#else + return static_cast(sysconf(_SC_PAGESIZE)); +#endif +} + +} // namespace ur::level_zero + +// Per-adapter translator for umf2urResult's provider-specific error path. +namespace umf { +namespace { +[[maybe_unused]] inline ur_result_t +levelZeroGetProviderNativeError(const char *providerName, int32_t nativeError) { + if (strcmp(providerName, "LEVEL_ZERO") == 0) { + auto zeResult = static_cast(nativeError); + if (zeResult == ZE_RESULT_ERROR_UNSUPPORTED_SIZE) { + return UR_RESULT_ERROR_INVALID_USM_SIZE; + } + return ur::level_zero::ze2urResult(zeResult); + } + if (strcmp(providerName, "Level Zero") == 0) { + return static_cast(nativeError); + } + return UR_RESULT_ERROR_UNKNOWN; +} +} // namespace +} // namespace umf + +#define UMF_GET_PROVIDER_NATIVE_ERROR ::umf::levelZeroGetProviderNativeError + +#include +#include diff --git a/unified-runtime/source/adapters/level_zero/image_common.cpp b/unified-runtime/source/adapters/level_zero/common/image_common.cpp similarity index 91% rename from unified-runtime/source/adapters/level_zero/image_common.cpp rename to unified-runtime/source/adapters/level_zero/common/image_common.cpp index fc1b494a6a258..d62c197d98716 100644 --- a/unified-runtime/source/adapters/level_zero/image_common.cpp +++ b/unified-runtime/source/adapters/level_zero/common/image_common.cpp @@ -9,19 +9,15 @@ #include -#include "common.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else -#include "context.hpp" -#endif +#include "device.hpp" #include "helpers/memory_helpers.hpp" #include "image_common.hpp" +#include "interfaces.hpp" #include "logger/ur_logger.hpp" #include "platform.hpp" #include "sampler.hpp" -#include "ur_interface_loader.hpp" +namespace ur::level_zero { namespace { /// Construct UR image format from ZE image desc. @@ -303,8 +299,10 @@ ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, ZeStruct ZeSamplerDesc; const bool Sampled = (pSamplerDesc != nullptr); if (Sampled) { - ze_api_version_t ZeApiVersion = hContext->getPlatform()->ZeApiVersion; - UR_CALL(ur2zeSamplerDesc(ZeApiVersion, pSamplerDesc, ZeSamplerDesc)); + ze_api_version_t ZeApiVersion = + common_cast(hContext)->getPlatform()->ZeApiVersion; + UR_CALL(ur::level_zero::ur2zeSamplerDesc(ZeApiVersion, pSamplerDesc, + ZeSamplerDesc)); BindlessDesc.pNext = &ZeSamplerDesc; BindlessDesc.flags |= ZE_IMAGE_BINDLESS_EXP_FLAG_SAMPLED_IMAGE; } @@ -315,7 +313,7 @@ ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr, ZE_MEMORY_TYPE_UNKNOWN, 0, 0}; - ze_context_handle_t zeCtx = hContext->getZeHandle(); + ze_context_handle_t zeCtx = common_cast(hContext)->getZeHandle(); ZE2UR_CALL(zeMemGetAllocProperties, (zeCtx, reinterpret_cast(hImageMem), @@ -1061,14 +1059,16 @@ bool verifyCommonImagePropertiesSupport( return supported; } -namespace ur::level_zero { - -ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t widthInBytes, - size_t height, size_t elementSizeBytes, - void **ppMem, size_t *pResultPitch) { + ::ur_usm_pool_handle_t pool, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(widthInBytes != 0, UR_RESULT_ERROR_INVALID_USM_SIZE); UR_ASSERT(ppMem && pResultPitch, UR_RESULT_ERROR_INVALID_NULL_POINTER); @@ -1089,16 +1089,20 @@ ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, *pResultPitch = RowPitch; size_t Size = height * RowPitch; - UR_CALL(ur::level_zero::urUSMDeviceAlloc(hContext, hDevice, pUSMDesc, pool, - Size, ppMem)); + auto *ddi = ddiTableOf(hContextOpque); + UR_CALL(ddi->USM.pfnDeviceAlloc(hContextOpque, common_cast(hDevice), pUSMDesc, + pool, Size, ppMem)); return UR_RESULT_SUCCESS; } ur_result_t urBindlessImagesImageAllocateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_exp_image_mem_native_handle_t *phImageMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(pImageFormat && pImageDesc && phImageMem, UR_RESULT_ERROR_INVALID_NULL_POINTER); @@ -1117,10 +1121,12 @@ ur_result_t urBindlessImagesImageAllocateExp( return UR_RESULT_SUCCESS; } -ur_result_t -urBindlessImagesImageFreeExp([[maybe_unused]] ur_context_handle_t hContext, - [[maybe_unused]] ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem) { +ur_result_t urBindlessImagesImageFreeExp( + [[maybe_unused]] ::ur_context_handle_t hContextOpque, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, + ur_exp_image_mem_native_handle_t hImageMem) { + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; ur_bindless_mem_handle_t *urImg = reinterpret_cast(hImageMem); delete urImg; @@ -1128,29 +1134,37 @@ urBindlessImagesImageFreeExp([[maybe_unused]] ur_context_handle_t hContext, } ur_result_t urBindlessImagesUnsampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, ur_exp_image_mem_native_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_exp_image_native_handle_t *phImage) { - UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, - pImageDesc, nullptr, phImage)); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; + UR_CALL(bindlessImagesCreateImpl(hContextOpque, hDevice, hImageMem, + pImageFormat, pImageDesc, nullptr, phImage)); return UR_RESULT_SUCCESS; } ur_result_t urBindlessImagesSampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, ur_exp_image_mem_native_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, const ur_sampler_desc_t *pSamplerDesc, ur_exp_image_native_handle_t *phImage) { - UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, - pImageDesc, pSamplerDesc, phImage)); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; + UR_CALL(bindlessImagesCreateImpl(hContextOpque, hDevice, hImageMem, + pImageFormat, pImageDesc, pSamplerDesc, + phImage)); return UR_RESULT_SUCCESS; } ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, ur_exp_image_native_handle_t hImage) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice && hImage, UR_RESULT_ERROR_INVALID_NULL_HANDLE); std::shared_lock Lock(hDevice->Mutex); @@ -1170,24 +1184,29 @@ ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( } ur_result_t urBindlessImagesSampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, ur_exp_image_native_handle_t hImage) { + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; // Sampled image is a combination of unsampled image and sampler. // The sampler is tied to the image on creation, and is destroyed together // with the image. return ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp( - hContext, hDevice, hImage); + hContextOpque, hDeviceOpque, hImage); } ur_result_t -urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, ur_exp_image_mem_native_handle_t hMem) { - return ur::level_zero::urBindlessImagesImageFreeExp(hContext, hDevice, hMem); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; + return ur::level_zero::urBindlessImagesImageFreeExp(hContextOpque, + hDeviceOpque, hMem); } ur_result_t urBindlessImagesImageGetInfoExp( - ur_context_handle_t, ur_exp_image_mem_native_handle_t hImageMem, + ::ur_context_handle_t, ur_exp_image_mem_native_handle_t hImageMem, ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { UR_ASSERT(hImageMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(UR_IMAGE_INFO_DEPTH >= propName, @@ -1238,7 +1257,7 @@ ur_result_t urBindlessImagesImageGetInfoExp( } ur_result_t urBindlessImagesMipmapGetLevelExp( - ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ::ur_context_handle_t /*hContext*/, ::ur_device_handle_t /*hDevice*/, ur_exp_image_mem_native_handle_t /*hImageMem*/, uint32_t /*mipmapLevel*/, ur_exp_image_mem_native_handle_t * /*phImageMem*/) { UR_LOG_LEGACY(ERR, @@ -1248,10 +1267,13 @@ ur_result_t urBindlessImagesMipmapGetLevelExp( } ur_result_t urBindlessImagesImportExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, - ur_exp_external_mem_type_t memHandleType, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t size, ur_exp_external_mem_type_t memHandleType, ur_exp_external_mem_desc_t *pExternalMemDesc, - ur_exp_external_mem_handle_t *phExternalMem) { + ::ur_exp_external_mem_handle_t *phExternalMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(pExternalMemDesc && phExternalMem, @@ -1340,10 +1362,13 @@ ur_result_t urBindlessImagesImportExternalMemoryExp( } ur_result_t urBindlessImagesMapExternalArrayExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_external_mem_handle_t hExternalMem, + ::ur_exp_external_mem_handle_t hExternalMem, ur_exp_image_mem_native_handle_t *phImageMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice && hExternalMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); @@ -1369,8 +1394,11 @@ ur_result_t urBindlessImagesMapExternalArrayExp( } ur_result_t urBindlessImagesReleaseExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_mem_handle_t hExternalMem) { + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + ::ur_exp_external_mem_handle_t hExternalMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice && hExternalMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); @@ -1397,10 +1425,13 @@ ur_result_t urBindlessImagesReleaseExternalMemoryExp( } ur_result_t urBindlessImagesImportExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, ur_exp_external_semaphore_type_t semHandleType, ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, - ur_exp_external_semaphore_handle_t *phExternalSemaphoreHandle) { + ::ur_exp_external_semaphore_handle_t *phExternalSemaphoreHandle) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; auto UrPlatform = hContext->getPlatform(); if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { @@ -1483,8 +1514,12 @@ ur_result_t urBindlessImagesImportExternalSemaphoreExp( } ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( - ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, - ur_exp_external_semaphore_handle_t hExternalSemaphore) { + ::ur_context_handle_t hContextOpque, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, + ::ur_exp_external_semaphore_handle_t hExternalSemaphore) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; auto UrPlatform = hContext->getPlatform(); if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] "), @@ -1499,8 +1534,12 @@ ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( } ur_result_t urBindlessImagesMapExternalLinearMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, - uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **phRetMem) { + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + uint64_t offset, uint64_t size, ::ur_exp_external_mem_handle_t hExternalMem, + void **phRetMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext && hDevice && hExternalMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(size, UR_RESULT_ERROR_INVALID_BUFFER_SIZE); @@ -1537,8 +1576,11 @@ ur_result_t urBindlessImagesMapExternalLinearMemoryExp( } ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( - ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, - void *pMem) { + ::ur_context_handle_t hContextOpque, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, void *pMem) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(hContext, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(pMem, UR_RESULT_ERROR_INVALID_NULL_POINTER); @@ -1547,9 +1589,12 @@ ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( } ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(std::find(hContext->getDevices().begin(), hContext->getDevices().end(), hDevice) != hContext->getDevices().end(), @@ -1563,9 +1608,12 @@ ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( } ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(std::find(hContext->getDevices().begin(), hContext->getDevices().end(), hDevice) != hContext->getDevices().end(), @@ -1594,9 +1642,12 @@ ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( } ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + auto hContext = common_cast(hContextOpque); + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; UR_ASSERT(std::find(hContext->getDevices().begin(), hContext->getDevices().end(), hDevice) != hContext->getDevices().end(), @@ -1611,8 +1662,10 @@ ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( } ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( - [[maybe_unused]] ur_device_handle_t hDevice, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, ur_exp_external_mem_type_t memHandleType, ur_bool_t *pSupportedRet) { + auto *hDevice = common_cast(hDeviceOpque); + (void)hDevice; #if defined(_WIN32) *pSupportedRet = (memHandleType == UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT) || diff --git a/unified-runtime/source/adapters/level_zero/image_common.hpp b/unified-runtime/source/adapters/level_zero/common/image_common.hpp similarity index 87% rename from unified-runtime/source/adapters/level_zero/image_common.hpp rename to unified-runtime/source/adapters/level_zero/common/image_common.hpp index 9ab37602bbc14..551fd34f3e984 100644 --- a/unified-runtime/source/adapters/level_zero/image_common.hpp +++ b/unified-runtime/source/adapters/level_zero/common/image_common.hpp @@ -1,9 +1,10 @@ //===--------- image_common.hpp - Level Zero Adapter //--------------------===// // +// Copyright (C) 2024-2025 Intel Corporation // -// Part of the LLVM Project, under the Apache License v2.0 with LLVM -// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +14,10 @@ #include #include -#include "v2/common.hpp" +#include "../v2/common.hpp" +#include "device.hpp" + +namespace ur::level_zero { struct ur_bindless_mem_handle_t { // Constructor for bindless image handle @@ -89,3 +93,10 @@ bool verifyCommonImagePropertiesSupport( const ur_device_handle_t hDevice, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType); + +ur_result_t +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hMem); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/common/interfaces.hpp b/unified-runtime/source/adapters/level_zero/common/interfaces.hpp new file mode 100644 index 0000000000000..540e3546cd55a --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/common/interfaces.hpp @@ -0,0 +1,197 @@ +//===--------- interfaces.hpp - Level Zero Adapter -----------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace ur::level_zero { + +struct ur_platform_handle_t_; +typedef struct ur_platform_handle_t_ *ur_platform_handle_t; +struct ur_device_handle_t_; +typedef struct ur_device_handle_t_ *ur_device_handle_t; +struct ur_adapter_handle_t_; +typedef struct ur_adapter_handle_t_ *ur_adapter_handle_t; +struct ur_sampler_handle_t_; +typedef struct ur_sampler_handle_t_ *ur_sampler_handle_t; +struct ur_physical_mem_handle_t_; +typedef struct ur_physical_mem_handle_t_ *ur_physical_mem_handle_t; +struct ur_exp_command_buffer_command_handle_t_; +typedef struct ur_exp_command_buffer_command_handle_t_ + *ur_exp_command_buffer_command_handle_t; +struct ur_program_handle_t_; +typedef struct ur_program_handle_t_ *ur_program_handle_t; + +namespace detail { +// Maps an opaque handle typedef to its corresponding internal struct. +template struct common_handle_traits; +template <> struct common_handle_traits<::ur_platform_handle_t> { + using type = ur_platform_handle_t_; +}; +template <> struct common_handle_traits<::ur_device_handle_t> { + using type = ur_device_handle_t_; +}; +template <> struct common_handle_traits<::ur_adapter_handle_t> { + using type = ur_adapter_handle_t_; +}; +template <> struct common_handle_traits<::ur_sampler_handle_t> { + using type = ur_sampler_handle_t_; +}; +template <> struct common_handle_traits<::ur_physical_mem_handle_t> { + using type = ur_physical_mem_handle_t_; +}; +template <> +struct common_handle_traits<::ur_exp_command_buffer_command_handle_t> { + using type = ur_exp_command_buffer_command_handle_t_; +}; +template <> struct common_handle_traits<::ur_program_handle_t> { + using type = ur_program_handle_t_; +}; + +template +using common_internal_t = typename common_handle_traits::type; + +// Reverse mapping: internal struct -> opaque handle. +template struct opaque_handle_for; +template <> struct opaque_handle_for { + using type = ::ur_platform_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_device_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_adapter_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_sampler_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_physical_mem_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_exp_command_buffer_command_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_program_handle_t; +}; +} // namespace detail + +// Opaque handle -> internal pointer. +template +inline detail::common_internal_t *common_cast(Opaque h) { + return reinterpret_cast *>(h); +} + +// Opaque handle array -> internal pointer array. +template +inline detail::common_internal_t **common_cast(Opaque *ph) { + return reinterpret_cast **>(ph); +} + +// const Opaque handle array -> const internal pointer array. +template +inline detail::common_internal_t *const *common_cast(const Opaque *ph) { + return reinterpret_cast *const *>(ph); +} + +// Internal pointer -> opaque handle (reverse direction). +template +inline typename detail::opaque_handle_for::type +common_cast(Internal *p) { + return reinterpret_cast::type>( + p); +} + +// Internal pointer array -> opaque handle array. +template +inline typename detail::opaque_handle_for::type * +common_cast(Internal **p) { + return reinterpret_cast::type *>( + p); +} + +// Element-wise cast of a vector of internal pointers to opaque handles. +template >::type>> +inline std::vector< + typename detail::opaque_handle_for>::type> +common_cast(const std::vector &v) { + using Opaque = + typename detail::opaque_handle_for>::type; + std::vector out; + out.reserve(v.size()); + for (auto p : v) + out.push_back(common_cast(p)); + return out; +} + +// Head layout shared by every UR handle. +struct handle_head_t { + const ur_dditable_t *ddi_table; + ur_shared_mutex Mutex; +}; + +template inline const ur_dditable_t *ddiTableOf(H handle) { + return reinterpret_cast(handle)->ddi_table; +} + +// getddi policy for handles shared across L0v1 and L0v2, whose `ddi_table` is +// origin-dependent. +struct null_ddi { + static const ur_dditable_t *value() { return nullptr; } +}; + +// Primary base for the common concrete handle types. Uses `null_ddi` because +// the table is filled at construction. +struct ur_object_t : ur::handle_base { + ur_shared_mutex Mutex; + bool OwnNativeHandle = false; +}; + +// Version-agnostic context base, shared by the L0v1 and L0v2 concrete contexts. +struct ur_context_common_t : ur_object_t { + ze_context_handle_t ZeContext = nullptr; + std::vector hDevices; + ur_platform_handle_t Platform = nullptr; // == hDevices[0]->Platform + ur_shared_mutex *MutexPtr = nullptr; // == &ur_object_t::Mutex + + ze_context_handle_t getZeHandle() const { return ZeContext; } + ur_platform_handle_t getPlatform() const { return Platform; } + const std::vector &getDevices() const { return hDevices; } + ur_shared_mutex &getMutex() const { return *MutexPtr; } + + bool isValidDevice(ur_device_handle_t hDevice) const; + +protected: + ur_context_common_t() = default; + ur_context_common_t(ze_context_handle_t ZeContext, + std::vector Devices, + ur_platform_handle_t Platform) + : ZeContext(ZeContext), hDevices(std::move(Devices)), Platform(Platform) { + MutexPtr = &Mutex; + if (!hDevices.empty()) + ddi_table = ddiTableOf(hDevices[0]); + } + + ur_context_common_t(const ur_context_common_t &) = delete; + ur_context_common_t &operator=(const ur_context_common_t &) = delete; +}; + +// Bridges the opaque context handle to the shared context base. +inline ur_context_common_t *common_cast(::ur_context_handle_t h) { + return reinterpret_cast(h); +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/memory_export.cpp b/unified-runtime/source/adapters/level_zero/common/memory_export.cpp similarity index 85% rename from unified-runtime/source/adapters/level_zero/memory_export.cpp rename to unified-runtime/source/adapters/level_zero/common/memory_export.cpp index 3f8f1b64a93ec..608585680d0f1 100644 --- a/unified-runtime/source/adapters/level_zero/memory_export.cpp +++ b/unified-runtime/source/adapters/level_zero/common/memory_export.cpp @@ -7,19 +7,18 @@ // //===----------------------------------------------------------------------===// -#include "common.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else -#include "context.hpp" -#endif +#include "device.hpp" +#include "interfaces.hpp" #include "unified-runtime/ur_api.h" namespace ur::level_zero { ur_result_t urMemoryExportAllocExportableMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, - size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem) { + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem) { + auto hContext = common_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); UR_ASSERT(handleTypeToExport == UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD || handleTypeToExport == UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT || @@ -54,16 +53,19 @@ ur_result_t urMemoryExportAllocExportableMemoryExp( } ur_result_t urMemoryExportFreeExportableMemoryExp( - ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, - void *pMem) { + ::ur_context_handle_t hContextOpque, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, void *pMem) { + auto hContext = common_cast(hContextOpque); ZE2UR_CALL(zeMemFree, (hContext->getZeHandle(), pMem)); return UR_RESULT_SUCCESS; } ur_result_t urMemoryExportExportMemoryHandleExp( - ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, + [[maybe_unused]] ::ur_device_handle_t hDeviceOpque, ur_exp_external_mem_type_t handleTypeToExport, void *pMem, void *pMemHandleRet) { + auto hContext = common_cast(hContextOpque); ze_memory_allocation_properties_t MemAllocProps{}; MemAllocProps.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/common/physical_mem.hpp similarity index 69% rename from unified-runtime/source/adapters/level_zero/physical_mem.hpp rename to unified-runtime/source/adapters/level_zero/common/physical_mem.hpp index 10938d1e7e282..2b6f9aad27da4 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/common/physical_mem.hpp @@ -1,23 +1,30 @@ //===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// // +// Copyright (C) 2023 Intel Corporation // -// Part of the LLVM Project, under the Apache License v2.0 with LLVM -// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +#include "api.hpp" #include "common/ur_ref_count.hpp" +#include "interfaces.hpp" -struct ur_physical_mem_handle_t_ : ur_object { +namespace ur::level_zero { + +struct ur_physical_mem_handle_t_ : ur::level_zero::ur_object_t { ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, ur_context_handle_t Context, ur_device_handle_t Device, size_t Size, bool EnableIpc) : ZePhysicalMem{ZePhysicalMem}, Context{Context}, Device{Device}, - Size{Size}, EnableIpc{EnableIpc} {} + Size{Size}, EnableIpc{EnableIpc} { + // Populate ddi_table from the owning context. + ddi_table = ddiTableOf(Context); + } // Level Zero physical memory handle. ze_physical_mem_handle_t ZePhysicalMem; @@ -36,3 +43,5 @@ struct ur_physical_mem_handle_t_ : ur_object { ur::RefCount RefCount; }; + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/platform.cpp b/unified-runtime/source/adapters/level_zero/common/platform.cpp similarity index 97% rename from unified-runtime/source/adapters/level_zero/platform.cpp rename to unified-runtime/source/adapters/level_zero/common/platform.cpp index fc495e0dfe1cd..65801aac69812 100644 --- a/unified-runtime/source/adapters/level_zero/platform.cpp +++ b/unified-runtime/source/adapters/level_zero/common/platform.cpp @@ -9,7 +9,6 @@ #include "platform.hpp" #include "adapter.hpp" -#include "ur_level_zero.hpp" // The default value of the driver versions skiplist // (SYCL_UR_L0_DRIVER_SKIPLIST) @@ -18,7 +17,7 @@ static const std::vector DriverSkiplistDefault = {}; namespace ur::level_zero { ur_result_t urPlatformGet( - ur_adapter_handle_t, + ::ur_adapter_handle_t hAdapterOpque, /// [in] the number of platforms to be added to phPlatforms. If phPlatforms /// is not NULL, then NumEntries should be greater than zero, otherwise /// ::UR_RESULT_ERROR_INVALID_SIZE, will be returned. @@ -26,18 +25,19 @@ ur_result_t urPlatformGet( /// [out][optional][range(0, NumEntries)] array of handle of platforms. /// If NumEntries is less than the number of platforms available, then /// ::urPlatformGet shall only retrieve that number of platforms. - ur_platform_handle_t *Platforms, + ::ur_platform_handle_t *Platforms, /// [out][optional] returns the total number of platforms available. uint32_t *NumPlatforms) { + auto hAdapter = common_cast(hAdapterOpque); // Platform handles are cached for reuse. This is to ensure consistent // handle pointers across invocations and to improve retrieval performance. - uint32_t nplatforms = (uint32_t)GlobalAdapter->Platforms.size(); + uint32_t nplatforms = (uint32_t)hAdapter->Platforms.size(); if (NumPlatforms) { *NumPlatforms = nplatforms; } if (Platforms) { for (uint32_t i = 0; i < std::min(nplatforms, NumEntries); ++i) { - Platforms[i] = GlobalAdapter->Platforms.at(i).get(); + Platforms[i] = common_cast(hAdapter->Platforms.at(i).get()); } } @@ -46,7 +46,7 @@ ur_result_t urPlatformGet( ur_result_t urPlatformGetInfo( /// [in] handle of the platform - ur_platform_handle_t Platform, + ::ur_platform_handle_t PlatformOpque, /// [in] type of the info to retrieve ur_platform_info_t ParamName, /// [in] the number of bytes pointed to by pPlatformInfo. @@ -64,11 +64,7 @@ ur_result_t urPlatformGetInfo( switch (ParamName) { case UR_PLATFORM_INFO_NAME: // TODO: Query Level Zero driver when relevant info is added there. -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - return ReturnValue("Intel(R) oneAPI Unified Runtime over Level-Zero V2"); -#else return ReturnValue("Intel(R) oneAPI Unified Runtime over Level-Zero"); -#endif case UR_PLATFORM_INFO_VENDOR_NAME: // TODO: Query Level Zero driver when relevant info is added there. return ReturnValue("Intel(R) Corporation"); @@ -94,11 +90,13 @@ ur_result_t urPlatformGetInfo( // OpenCL. Follow the same notation here. // - return ReturnValue(Platform->ZeDriverApiVersion.c_str()); + return ReturnValue(common_cast(PlatformOpque)->ZeDriverApiVersion.c_str()); case UR_PLATFORM_INFO_BACKEND: return ReturnValue(UR_BACKEND_LEVEL_ZERO); case UR_PLATFORM_INFO_ADAPTER: - return ReturnValue(GlobalAdapter); + // Whichever adapter (L0v1 or L0v2) won selection in urAdapterGet is + // also the one that stamped this platform's DDI table. + return ReturnValue(common_cast(GlobalAdapter)); default: UR_LOG(DEBUG, "urPlatformGetInfo: unrecognized ParamName"); return UR_RESULT_ERROR_INVALID_VALUE; @@ -109,7 +107,7 @@ ur_result_t urPlatformGetInfo( ur_result_t urPlatformGetApiVersion( /// [in] handle of the platform - ur_platform_handle_t /*Driver*/, + ::ur_platform_handle_t /*Driver*/, /// [out] api version ur_api_version_t *Version) { *Version = UR_API_VERSION_CURRENT; @@ -118,31 +116,31 @@ ur_result_t urPlatformGetApiVersion( ur_result_t urPlatformGetNativeHandle( /// [in] handle of the platform. - ur_platform_handle_t Platform, + ::ur_platform_handle_t PlatformOpque, /// [out] a pointer to the native handle of the platform. - ur_native_handle_t *NativePlatform) { + ::ur_native_handle_t *NativePlatform) { // Extract the Level Zero driver handle from the given PI platform - *NativePlatform = reinterpret_cast(Platform->ZeDriver); + *NativePlatform = reinterpret_cast( + common_cast(PlatformOpque)->ZeDriver); return UR_RESULT_SUCCESS; } ur_result_t urPlatformCreateWithNativeHandle( /// [in] the native handle of the platform. - ur_native_handle_t NativePlatform, ur_adapter_handle_t, + ::ur_native_handle_t NativePlatform, ::ur_adapter_handle_t hAdapterOpque, /// [in][optional] pointer to native platform properties struct. const ur_platform_native_properties_t * /*Properties*/, /// [out] pointer to the handle of the platform object created. - ur_platform_handle_t *Platform) { + ::ur_platform_handle_t *Platform) { auto ZeDriver = ur_cast(NativePlatform); uint32_t NumPlatforms = 0; - ur_adapter_handle_t AdapterHandle = GlobalAdapter; UR_CALL( - ur::level_zero::urPlatformGet(AdapterHandle, 0, nullptr, &NumPlatforms)); + ur::level_zero::urPlatformGet(hAdapterOpque, 0, nullptr, &NumPlatforms)); if (NumPlatforms) { - std::vector Platforms(NumPlatforms); - UR_CALL(ur::level_zero::urPlatformGet(AdapterHandle, NumPlatforms, + std::vector<::ur_platform_handle_t> Platforms(NumPlatforms); + UR_CALL(ur::level_zero::urPlatformGet(hAdapterOpque, NumPlatforms, Platforms.data(), nullptr)); // The SYCL spec requires that the set of platforms must remain fixed for @@ -150,9 +148,9 @@ ur_result_t urPlatformCreateWithNativeHandle( // of the Level Zero drivers when we initialized the platform cache, so the // "NativeHandle" must already be in the cache. If it is not, this must not // be a valid Level Zero driver. - for (const ur_platform_handle_t &CachedPlatform : Platforms) { - if (CachedPlatform->ZeDriver == ZeDriver) { - *Platform = CachedPlatform; + for (const ::ur_platform_handle_t &CachedPlatformGlobal : Platforms) { + if (common_cast(CachedPlatformGlobal)->ZeDriver == ZeDriver) { + *Platform = CachedPlatformGlobal; return UR_RESULT_SUCCESS; } } @@ -169,7 +167,7 @@ ur_result_t urPlatformCreateWithNativeHandle( // frontend_option=-ftarget-compile-fast. ur_result_t urPlatformGetBackendOption( /// [in] handle of the platform instance. - ur_platform_handle_t /*Platform*/, + ::ur_platform_handle_t /*Platform*/, /// [in] string containing the frontend option. const char *FrontendOption, /// [out] returns the correct platform specific compiler option based on @@ -204,8 +202,6 @@ ur_result_t urPlatformGetBackendOption( return UR_RESULT_ERROR_INVALID_VALUE; } -} // namespace ur::level_zero - ur_result_t ur_platform_handle_t_::initialize() { ZE2UR_CALL(zeDriverGetApiVersion, (ZeDriver, &ZeApiVersion)); ZeDriverApiVersion = std::to_string(ZE_MAJOR_VERSION(ZeApiVersion)) + "." + @@ -1138,3 +1134,5 @@ ur_device_handle_t ur_platform_handle_t_::getDeviceById(DeviceId id) { } return nullptr; } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/platform.hpp b/unified-runtime/source/adapters/level_zero/common/platform.hpp similarity index 97% rename from unified-runtime/source/adapters/level_zero/platform.hpp rename to unified-runtime/source/adapters/level_zero/common/platform.hpp index 488f1cc66aae8..ac20cfc7ce1bf 100644 --- a/unified-runtime/source/adapters/level_zero/platform.hpp +++ b/unified-runtime/source/adapters/level_zero/common/platform.hpp @@ -1,4 +1,4 @@ -//===--------- platform.hpp - Level Zero Adapter --------------------------===// +//===--------- platform.hpp - Level Zero Adapter -------------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -8,17 +8,14 @@ //===----------------------------------------------------------------------===// #pragma once -#include - -#include "common.hpp" +#include "device.hpp" +#include "interfaces.hpp" #include "unified-runtime/ur_api.h" #include "ze_api.h" #include "ze_ddi.h" #include "zes_api.h" -struct ur_device_handle_t_; - -typedef size_t DeviceId; +namespace ur::level_zero { struct ur_zes_device_handle_data_t { zes_device_handle_t ZesDevice; @@ -26,10 +23,12 @@ struct ur_zes_device_handle_data_t { ze_bool_t SubDevice = false; }; -struct ur_platform_handle_t_ : ur::handle_base, - public ur_platform { +struct ur_platform_handle_t_; +typedef struct ur_platform_handle_t_ *ur_platform_handle_t; + +struct ur_platform_handle_t_ : ur::level_zero::ur_object_t, public ur_platform { ur_platform_handle_t_(ze_driver_handle_t Driver) - : handle_base(), ZeDriver{Driver}, ZeApiVersion{ZE_API_VERSION_CURRENT} {} + : ZeDriver{Driver}, ZeApiVersion{ZE_API_VERSION_CURRENT} {} // Performs initialization of a newly constructed PI platform. ur_result_t initialize(); @@ -286,3 +285,5 @@ struct ur_platform_handle_t_ : ur::handle_base, // Windows. For details, see https://github.com/intel/llvm/issues/20927. bool ZeDeviceSynchronizeSupported{false}; }; + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/common/program.cpp similarity index 88% rename from unified-runtime/source/adapters/level_zero/program.cpp rename to unified-runtime/source/adapters/level_zero/common/program.cpp index e9fa8fa190af2..648b9525d86fb 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/common/program.cpp @@ -7,18 +7,16 @@ // //===----------------------------------------------------------------------===// -#include "program.hpp" +#include "../program.hpp" +#include "api.hpp" #include "device.hpp" #include "logger/ur_logger.hpp" -#include "ur_interface_loader.hpp" +#include "platform.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else -#include "context.hpp" -#endif +#include "interfaces.hpp" -extern "C" { +namespace ur::level_zero { +namespace { // Check to see if a Level Zero module has any unresolved symbols. // // @param ZeModule The module handle to check. @@ -38,7 +36,7 @@ checkUnresolvedSymbols(ze_module_handle_t ZeModule, // no imported symbols, it's not possible to have any unresolved symbols. We // do this check first because we assume it's faster than the call to // zeModuleDynamicLink below. - ZeStruct ZeModuleProps; + ur::level_zero::ZeStruct ZeModuleProps; ze_result_t ZeResult = ZE_CALL_NOCHECK(zeModuleGetProperties, (ZeModule, &ZeModuleProps)); if (ZeResult != ZE_RESULT_SUCCESS) @@ -52,7 +50,7 @@ checkUnresolvedSymbols(ze_module_handle_t ZeModule, } return ZE_RESULT_SUCCESS; } -} // extern "C" +} // namespace static ur_program_handle_t_::CodeFormat matchILCodeFormat(const void *Input, size_t Length) { @@ -74,11 +72,9 @@ static bool isCodeFormatIL(ur_program_handle_t_::CodeFormat CodeFormat) { return CodeFormat == ur_program_handle_t_::CodeFormat::SPIRV; } -namespace ur::level_zero { - ur_result_t urProgramCreateWithIL( /// [in] handle of the context instance - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] pointer to IL binary. const void *IL, /// [in] length of `pIL` in bytes. @@ -86,16 +82,16 @@ ur_result_t urProgramCreateWithIL( /// [in][optional] pointer to program creation properties. const ur_program_properties_t * /*Properties*/, /// [out] pointer to handle of program object created. - ur_program_handle_t *Program) { - UR_ASSERT(Context, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + ::ur_program_handle_t *Program) { + UR_ASSERT(ContextOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(IL && Program, UR_RESULT_ERROR_INVALID_NULL_POINTER); const ur_program_handle_t_::CodeFormat CodeFormat = matchILCodeFormat(IL, Length); UR_ASSERT(isCodeFormatIL(CodeFormat), UR_RESULT_ERROR_INVALID_BINARY); try { ur_program_handle_t_ *UrProgram = new ur_program_handle_t_( - ur_program_handle_t_::IL, Context, IL, Length, CodeFormat); - *Program = reinterpret_cast(UrProgram); + ur_program_handle_t_::IL, ContextOpque, IL, Length, CodeFormat); + *Program = common_cast(UrProgram); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -107,12 +103,12 @@ ur_result_t urProgramCreateWithIL( ur_result_t urProgramCreateWithBinary( /// [in] handle of the context instance - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] number of devices uint32_t numDevices, /// [in][range(0, numDevices)] a pointer to a list of device handles. /// The binaries are loaded for devices specified in this list. - ur_device_handle_t *phDevices, + ::ur_device_handle_t *phDevicesOpque, /// [in][range(0, numDevices)] array of sizes of program binaries specified /// by `pBinaries` (in bytes). size_t *pLengths, @@ -122,7 +118,9 @@ ur_result_t urProgramCreateWithBinary( /// [in][optional] pointer to program creation properties. const ur_program_properties_t *pProperties, /// [out] pointer to handle of Program object created. - ur_program_handle_t *phProgram) { + ::ur_program_handle_t *phProgram) { + auto hContext = common_cast(hContextOpque); + auto phDevices = common_cast(phDevicesOpque); // In OpenCL, clCreateProgramWithBinary() can be used to load any of the // following: "program executable", "compiled program", or "library of // compiled programs". In addition, the loaded program can be either @@ -140,9 +138,9 @@ ur_result_t urProgramCreateWithBinary( UR_RESULT_ERROR_INVALID_DEVICE); } ur_program_handle_t_ *UrProgram = new ur_program_handle_t_( - ur_program_handle_t_::Native, hContext, numDevices, phDevices, + ur_program_handle_t_::Native, hContextOpque, numDevices, phDevices, pProperties, ppBinaries, pLengths); - *phProgram = reinterpret_cast(UrProgram); + *phProgram = common_cast(UrProgram); return UR_RESULT_SUCCESS; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; @@ -153,32 +151,35 @@ ur_result_t urProgramCreateWithBinary( ur_result_t urProgramBuild( /// [in] handle of the context instance. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] Handle of the program to build. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in][optional] pointer to build options null-terminated string. const char *Options) { + auto Context = common_cast(ContextOpque); std::vector Devices = Context->getDevices(); - return ur::level_zero::urProgramBuildExp(Program, Devices.size(), - Devices.data(), + return ur::level_zero::urProgramBuildExp(ProgramOpque, Devices.size(), + common_cast(Devices.data()), ur_exp_program_flags_t{}, Options); } ur_result_t urProgramBuildExp( /// [in] Handle of the program to build. - ur_program_handle_t hProgram, + ::ur_program_handle_t hProgramOpque, /// [in] number of devices uint32_t numDevices, /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, + ::ur_device_handle_t *phDevicesOpque, /// [in] program information flags ur_exp_program_flags_t flags, /// [in][optional] pointer to build options null-terminated string. const char *pOptions) { + auto hProgram = common_cast(hProgramOpque); + auto phDevices = common_cast(phDevicesOpque); // TODO // Check if device belongs to associated context. // UR_ASSERT(Program->Context, UR_RESULT_ERROR_INVALID_PROGRAM); - // UR_ASSERT(Program->Context->isValidDevice(Devices[0]), + // UR_ASSERT(Program->common_cast(Context)->isValidDevice(Devices[0]), // UR_RESULT_ERROR_INVALID_VALUE); std::scoped_lock Guard(hProgram->Mutex); @@ -233,7 +234,8 @@ ur_result_t urProgramBuildExp( } ZeModuleDesc.inputSize = hProgram->getCodeSize(ZeDevice); ZeModuleDesc.pInputModule = Code; - ze_context_handle_t ZeContext = hProgram->Context->getZeHandle(); + auto Context = common_cast(hProgram->Context); + ze_context_handle_t ZeContext = Context->getZeHandle(); ze_module_handle_t ZeModuleHandle = nullptr; ze_module_build_log_handle_t ZeBuildLog{}; @@ -278,15 +280,17 @@ ur_result_t urProgramBuildExp( ur_result_t urProgramCompileExp( /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, + ::ur_program_handle_t hProgramOpque, /// [in] number of devices uint32_t numDevices, /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, + ::ur_device_handle_t *phDevicesOpque, /// [in] program information flags [[maybe_unused]] ur_exp_program_flags_t flags, /// [in][optional] pointer to build options null-terminated string. const char *pOptions) { + auto hProgram = common_cast(hProgramOpque); + auto phDevices = common_cast(phDevicesOpque); std::scoped_lock Guard(hProgram->Mutex); // Check that state is IL for all devices in the context and set the state to // Object. @@ -323,51 +327,56 @@ ur_result_t urProgramCompileExp( ur_result_t urProgramCompile( /// [in] handle of the context instance. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in][out] handle of the program to compile. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in][optional] pointer to build options null-terminated string. const char *Options) { - auto devices = Context->getDevices(); - return ur::level_zero::urProgramCompileExp(Program, devices.size(), - devices.data(), + auto Context = common_cast(ContextOpque); + auto Devices = common_cast(Context->getDevices()); + return ur::level_zero::urProgramCompileExp(ProgramOpque, Devices.size(), + Devices.data(), ur_exp_program_flags_t{}, Options); } ur_result_t urProgramLink( /// [in] handle of the context instance. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] number of program handles in `phPrograms`. uint32_t Count, /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *Programs, + const ::ur_program_handle_t *ProgramsOpque, /// [in][optional] pointer to linker options null-terminated string. const char *Options, /// [out] pointer to handle of program object created. - ur_program_handle_t *Program) { - std::vector Devices = Context->getDevices(); + ::ur_program_handle_t *Program) { + auto Context = common_cast(ContextOpque); + auto Devices = common_cast(Context->getDevices()); return ur::level_zero::urProgramLinkExp( - Context, Devices.size(), Devices.data(), ur_exp_program_flags_t{}, Count, - Programs, Options, Program); + ContextOpque, Devices.size(), Devices.data(), ur_exp_program_flags_t{}, + Count, ProgramsOpque, Options, Program); } ur_result_t urProgramLinkExp( /// [in] handle of the context instance. - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] number of devices uint32_t numDevices, /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, + ::ur_device_handle_t *phDevicesOpque, /// [in] program information flags ur_exp_program_flags_t flags, /// [in] number of program handles in `phPrograms`. uint32_t count, /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, + const ::ur_program_handle_t *phProgramsOpque, /// [in][optional] pointer to linker options null-terminated string. const char *pOptions, /// [out] pointer to handle of program object created. - ur_program_handle_t *phProgram) { + ::ur_program_handle_t *phProgram) { + auto hContext = common_cast(hContextOpque); + auto phDevices = common_cast(phDevicesOpque); + auto phPrograms = common_cast(phProgramsOpque); if (nullptr != phProgram) { *phProgram = nullptr; } @@ -384,8 +393,8 @@ ur_result_t urProgramLinkExp( ErrorMessage.append(pOptions); ErrorMessage.push_back('\"'); ur_program_handle_t_ *UrProgram = new ur_program_handle_t_( - ur_program_handle_t_::Invalid, hContext, ErrorMessage); - *phProgram = reinterpret_cast(UrProgram); + ur_program_handle_t_::Invalid, hContextOpque, ErrorMessage); + *phProgram = common_cast(UrProgram); return UR_RESULT_ERROR_PROGRAM_LINK_FAILURE; } ur_result_t UrResult = UR_RESULT_SUCCESS; @@ -455,11 +464,10 @@ ur_result_t urProgramLinkExp( SpecConstShims.reserve(count); for (uint32_t I = 0; I < count; I++) { - ur_program_handle_t Program = phPrograms[I]; - CodeSizes[I] = Program->getCodeSize(); - CodeBufs[I] = Program->getCode(); + CodeSizes[I] = phPrograms[I]->getCodeSize(); + CodeBufs[I] = phPrograms[I]->getCode(); if (!NativeInput) { - SpecConstShims.emplace_back(Program); + SpecConstShims.emplace_back(phPrograms[I]); SpecConstPtrs[I] = SpecConstShims[I].ze(); } } @@ -523,8 +531,8 @@ ur_result_t urProgramLinkExp( } } - ur_program_handle_t_ *UrProgram = new ur_program_handle_t_(hContext); - *phProgram = reinterpret_cast(UrProgram); + ur_program_handle_t_ *UrProgram = new ur_program_handle_t_(hContextOpque); + *phProgram = common_cast(UrProgram); for (uint32_t i = 0; i < numDevices; i++) { // Call the Level Zero API to compile, link, and create the module. ze_device_handle_t ZeDevice = phDevices[i]->ZeDevice; @@ -595,11 +603,13 @@ ur_result_t urProgramLinkExp( ur_result_t urProgramDynamicLinkExp( /// [in] handle of the context instance. - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] number of program handles in `phPrograms`. uint32_t count, /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) { + const ::ur_program_handle_t *phProgramsOpque) { + auto hContext = common_cast(hContextOpque); + auto phPrograms = common_cast(phProgramsOpque); ur_result_t UrResult = UR_RESULT_SUCCESS; try { @@ -640,16 +650,17 @@ ur_result_t urProgramDynamicLinkExp( ur_result_t urProgramRetain( /// [in] handle for the Program to retain - ur_program_handle_t Program) { - Program->RefCount.retain(); + ::ur_program_handle_t ProgramOpque) { + common_cast(ProgramOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urProgramRelease( /// [in] handle for the Program to release - ur_program_handle_t Program) { - if (!Program) + ::ur_program_handle_t ProgramOpque) { + if (!ProgramOpque) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + auto Program = common_cast(ProgramOpque); // Detect double-release by attempting to safely access program members // and catching any access violations that indicate freed memory @@ -717,16 +728,18 @@ static bool is_in_separated_string(const std::string &str, char delimiter, ur_result_t urProgramGetFunctionPointer( /// [in] handle of the device to retrieve pointer for. - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in] handle of the program to search for function in. The program /// must already be built to the specified device, or otherwise /// ::UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE is returned. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] A null-terminates string denoting the mangled function name. const char *FunctionName, /// [out] Returns the pointer to the function if it is found in the /// program. void **FunctionPointerRet) { + auto Device = common_cast(DeviceOpque); + auto Program = common_cast(ProgramOpque); std::shared_lock Guard(Program->Mutex); if (Program->getState(Device->ZeDevice) != ur_program_handle_t_::Exe) { return UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE; @@ -749,12 +762,12 @@ ur_result_t urProgramGetFunctionPointer( size_t Size; *FunctionPointerRet = 0; UR_CALL(ur::level_zero::urProgramGetInfo( - Program, UR_PROGRAM_INFO_KERNEL_NAMES, 0, nullptr, &Size)); + ProgramOpque, UR_PROGRAM_INFO_KERNEL_NAMES, 0, nullptr, &Size)); std::string ClResult(Size, ' '); UR_CALL(ur::level_zero::urProgramGetInfo( - Program, UR_PROGRAM_INFO_KERNEL_NAMES, ClResult.size(), &ClResult[0], - nullptr)); + ProgramOpque, UR_PROGRAM_INFO_KERNEL_NAMES, ClResult.size(), + &ClResult[0], nullptr)); // Get rid of the null terminator and search for kernel_name // If function can be found return error code to indicate it @@ -776,9 +789,9 @@ ur_result_t urProgramGetFunctionPointer( ur_result_t urProgramGetGlobalVariablePointer( /// [in] handle of the device to retrieve the pointer for. - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in] handle of the program where the global variable is. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] mangled name of the global variable to retrieve the pointer for. const char *GlobalVariableName, /// [out][optional] Returns the size of the global variable if it is found @@ -787,6 +800,8 @@ ur_result_t urProgramGetGlobalVariablePointer( /// [out] Returns the pointer to the global variable if it is found in the /// program. void **GlobalVariablePointerRet) { + auto Device = common_cast(DeviceOpque); + auto Program = common_cast(ProgramOpque); std::scoped_lock lock(Program->Mutex); if (Program->getState(Device->ZeDevice) != ur_program_handle_t_::Exe) { return UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE; @@ -808,7 +823,7 @@ ur_result_t urProgramGetGlobalVariablePointer( ur_result_t urProgramGetInfo( /// [in] handle of the Program object - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] name of the Program property to query ur_program_info_t PropName, /// [in] the size of the Program property. @@ -822,6 +837,7 @@ ur_result_t urProgramGetInfo( /// [out][optional] pointer to the actual size in bytes of data copied to /// propName. size_t *PropSizeRet) { + auto Program = common_cast(ProgramOpque); UrReturnHelper ReturnValue(PropSize, ProgramInfo, PropSizeRet); switch (PropName) { @@ -963,9 +979,9 @@ ur_result_t urProgramGetInfo( ur_result_t urProgramGetBuildInfo( /// [in] handle of the Program object - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] handle of the Device object - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in] name of the Program build info to query ur_program_build_info_t PropName, /// [in] size of the Program build info property. @@ -978,7 +994,8 @@ ur_result_t urProgramGetBuildInfo( /// [out][optional] pointer to the actual size in bytes of data being /// queried by propName. size_t *PropSizeRet) { - + auto Device = common_cast(DeviceOpque); + auto Program = common_cast(ProgramOpque); std::shared_lock Guard(Program->Mutex); UrReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet); if (PropName == UR_PROGRAM_BUILD_INFO_BINARY_TYPE) { @@ -1045,7 +1062,7 @@ ur_result_t urProgramGetBuildInfo( ur_result_t urProgramSetSpecializationConstant( /// [in] handle of the Program object - ur_program_handle_t /*Program*/, + ::ur_program_handle_t /*Program*/, /// [in] specification constant Id uint32_t /*SpecId*/, /// [in] size of the specialization constant value @@ -1060,10 +1077,10 @@ ur_result_t urProgramSetSpecializationConstant( ur_result_t urProgramGetNativeHandle( /// [in] handle of the program. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [out] a pointer to the native handle of the program. - ur_native_handle_t *NativeProgram) { - auto ZeModule = ur_cast(NativeProgram); + ::ur_native_handle_t *NativeProgram) { + auto Program = common_cast(ProgramOpque); std::shared_lock Guard(Program->Mutex); assert(Program->AssociatedDevices.size() > 0); @@ -1080,20 +1097,20 @@ ur_result_t urProgramGetNativeHandle( // L0 only supprts returning native handle from built programs. return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - *ZeModule = Module; + *ur_cast(NativeProgram) = Module; return UR_RESULT_SUCCESS; } ur_result_t urProgramCreateWithNativeHandle( /// [in] the native handle of the program. - ur_native_handle_t NativeProgram, + ::ur_native_handle_t NativeProgram, /// [in] handle of the context instance - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in][optional] pointer to native program properties struct. const ur_program_native_properties_t *Properties, /// [out] pointer to the handle of the program object created. - ur_program_handle_t *Program) { - UR_ASSERT(Context && NativeProgram, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + ::ur_program_handle_t *Program) { + UR_ASSERT(ContextOpque && NativeProgram, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(Program, UR_RESULT_ERROR_INVALID_NULL_POINTER); auto ZeModule = ur_cast(NativeProgram); // We assume here that programs created from a native handle always @@ -1102,9 +1119,9 @@ ur_result_t urProgramCreateWithNativeHandle( try { ur_program_handle_t_ *UrProgram = new ur_program_handle_t_( - ur_program_handle_t_::Exe, Context, ZeModule, + ur_program_handle_t_::Exe, ContextOpque, ZeModule, Properties ? Properties->isNativeHandleOwned : false); - *Program = reinterpret_cast(UrProgram); + *Program = common_cast(UrProgram); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -1115,12 +1132,13 @@ ur_result_t urProgramCreateWithNativeHandle( ur_result_t urProgramSetSpecializationConstants( /// [in] handle of the Program object - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] the number of elements in the pSpecConstants array uint32_t Count, /// [in][range(0, count)] array of specialization constant value /// descriptions const ur_specialization_constant_info_t *SpecConstants) { + auto Program = common_cast(ProgramOpque); std::scoped_lock Guard(Program->Mutex); // Remember the value of this specialization constant until the program is @@ -1137,19 +1155,19 @@ ur_result_t urProgramSetSpecializationConstants( return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero - ur_program_handle_t_::ur_program_handle_t_(state St, ur_context_handle_t Context, const void *Input, size_t Length, CodeFormat CodeFormat) : Context{Context}, NativeProperties{nullptr}, OwnZeModule{true}, - AssociatedDevices(Context->getDevices()), ILCode{new uint8_t[Length]}, - ILCodeLength{Length}, ILCodeFormat(CodeFormat) { + AssociatedDevices(common_cast(Context)->getDevices()), + ILCode{new uint8_t[Length]}, ILCodeLength{Length}, + ILCodeFormat(CodeFormat) { + ddi_table = ddiTableOf(Context); assert(isCodeFormatIL(CodeFormat)); std::memcpy(ILCode.get(), Input, Length); // All devices have the program in IL state. - for (auto &Device : Context->getDevices()) { + for (auto &Device : common_cast(Context)->getDevices()) { DeviceData &PerDevData = DeviceDataMap[Device->ZeDevice]; PerDevData.State = St; } @@ -1157,11 +1175,12 @@ ur_program_handle_t_::ur_program_handle_t_(state St, ur_program_handle_t_::ur_program_handle_t_( state St, ur_context_handle_t Context, const uint32_t NumDevices, - const ur_device_handle_t *Devices, + const ur::level_zero::ur_device_handle_t *Devices, const ur_program_properties_t *Properties, const uint8_t **Inputs, const size_t *Lengths) : Context{Context}, NativeProperties(Properties), OwnZeModule{true}, AssociatedDevices(Devices, Devices + NumDevices) { + ddi_table = ddiTableOf(Context); for (uint32_t I = 0; I < NumDevices; ++I) { DeviceData &PerDevData = DeviceDataMap[Devices[I]->ZeDevice]; PerDevData.State = St; @@ -1173,20 +1192,25 @@ ur_program_handle_t_::ur_program_handle_t_( ur_program_handle_t_::ur_program_handle_t_(ur_context_handle_t Context) : Context{Context}, NativeProperties{nullptr}, OwnZeModule{true}, - AssociatedDevices(Context->getDevices()) {} + AssociatedDevices(common_cast(Context)->getDevices()) { + ddi_table = ddiTableOf(Context); +} ur_program_handle_t_::ur_program_handle_t_(state, ur_context_handle_t Context, ze_module_handle_t InteropZeModule) : Context{Context}, NativeProperties{nullptr}, OwnZeModule{true}, - AssociatedDevices({Context->getDevices()[0]}), - InteropZeModule{InteropZeModule} {} + AssociatedDevices({common_cast(Context)->getDevices()[0]}), + InteropZeModule{InteropZeModule} { + ddi_table = ddiTableOf(Context); +} ur_program_handle_t_::ur_program_handle_t_(state, ur_context_handle_t Context, ze_module_handle_t InteropZeModule, bool OwnZeModule) : Context{Context}, NativeProperties{nullptr}, OwnZeModule{OwnZeModule}, - AssociatedDevices({Context->getDevices()[0]}), + AssociatedDevices({common_cast(Context)->getDevices()[0]}), InteropZeModule{InteropZeModule} { + ddi_table = ddiTableOf(Context); // TODO: Currently it is not possible to understand the device associated // with provided ZeModule. So we can't set the state on that device to Exe. } @@ -1195,8 +1219,10 @@ ur_program_handle_t_::ur_program_handle_t_(state St, ur_context_handle_t Context, const std::string &ErrorMessage) : Context{Context}, NativeProperties{nullptr}, OwnZeModule{true}, - ErrorMessage{ErrorMessage}, AssociatedDevices(Context->getDevices()) { - for (auto &Device : Context->getDevices()) { + ErrorMessage{ErrorMessage}, + AssociatedDevices(common_cast(Context)->getDevices()) { + ddi_table = ddiTableOf(Context); + for (auto &Device : common_cast(Context)->getDevices()) { DeviceData &PerDevData = DeviceDataMap[Device->ZeDevice]; PerDevData.State = St; } @@ -1219,16 +1245,17 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { } if (!resourcesReleased) { for (auto &[ZeDevice, DeviceData] : this->DeviceDataMap) { - if (DeviceData.ZeBuildLog && checkL0LoaderTeardown()) + if (DeviceData.ZeBuildLog && ::ur::level_zero::checkL0LoaderTeardown()) ZE_CALL_NOCHECK(zeModuleBuildLogDestroy, (DeviceData.ZeBuildLog)); } // interop api - if (InteropZeModule && OwnZeModule && checkL0LoaderTeardown()) { + if (InteropZeModule && OwnZeModule && + ::ur::level_zero::checkL0LoaderTeardown()) { ZE_CALL_NOCHECK(zeModuleDestroy, (InteropZeModule)); } for (auto &[ZeDevice, DeviceData] : this->DeviceDataMap) - if (DeviceData.ZeModule && checkL0LoaderTeardown()) + if (DeviceData.ZeModule && ::ur::level_zero::checkL0LoaderTeardown()) ZE_CALL_NOCHECK(zeModuleDestroy, (DeviceData.ZeModule)); this->DeviceDataMap.clear(); @@ -1236,3 +1263,5 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { resourcesReleased = true; } } + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/common/sampler.cpp similarity index 89% rename from unified-runtime/source/adapters/level_zero/sampler.cpp rename to unified-runtime/source/adapters/level_zero/common/sampler.cpp index 7f1e24c658961..96f4e52589bf5 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/common/sampler.cpp @@ -8,13 +8,12 @@ //===----------------------------------------------------------------------===// #include "sampler.hpp" +#include "device.hpp" +#include "interfaces.hpp" #include "logger/ur_logger.hpp" +#include "platform.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else -#include "context.hpp" -#endif +namespace ur::level_zero { ur_result_t ur2zeSamplerDesc(ze_api_version_t ZeApiVersion, const ur_sampler_desc_t *SamplerDesc, @@ -78,17 +77,16 @@ ur_result_t ur2zeSamplerDesc(ze_api_version_t ZeApiVersion, return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - ur_result_t urSamplerCreate( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] specifies a list of sampler property names and their /// corresponding values. const ur_sampler_desc_t *Props, /// [out] pointer to handle of sampler object created - ur_sampler_handle_t *Sampler) { - std::shared_lock Lock(Context->Mutex); + ::ur_sampler_handle_t *SamplerOpque) { + auto Context = common_cast(ContextOpque); + std::shared_lock Lock(Context->getMutex()); // Have the "0" device in context to own the sampler. Rely on Level-Zero // drivers to perform migration as necessary for sharing it across multiple @@ -109,9 +107,10 @@ ur_result_t urSamplerCreate( &ZeSampler)); try { - ur_sampler_handle_t_ *UrSampler = new ur_sampler_handle_t_(ZeSampler); + ur_sampler_handle_t_ *UrSampler = + new ur_sampler_handle_t_(ZeSampler, ContextOpque); UrSampler->ZeSamplerDesc = ZeSamplerDesc; - *Sampler = reinterpret_cast(UrSampler); + *SamplerOpque = common_cast(UrSampler); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -122,14 +121,15 @@ ur_result_t urSamplerCreate( ur_result_t urSamplerRetain( /// [in] handle of the sampler object to get access - ur_sampler_handle_t Sampler) { - Sampler->RefCount.retain(); + ::ur_sampler_handle_t SamplerOpque) { + common_cast(SamplerOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release - ur_sampler_handle_t Sampler) { + ::ur_sampler_handle_t SamplerOpque) { + auto Sampler = common_cast(SamplerOpque); if (!Sampler->RefCount.release()) return UR_RESULT_SUCCESS; @@ -150,7 +150,7 @@ ur_result_t urSamplerRelease( ur_result_t urSamplerGetInfo( /// [in] handle of the sampler object - ur_sampler_handle_t /*Sampler*/, + ::ur_sampler_handle_t /*Sampler*/, /// [in] name of the sampler property to query ur_sampler_info_t /*PropName*/, /// [in] size in bytes of the sampler property value provided @@ -167,9 +167,9 @@ ur_result_t urSamplerGetInfo( ur_result_t urSamplerGetNativeHandle( /// [in] handle of the sampler. - ur_sampler_handle_t /*Sampler*/, + ::ur_sampler_handle_t /*Sampler*/, /// [out] a pointer to the native handle of the sampler. - ur_native_handle_t * /*NativeSampler*/) { + ::ur_native_handle_t * /*NativeSampler*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -178,13 +178,13 @@ ur_result_t urSamplerGetNativeHandle( ur_result_t urSamplerCreateWithNativeHandle( /// [in] the native handle of the sampler. - ur_native_handle_t /*NativeSampler*/, + ::ur_native_handle_t /*NativeSampler*/, /// [in] handle of the context object - ur_context_handle_t /*Context*/, + ::ur_context_handle_t /*Context*/, /// [in][optional] pointer to native sampler properties struct. const ur_sampler_native_properties_t * /*Properties*/, /// [out] pointer to the handle of the sampler object created. - ur_sampler_handle_t * /*Sampler*/) { + ::ur_sampler_handle_t * /*Sampler*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); diff --git a/unified-runtime/source/adapters/level_zero/sampler.hpp b/unified-runtime/source/adapters/level_zero/common/sampler.hpp similarity index 54% rename from unified-runtime/source/adapters/level_zero/sampler.hpp rename to unified-runtime/source/adapters/level_zero/common/sampler.hpp index 74b7f1e4d80bc..eecbafc56f7ef 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.hpp +++ b/unified-runtime/source/adapters/level_zero/common/sampler.hpp @@ -1,18 +1,27 @@ //===--------- sampler.hpp - Level Zero Adapter ---------------------------===// // +// Copyright (C) 2023 Intel Corporation // -// Part of the LLVM Project, under the Apache License v2.0 with LLVM -// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +#include "api.hpp" #include "common/ur_ref_count.hpp" +#include "helpers/shared_helpers.hpp" +#include "interfaces.hpp" -struct ur_sampler_handle_t_ : ur_object { - ur_sampler_handle_t_(ze_sampler_handle_t Sampler) : ZeSampler{Sampler} {} +namespace ur::level_zero { + +struct ur_sampler_handle_t_ : ur::level_zero::ur_object_t { + ur_sampler_handle_t_(ze_sampler_handle_t Sampler, ur_context_handle_t Context) + : ZeSampler{Sampler} { + // Populate ddi_table from the owning context. + ddi_table = ddiTableOf(Context); + } // Level Zero sampler handle. ze_sampler_handle_t ZeSampler; @@ -26,3 +35,5 @@ struct ur_sampler_handle_t_ : ur_object { ur_result_t ur2zeSamplerDesc(ze_api_version_t ZeApiVersion, const ur_sampler_desc_t *SamplerDesc, ZeStruct &ZeSamplerDesc); + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp b/unified-runtime/source/adapters/level_zero/common/virtual_mem.cpp similarity index 74% rename from unified-runtime/source/adapters/level_zero/virtual_mem.cpp rename to unified-runtime/source/adapters/level_zero/common/virtual_mem.cpp index 5100c0025fbe5..bdfe2672f9bb8 100644 --- a/unified-runtime/source/adapters/level_zero/virtual_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/common/virtual_mem.cpp @@ -7,23 +7,20 @@ // //===----------------------------------------------------------------------===// -#include "common.hpp" #include "device.hpp" +#include "interfaces.hpp" #include "logger/ur_logger.hpp" #include "physical_mem.hpp" -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else -#include "context.hpp" -#endif - namespace ur::level_zero { ur_result_t urVirtualMemGranularityGetInfo( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, size_t allocationSize, ur_virtual_mem_granularity_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto hContext = common_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: @@ -38,32 +35,40 @@ ur_result_t urVirtualMemGranularityGetInfo( return ReturnValue(PageSize); } default: - UR_LOG(ERR, "Unsupported propName in urQueueGetInfo: propName={}({})", + UR_LOG(ERR, + "Unsupported propName in urVirtualMemGranularityGetInfo: " + "propName={}({})", propName, propName); return UR_RESULT_ERROR_INVALID_VALUE; } return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemReserve(ur_context_handle_t hContext, +ur_result_t urVirtualMemReserve(::ur_context_handle_t hContextOpque, const void *pStart, size_t size, void **ppStart) { + auto hContext = common_cast(hContextOpque); + ZE2UR_CALL(zeVirtualMemReserve, (hContext->getZeHandle(), pStart, size, ppStart)); return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemFree(ur_context_handle_t hContext, const void *pStart, - size_t size) { +ur_result_t urVirtualMemFree(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size) { + auto hContext = common_cast(hContextOpque); + ZE2UR_CALL(zeVirtualMemFree, (hContext->getZeHandle(), pStart, size)); return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemSetAccess(ur_context_handle_t hContext, +ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContextOpque, const void *pStart, size_t size, ur_virtual_mem_access_flags_t flags) { + auto hContext = common_cast(hContextOpque); + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; @@ -76,10 +81,14 @@ ur_result_t urVirtualMemSetAccess(ur_context_handle_t hContext, return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, - size_t size, ur_physical_mem_handle_t hPhysicalMem, +ur_result_t urVirtualMemMap(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size, + ::ur_physical_mem_handle_t hPhysicalMemOpque, size_t offset, ur_virtual_mem_access_flags_t flags) { + auto hContext = common_cast(hContextOpque); + auto hPhysicalMem = common_cast(hPhysicalMemOpque); + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; @@ -93,18 +102,22 @@ ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemUnmap(ur_context_handle_t hContext, const void *pStart, - size_t size) { +ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContextOpque, + const void *pStart, size_t size) { + auto hContext = common_cast(hContextOpque); + ZE2UR_CALL(zeVirtualMemUnmap, (hContext->getZeHandle(), pStart, size)); return UR_RESULT_SUCCESS; } -ur_result_t urVirtualMemGetInfo(ur_context_handle_t hContext, +ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContextOpque, const void *pStart, [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto hContext = common_cast(hContextOpque); + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { @@ -120,7 +133,7 @@ ur_result_t urVirtualMemGetInfo(ur_context_handle_t hContext, return ReturnValue(RetFlags); } default: - UR_LOG(ERR, "Unsupported propName in urQueueGetInfo: propName={}({})", + UR_LOG(ERR, "Unsupported propName in urVirtualMemGetInfo: propName={}({})", propName, propName); return UR_RESULT_ERROR_INVALID_VALUE; } diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index cc58068623020..c8d565654c079 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -18,17 +18,18 @@ #include "queue.hpp" #include "ur_level_zero.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v1 { ur_result_t urContextCreate( /// [in] the number of devices given in phDevices uint32_t DeviceCount, /// [in][range(0, DeviceCount)] array of handle of devices. - const ur_device_handle_t *Devices, + const ::ur_device_handle_t *DevicesOpque, /// [in][optional] pointer to context creation properties. const ur_context_properties_t * /*Properties*/, /// [out] pointer to handle of context object created - ur_context_handle_t *RetContext) { + ::ur_context_handle_t *RetContextOpque) { + auto Devices = common_cast(DevicesOpque); ur_platform_handle_t Platform = Devices[0]->Platform; ZeStruct ContextDesc{}; @@ -36,16 +37,17 @@ ur_result_t urContextCreate( ze_context_handle_t ZeContext{}; ZE2UR_CALL(zeContextCreate, (Platform->ZeDriver, &ContextDesc, &ZeContext)); try { - ur_context_handle_t_ *Context = - new ur_context_handle_t_(ZeContext, DeviceCount, Devices, true); + ur::level_zero::v1::ur_context_handle_t_ *Context = + new ur::level_zero::v1::ur_context_handle_t_(ZeContext, DeviceCount, + Devices, true); Context->initialize(); - *RetContext = reinterpret_cast(Context); + *v1_cast(RetContextOpque) = Context; // TODO: delete below 'if' when memory isolation in the context is // implemented in the driver if (IndirectAccessTrackingEnabled) { std::scoped_lock Lock(Platform->ContextsMutex); - Platform->Contexts.push_back(*RetContext); + Platform->Contexts.push_back(*RetContextOpque); } } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; @@ -61,14 +63,15 @@ ur_result_t urContextCreate( ur_result_t urContextRetain( /// [in] handle of the context to get a reference of. - ur_context_handle_t Context) { - Context->RefCount.retain(); + ::ur_context_handle_t ContextOpque) { + v1_cast(ContextOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urContextRelease( /// [in] handle of the context to release. - ur_context_handle_t Context) { + ::ur_context_handle_t ContextOpque) { + auto Context = v1_cast(ContextOpque); ur_platform_handle_t Plt = Context->getPlatform(); std::unique_lock ContextsLock(Plt->ContextsMutex, std::defer_lock); @@ -92,7 +95,7 @@ static const bool UseMemcpy2DOperations = [] { ur_result_t urContextGetInfo( /// [in] handle of the context - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] type of the info to retrieve ur_context_info_t ContextInfoType, /// [in] the number of bytes of memory pointed to by pContextInfo. @@ -105,14 +108,16 @@ ur_result_t urContextGetInfo( /// [out][optional] pointer to the actual size in bytes of data queried by /// ContextInfoType. size_t *PropSizeRet) { + auto Context = v1_cast(ContextOpque); std::shared_lock Lock(Context->Mutex); UrReturnHelper ReturnValue(PropSize, ContextInfo, PropSizeRet); switch ( (uint32_t)ContextInfoType) { // cast to avoid warnings on EXT enum values case UR_CONTEXT_INFO_DEVICES: - return ReturnValue(&Context->Devices[0], Context->Devices.size()); + return ReturnValue(Context->getDevices().data(), + Context->getDevices().size()); case UR_CONTEXT_INFO_NUM_DEVICES: - return ReturnValue(uint32_t(Context->Devices.size())); + return ReturnValue(uint32_t(Context->getDevices().size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: return ReturnValue(uint32_t{Context->RefCount.getCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: @@ -130,29 +135,33 @@ ur_result_t urContextGetInfo( ur_result_t urContextGetNativeHandle( /// [in] handle of the context. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [out] a pointer to the native handle of the context. - ur_native_handle_t *NativeContext) { - *NativeContext = reinterpret_cast(Context->ZeContext); + ::ur_native_handle_t *NativeContext) { + *NativeContext = + reinterpret_cast(v1_cast(ContextOpque)->ZeContext); return UR_RESULT_SUCCESS; } ur_result_t urContextCreateWithNativeHandle( - ur_native_handle_t + ::ur_native_handle_t /// [in] the native handle of the context. NativeContext, - ur_adapter_handle_t, uint32_t NumDevices, const ur_device_handle_t *Devices, + ::ur_adapter_handle_t, uint32_t NumDevices, + const ::ur_device_handle_t *DevicesOpque, const ur_context_native_properties_t *Properties, /// [out] pointer to the handle of the context object created. - ur_context_handle_t *Context) { + ::ur_context_handle_t *ContextOpque) { + auto Devices = common_cast(DevicesOpque); bool OwnNativeHandle = Properties ? Properties->isNativeHandleOwned : false; try { ze_context_handle_t ZeContext = reinterpret_cast(NativeContext); - ur_context_handle_t_ *UrContext = new ur_context_handle_t_( - ZeContext, NumDevices, Devices, OwnNativeHandle); + ur::level_zero::v1::ur_context_handle_t_ *UrContext = + new ur::level_zero::v1::ur_context_handle_t_(ZeContext, NumDevices, + Devices, OwnNativeHandle); UrContext->initialize(); - *Context = reinterpret_cast(UrContext); + *v1_cast(ContextOpque) = UrContext; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -163,7 +172,7 @@ ur_result_t urContextCreateWithNativeHandle( ur_result_t urContextSetExtendedDeleter( /// [in] handle of the context. - ur_context_handle_t /*Context*/, + ::ur_context_handle_t /*Context*/, /// [in] Function pointer to extended deleter. ur_context_extended_deleter_t /*Deleter*/, /// [in][out][optional] pointer to data to be passed to callback. @@ -173,9 +182,14 @@ ur_result_t urContextSetExtendedDeleter( "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero -ur_result_t ur_context_handle_t_::initialize() { +ur_result_t ur::level_zero::v1::ur_context_handle_t_::initialize() { + assert( + static_cast(this) == + static_cast( + &static_cast(this)->ddi_table) && + "ddi_table must be at offset 0 for loader intercept dispatch"); + // Create the immediate command list to be used for initializations. // Created as synchronous so level-zero performs implicit synchronization and // there is no need to query for completion in the plugin @@ -186,7 +200,7 @@ ur_result_t ur_context_handle_t_::initialize() { // D2D migartion, if no P2P, is broken since it should use // immediate command-list for the specfic devices, and this single one. // - ur_device_handle_t Device = Devices[0]; + ur_device_handle_t Device = getDevices()[0]; // Prefer to use copy engine for initialization copies, // if available and allowed (main copy engine with index 0). @@ -219,7 +233,9 @@ ur_result_t ur_context_handle_t_::initialize() { return UR_RESULT_SUCCESS; } -ur_device_handle_t ur_context_handle_t_::getRootDevice() const { +ur_device_handle_t +ur::level_zero::v1::ur_context_handle_t_::getRootDevice() const { + const auto &Devices = getDevices(); assert(Devices.size() > 0); if (Devices.size() == 1) @@ -258,7 +274,7 @@ ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { if (IndirectAccessTrackingEnabled) { ur_platform_handle_t Plt = Context->getPlatform(); auto &Contexts = Plt->Contexts; - auto It = std::find(Contexts.begin(), Contexts.end(), Context); + auto It = std::find(Contexts.begin(), Contexts.end(), v1_cast(Context)); if (It != Contexts.end()) Contexts.erase(It); } @@ -292,11 +308,7 @@ ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { return Result; } -ur_platform_handle_t ur_context_handle_t_::getPlatform() const { - return Devices[0]->Platform; -} - -ur_result_t ur_context_handle_t_::finalize() { +ur_result_t ur::level_zero::v1::ur_context_handle_t_::finalize() { // This function is called when ur_context_handle_t is deallocated, // urContextRelease. There could be some memory that may have not been // deallocated. For example, event and event pool caches would be still alive. @@ -403,7 +415,8 @@ static const uint32_t MaxNumEventsPerPool = [] { return Result; }(); -ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool( +ur_result_t +ur::level_zero::v1::ur_context_handle_t_::getFreeSlotInExistingOrNewPool( ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible, bool ProfilingEnabled, ur_device_handle_t Device, bool CounterBasedEventEnabled, bool UsingImmCmdList, @@ -456,7 +469,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool( if (ZeDevice) { ZeDevices.push_back(ZeDevice); } else { - std::for_each(Devices.begin(), Devices.end(), + std::for_each(getDevices().begin(), getDevices().end(), [&](const ur_device_handle_t &D) { ZeDevices.push_back(D->ZeDevice); }); @@ -573,7 +586,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool( if (ZeDevice) { ZeDevices.push_back(ZeDevice); } else { - std::for_each(Devices.begin(), Devices.end(), + std::for_each(getDevices().begin(), getDevices().end(), [&](const ur_device_handle_t &D) { ZeDevices.push_back(D->ZeDevice); }); @@ -610,7 +623,8 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool( return UR_RESULT_SUCCESS; } -ur_event_handle_t ur_context_handle_t_::getEventFromContextCache( +ur_event_handle_t +ur::level_zero::v1::ur_context_handle_t_::getEventFromContextCache( bool HostVisible, bool WithProfiling, ur_device_handle_t Device, bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) { // Don't reuse events with profiling enabled because zeEventHostReset @@ -650,7 +664,8 @@ ur_event_handle_t ur_context_handle_t_::getEventFromContextCache( return Event; } -void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) { +void ur::level_zero::v1::ur_context_handle_t_::addEventToContextCache( + ur_event_handle_t Event) { std::scoped_lock Lock(EventCacheMutex); ur_device_handle_t Device = nullptr; @@ -670,7 +685,8 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) { } ur_result_t -ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) { +ur::level_zero::v1::ur_context_handle_t_::decrementUnreleasedEventsInPool( + ur_event_handle_t Event) { std::shared_lock EventLock(Event->Mutex, std::defer_lock); std::scoped_lock> LockAll( ZeEventPoolCacheMutex, EventLock); @@ -927,20 +943,4 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList( return ur_result; } -bool ur_context_handle_t_::isValidDevice(ur_device_handle_t Device) const { - while (Device) { - if (std::find(Devices.begin(), Devices.end(), Device) != Devices.end()) - return true; - Device = Device->RootDevice; - } - return false; -} - -const std::vector & -ur_context_handle_t_::getDevices() const { - return Devices; -} - -ze_context_handle_t ur_context_handle_t_::getZeHandle() const { - return ZeContext; -} +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/context.hpp b/unified-runtime/source/adapters/level_zero/context.hpp index 2ec7652e5f046..3fb8c304ab9f1 100644 --- a/unified-runtime/source/adapters/level_zero/context.hpp +++ b/unified-runtime/source/adapters/level_zero/context.hpp @@ -22,12 +22,15 @@ #include #include "common.hpp" +#include "common/interfaces.hpp" #include "queue.hpp" #include "usm.hpp" #include "common/ur_ref_count.hpp" #include +namespace ur::level_zero::v1 { + struct l0_command_list_cache_info { ZeStruct ZeQueueDesc; bool InOrderList = false; @@ -54,33 +57,21 @@ typedef struct _ze_intel_event_sync_mode_exp_desc_t { extern const bool UseUSMAllocator; -struct ur_context_handle_t_ : ur_object { +struct ur_context_handle_t_ : ur_context_common_t { ur_context_handle_t_(ze_context_handle_t ZeContext, uint32_t NumDevices, const ur_device_handle_t *Devs, bool OwnZeContext) - : ZeContext{ZeContext}, Devices{Devs, Devs + NumDevices}, - NumDevices{NumDevices}, DefaultPool{this, nullptr, !UseUSMAllocator}, + : ur_context_common_t(ZeContext, {Devs, Devs + NumDevices}, + NumDevices ? Devs[0]->Platform : nullptr), + DefaultPool{this, nullptr, !UseUSMAllocator}, AsyncPool{this, nullptr, !UseUSMAllocator} { OwnNativeHandle = OwnZeContext; } ur_context_handle_t_(ze_context_handle_t ZeContext) - : ZeContext{ZeContext}, DefaultPool{this, nullptr, !UseUSMAllocator}, + : ur_context_common_t(ZeContext, {}, nullptr), + DefaultPool{this, nullptr, !UseUSMAllocator}, AsyncPool{this, nullptr, !UseUSMAllocator} {} - // A L0 context handle is primarily used during creation and management of - // resources that may be used by multiple devices. - // This field is only set at ur_context_handle_t creation time, and cannot - // change. Therefore it can be accessed without holding a lock on this - // ur_context_handle_t. - const ze_context_handle_t ZeContext{}; - - // Keep the PI devices this PI context was created for. - // This field is only set at ur_context_handle_t creation time, and cannot - // change. Therefore it can be accessed without holding a lock on this - // ur_context_handle_t. const std::vector Devices; - std::vector Devices; - uint32_t NumDevices{}; - // Immediate Level Zero command list for the device in this context, to be // used for initializations. To be created as: // - Immediate command list: So any command appended to it is immediately @@ -191,12 +182,6 @@ struct ur_context_handle_t_ : ur_object { // Finalize the PI context ur_result_t finalize(); - // Return the Platform, which is the same for all devices in the context - ur_platform_handle_t getPlatform() const; - - // Get vector of devices from this context - const std::vector &getDevices() const; - // Get index of the free slot in the available pool. If there is no available // pool then create new one. The HostVisible parameter tells if we need a // slot for a host-visible event. The ProfilingEnabled tells is we need a @@ -350,13 +335,6 @@ struct ur_context_handle_t_ : ur_object { const ur_event_handle_t *EventWaitList, bool AllowBatching, ze_command_queue_handle_t *ForcedCmdQueue); - // Checks if Device is covered by this context. - // For that the Device or its root devices need to be in the context. - bool isValidDevice(ur_device_handle_t Device) const; - - // Get handle to the L0 context - ze_context_handle_t getZeHandle() const; - ur::RefCount RefCount; private: @@ -411,3 +389,5 @@ struct ur_context_handle_t_ : ur_object { // mutex guarding the container with contexts because the context can be removed // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context); + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 07cf4fb5b7fc6..fc4a9f7d8cc3f 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1,4 +1,4 @@ -//===--------- device.cpp - Level Zero Adapter ----------------------------===// +//===--------- device.cpp - Level Zero Adapter ---------------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -6,2444 +6,40 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -#include "device.hpp" -#include "adapter.hpp" -#include "logger/ur_logger.hpp" -#include "ur_interface_loader.hpp" -#include "ur_level_zero.hpp" -#include "ur_util.hpp" -#include -#include -#if defined(__linux__) -#include -#elif defined(_WIN32) -#include -#endif -#include -#include - -// UR_L0_USE_COPY_ENGINE can be set to an integer value, or -// a pair of integer values of the form "lower_index:upper_index". -// Here, the indices point to copy engines in a list of all available copy -// engines. -// This functions returns this pair of indices. -// If the user specifies only a single integer, a value of 0 indicates that -// the copy engines will not be used at all. A value of 1 indicates that all -// available copy engines can be used. -const std::pair -getRangeOfAllowedCopyEngines(const ur_device_handle_t &Device) { - const char *UrRet = std::getenv("UR_L0_USE_COPY_ENGINE"); - const char *PiRet = std::getenv("SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE"); - static const char *EnvVar = UrRet ? UrRet : (PiRet ? PiRet : nullptr); - // If the environment variable is not set, no copy engines are used when - // immediate commandlists are being used. For standard commandlists all are - // used. - if (!EnvVar) { - if (Device->ImmCommandListUsed) - return std::pair(0, 0); // Only main copy engine will be used. - return std::pair(0, INT_MAX); // All copy engines will be used. - } - std::string CopyEngineRange = EnvVar; - // Environment variable can be a single integer or a pair of integers - // separated by ":" - auto pos = CopyEngineRange.find(":"); - if (pos == std::string::npos) { - bool UseCopyEngine = (std::stoi(CopyEngineRange) != 0); - if (UseCopyEngine) - return std::pair(0, INT_MAX); // All copy engines can be used. - return std::pair(-1, -1); // No copy engines will be used. - } - int LowerCopyEngineIndex = std::stoi(CopyEngineRange.substr(0, pos)); - int UpperCopyEngineIndex = std::stoi(CopyEngineRange.substr(pos + 1)); - if ((LowerCopyEngineIndex > UpperCopyEngineIndex) || - (LowerCopyEngineIndex < -1) || (UpperCopyEngineIndex < -1)) { - UR_LOG(ERR, "UR_L0_LEVEL_ZERO_USE_COPY_ENGINE: invalid value provided, " - "default set."); - LowerCopyEngineIndex = 0; - UpperCopyEngineIndex = INT_MAX; - } - return std::pair(LowerCopyEngineIndex, UpperCopyEngineIndex); -} - -namespace ur::level_zero { - -ur_result_t urDeviceGet( - /// [in] handle of the platform instance - ur_platform_handle_t Platform, - /// [in] the type of the devices. - ur_device_type_t DeviceType, - /// [in] the number of devices to be added to phDevices. If phDevices in not - /// NULL then NumEntries should be greater than zero, otherwise - /// ::UR_RESULT_ERROR_INVALID_SIZE, will be returned. - uint32_t NumEntries, - /// [out][optional][range(0, NumEntries)] array of handle of devices. If - /// NumEntries is less than the number of devices available, then - /// platform shall only retrieve that number of devices. - ur_device_handle_t *Devices, - /// [out][optional] pointer to the number of devices. pNumDevices will be - /// updated with the total number of devices available. - uint32_t *NumDevices) { - - auto Res = Platform->populateDeviceCacheIfNeeded(); - if (Res != UR_RESULT_SUCCESS) { - return Res; - } - - // Filter available devices based on input DeviceType. - std::vector MatchedDevices; - std::shared_lock Lock(Platform->URDevicesCacheMutex); - // We need to filter out composite devices when - // ZE_FLAT_DEVICE_HIERARCHY=COMBINED. We can know if we are in combined - // mode depending on the return value of zeDeviceGetRootDevice: - // - If COMPOSITE, L0 returns cards as devices. Since we filter out - // subdevices early, zeDeviceGetRootDevice must return nullptr, because we - // only query for root-devices and they don't have any device higher up in - // the hierarchy. - // - If FLAT, according to L0 spec, zeDeviceGetRootDevice always returns - // nullptr in this mode. - // - If COMBINED, L0 returns tiles as devices, and zeDeviceGetRootdevice - // returns the card containing a given tile. - - // Track best discrete and integrated GPU candidates (device, max compute - // units) - std::pair GPUDeviceDiscrete = {nullptr, 0}; - std::pair GPUDeviceIntegrated = {nullptr, 0}; - bool DeviceDefaultGPU = false; - - bool isCombinedMode = - std::any_of(Platform->URDevicesCache.begin(), - Platform->URDevicesCache.end(), [](const auto &D) { - if (D->isSubDevice()) - return false; - ze_device_handle_t RootDev = nullptr; - // Query Root Device for root-devices. - // We cannot use ZE2UR_CALL because under some circumstances - // this call may return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, - // and ZE2UR_CALL will abort because it's not - // UR_RESULT_SUCCESS. Instead, we use ZE_CALL_NOCHECK and we - // check manually that the result is either - // ZE_RESULT_SUCCESS or ZE_RESULT_ERROR_UNSUPPORTED_FEATURE. - auto errc = ZE_CALL_NOCHECK(zeDeviceGetRootDevice, - (D->ZeDevice, &RootDev)); - return (errc == ZE_RESULT_SUCCESS && RootDev != nullptr); - }); - for (auto &D : Platform->URDevicesCache) { - // Only ever return root-devices from urDeviceGet, but the - // devices cache also keeps sub-devices. - if (D->isSubDevice()) - continue; - - bool Matched = false; - switch (DeviceType) { - case UR_DEVICE_TYPE_ALL: - Matched = true; - break; - case UR_DEVICE_TYPE_GPU: - Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU); - break; - case UR_DEVICE_TYPE_DEFAULT: - Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU); - DeviceDefaultGPU = true; - break; - case UR_DEVICE_TYPE_CPU: - Matched = (D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_CPU); - break; - case UR_DEVICE_TYPE_FPGA: - Matched = D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_FPGA; - break; - case UR_DEVICE_TYPE_MCA: - Matched = D->ZeDeviceProperties->type == ZE_DEVICE_TYPE_MCA; - break; - default: - Matched = false; - UR_LOG(WARN, "Unknown device type"); - break; - } - - if (Matched) { - bool isComposite = - isCombinedMode && (D->ZeDeviceProperties->flags & - ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) == 0; - if (!isComposite) { - // In case of DeviceType is DEFAULT, pick only the most powerful GPU - // device. - if (DeviceDefaultGPU) { - uint32_t maxComputeUnits = 0; - ur_result_t UrRet = ur::level_zero::urDeviceGetInfo( - D.get(), UR_DEVICE_INFO_MAX_COMPUTE_UNITS, - sizeof(maxComputeUnits), &maxComputeUnits, nullptr); - maxComputeUnits = (UrRet == UR_RESULT_SUCCESS) ? maxComputeUnits : 0; - auto &BestGpu = - D->isIntegrated() ? GPUDeviceIntegrated : GPUDeviceDiscrete; - if (!BestGpu.first || maxComputeUnits > BestGpu.second) - BestGpu = std::make_pair(D.get(), maxComputeUnits); - } else { - MatchedDevices.push_back(D.get()); - } - } - } - } - - // Handle GPU/DEFAULT device selection outside the loop - if (DeviceDefaultGPU) { - // Prefer discrete GPU over integrated GPU - if (GPUDeviceDiscrete.first) { - MatchedDevices = {GPUDeviceDiscrete.first}; - } else if (GPUDeviceIntegrated.first) { - MatchedDevices = {GPUDeviceIntegrated.first}; - } - } - - uint32_t ZeDeviceCount = MatchedDevices.size(); - - auto N = (std::min)(ZeDeviceCount, NumEntries); - if (Devices) - std::copy_n(MatchedDevices.begin(), N, Devices); - - if (NumDevices) - *NumDevices = ZeDeviceCount; - - return UR_RESULT_SUCCESS; -} - -uint64_t calculateGlobalMemSize(ur_device_handle_t Device) { - // Cache GlobalMemSize - Device->ZeGlobalMemSize.Compute = - [Device](struct ze_global_memsize &GlobalMemSize) { - for (const auto &ZeDeviceMemoryExtProperty : - Device->ZeDeviceMemoryProperties->second) { - GlobalMemSize.value += ZeDeviceMemoryExtProperty.physicalSize; - } - if (GlobalMemSize.value == 0) { - for (const auto &ZeDeviceMemoryProperty : - Device->ZeDeviceMemoryProperties->first) { - GlobalMemSize.value += ZeDeviceMemoryProperty.totalSize; - } - } - }; - return Device->ZeGlobalMemSize.get().value; -} - -static bool -supportsDeviceUsableMemSizeExtension(ur_platform_handle_t Platform) { -#ifdef ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME - constexpr const char *ExtensionName = - ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME; - constexpr uint32_t MinVersion = ZE_MAKE_VERSION(1, 0); - - auto Extension = Platform->zeDriverExtensionMap.find(ExtensionName); - return Extension != Platform->zeDriverExtensionMap.end() && - Extension->second >= MinVersion; -#else - std::ignore = Platform; - return false; -#endif -} - -static std::optional -getDeviceUsableMemSizeFromCore(ur_device_handle_t Device) { -#ifdef ZE_DEVICE_USABLEMEM_SIZE_PROPERTIES_EXT_NAME - if (!supportsDeviceUsableMemSizeExtension(Device->Platform)) { - return std::nullopt; - } - - ZeStruct DeviceProperties; - ZeStruct UsableMemProperties; - DeviceProperties.pNext = &UsableMemProperties; - - auto ZeResult = ZE_CALL_NOCHECK(zeDeviceGetProperties, - (Device->ZeDevice, &DeviceProperties)); - if (ZeResult != ZE_RESULT_SUCCESS) { - return std::nullopt; - } - - return UsableMemProperties.currUsableMemSize; -#else - std::ignore = Device; - return std::nullopt; -#endif -} - -// Return the Sysman device handle and correpsonding data for the given UR -// device. -static std::tuple -getZesDeviceData(ur_device_handle_t Device) { - bool SysManEnv = getenv_tobool("ZES_ENABLE_SYSMAN", false); - if ((Device->Platform->ZedeviceToZesDeviceMap.size() == 0) && !SysManEnv) { - UR_LOG(ERR, "SysMan support is unavailable on this system. Please " - "check your level zero driver installation."); - return {nullptr, {}, UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION}; - } - - zes_device_handle_t ZesDevice = Device->ZeDevice; - ur_zes_device_handle_data_t ZesDeviceData = {}; - // If legacy sysman is enabled thru the environment variable, then zesInit - // will fail, but sysman is still usable so go the legacy route. - if (!SysManEnv) { - auto It = Device->Platform->ZedeviceToZesDeviceMap.find(Device->ZeDevice); - if (It == Device->Platform->ZedeviceToZesDeviceMap.end()) { - // no matching device - return {nullptr, {}, UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION}; - } else { - ZesDeviceData = - Device->Platform->ZedeviceToZesDeviceMap[Device->ZeDevice]; - ZesDevice = ZesDeviceData.ZesDevice; - } - } else { - ZesDeviceData.SubDevice = Device->isSubDevice(); - ZesDeviceData.SubDeviceId = Device->ZeDeviceProperties->subdeviceId; - } - - return {ZesDevice, ZesDeviceData, UR_RESULT_SUCCESS}; -} -ur_result_t urDeviceGetInfo( - /// [in] handle of the device instance - ur_device_handle_t Device, - /// [in] type of the info to retrieve - ur_device_info_t ParamName, - /// [in] the number of bytes pointed to by ParamValue. - size_t propSize, - /// [out][optional] array of bytes holding the info. If propSize is not - /// equal to or greater than the real number of bytes needed to return the - /// info then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pDeviceInfo is not used. - void *ParamValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// infoType. - size_t *pSize) { - UrReturnHelper ReturnValue(propSize, ParamValue, pSize); +#include +#include - ze_device_handle_t ZeDevice = Device->ZeDevice; - - switch ((int)ParamName) { - case UR_DEVICE_INFO_TYPE: { - switch (Device->ZeDeviceProperties->type) { - case ZE_DEVICE_TYPE_GPU: - return ReturnValue(UR_DEVICE_TYPE_GPU); - case ZE_DEVICE_TYPE_CPU: - return ReturnValue(UR_DEVICE_TYPE_CPU); - case ZE_DEVICE_TYPE_FPGA: - return ReturnValue(UR_DEVICE_TYPE_FPGA); - default: - UR_LOG(ERR, "This device type is not supported"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - } - case UR_DEVICE_INFO_PARENT_DEVICE: - return ReturnValue(Device->RootDevice); - case UR_DEVICE_INFO_PLATFORM: - return ReturnValue(Device->Platform); - case UR_DEVICE_INFO_VENDOR_ID: - return ReturnValue(uint32_t{Device->ZeDeviceProperties->vendorId}); - case UR_DEVICE_INFO_UUID: { - // Intel extension for device UUID. This returns the UUID as - // std::array. For details about this extension, - // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. - const auto &UUID = Device->ZeDeviceProperties->uuid.id; - return ReturnValue(UUID, sizeof(UUID)); - } - case UR_DEVICE_INFO_MAX_MEMORY_BANDWIDTH: { - // ZeDeviceMemoryProperties should be set already by initialization - if (Device->ZeDeviceMemoryProperties->second.empty()) - return ReturnValue(uint64_t{0}); - - uint32_t maxBandwidth = 0; - for (const auto &extProp : Device->ZeDeviceMemoryProperties->second) { - // Only consider bandwidth if the unit is BYTES_PER_NANOSEC - if (extProp.bandwidthUnit == ZE_BANDWIDTH_UNIT_BYTES_PER_NANOSEC) { - maxBandwidth = std::max( - {maxBandwidth, extProp.readBandwidth, extProp.writeBandwidth}); - } - } - // Convert to Bytes/sec from Bytes/nanosec - return ReturnValue(static_cast(maxBandwidth * 1e9)); - } - case UR_DEVICE_INFO_ATOMIC_64: - return ReturnValue( - static_cast(Device->ZeDeviceModuleProperties->flags & - ZE_DEVICE_MODULE_FLAG_INT64_ATOMICS)); - case UR_DEVICE_INFO_EXTENSIONS: { - // Convention adopted from OpenCL: - // "Returns a space separated list of extension names (the extension - // names themselves do not contain any spaces) supported by the device." - // - // TODO: Use proper mechanism to get this information from Level Zero after - // it is added to Level Zero. - // Hardcoding the few we know are supported by the current hardware. - // - // - std::string SupportedExtensions; - - // cl_khr_il_program - OpenCL 2.0 KHR extension for SPIR-V support. Core - // feature in >OpenCL 2.1 - // cl_khr_subgroups - Extension adds support for implementation-controlled - // subgroups. - // cl_intel_subgroups - Extension adds subgroup features, defined by Intel. - // cl_intel_subgroups_short - Extension adds subgroup functions described in - // the cl_intel_subgroups extension to support 16-bit integer data types - // for performance. - // cl_intel_required_subgroup_size - Extension to allow programmers to - // optionally specify the required subgroup size for a kernel function. - // cl_khr_fp16 - Optional half floating-point support. - // cl_khr_fp64 - Support for double floating-point precision. - // cl_khr_int64_base_atomics, cl_khr_int64_extended_atomics - Optional - // extensions that implement atomic operations on 64-bit signed and - // unsigned integers to locations in __global and __local memory. - // cl_khr_3d_image_writes - Extension to enable writes to 3D image memory - // objects. - // - // Hardcoding some extensions we know are supported by all Level Zero - // devices. - SupportedExtensions += (ZE_SUPPORTED_EXTENSIONS); - if (Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16) - SupportedExtensions += ("cl_khr_fp16 "); - if (Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP64) - SupportedExtensions += ("cl_khr_fp64 "); - if (Device->ZeDeviceModuleProperties->flags & - ZE_DEVICE_MODULE_FLAG_INT64_ATOMICS) - // int64AtomicsSupported indicates support for both. - SupportedExtensions += - ("cl_khr_int64_base_atomics cl_khr_int64_extended_atomics "); - if (Device->ZeDeviceImageProperties->maxImageDims3D > 0) - // Supports reading and writing of images. - SupportedExtensions += ("cl_khr_3d_image_writes "); - - if (Device->Platform->zeDriverExtensionMap.count( - ZE_BFLOAT16_CONVERSIONS_EXT_NAME)) - SupportedExtensions += ("cl_intel_bfloat16_conversions "); - else if ((Device->ZeDeviceProperties->deviceId & 0xfff) == 0x201 || - (Device->ZeDeviceProperties->deviceId & 0xff0) == 0xbd0) - SupportedExtensions += ("cl_intel_bfloat16_conversions "); - - return ReturnValue(SupportedExtensions.c_str()); - } - case UR_DEVICE_INFO_NAME: - return ReturnValue(Device->ZeDeviceProperties->name); - // zeModuleCreate allows using root device module for sub-devices: - // > The application must only use the module for the device, or its - // > sub-devices, which was provided during creation. - case UR_DEVICE_INFO_BUILD_ON_SUBDEVICE: - return ReturnValue(ur_bool_t{0}); - case UR_DEVICE_INFO_COMPILER_AVAILABLE: - return ReturnValue(static_cast(true)); - case UR_DEVICE_INFO_LINKER_AVAILABLE: - return ReturnValue(static_cast(true)); - case UR_DEVICE_INFO_MAX_COMPUTE_UNITS: { - uint32_t MaxComputeUnits = - Device->ZeDeviceProperties->numEUsPerSubslice * - Device->ZeDeviceProperties->numSubslicesPerSlice * - Device->ZeDeviceProperties->numSlices; - - bool RepresentsCSlice = - Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] - .ZeIndex >= 0; - if (RepresentsCSlice) - MaxComputeUnits /= Device->RootDevice->SubDevices.size(); - - return ReturnValue(uint32_t{MaxComputeUnits}); - } - case UR_DEVICE_INFO_NUM_COMPUTE_UNITS: { - uint32_t NumComputeUnits = - Device->ZeDeviceProperties->numSubslicesPerSlice * - Device->ZeDeviceProperties->numSlices; - return ReturnValue(uint32_t{NumComputeUnits}); - } - case UR_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS: - // Level Zero spec defines only three dimensions - return ReturnValue(uint32_t{3}); - case UR_DEVICE_INFO_MAX_WORK_GROUP_SIZE: - return ReturnValue( - uint64_t{Device->ZeDeviceComputeProperties->maxTotalGroupSize}); - case UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES: { - struct { - size_t Arr[3]; - } MaxGroupSize = {{Device->ZeDeviceComputeProperties->maxGroupSizeX, - Device->ZeDeviceComputeProperties->maxGroupSizeY, - Device->ZeDeviceComputeProperties->maxGroupSizeZ}}; - return ReturnValue(MaxGroupSize); - } - case UR_DEVICE_INFO_MAX_WORK_GROUPS_3D: { - struct { - size_t Arr[3]; - } MaxGroupCounts = {{Device->ZeDeviceComputeProperties->maxGroupCountX, - Device->ZeDeviceComputeProperties->maxGroupCountY, - Device->ZeDeviceComputeProperties->maxGroupCountZ}}; - return ReturnValue(MaxGroupCounts); - } - case UR_DEVICE_INFO_MAX_WORK_GROUPS: { - // Multiply the max group counts in each dimension to get the total max - // number of work groups. Prevent overflow. - return ReturnValue(multiplyWithOverflowCheck( - Device->ZeDeviceComputeProperties->maxGroupCountX, - Device->ZeDeviceComputeProperties->maxGroupCountY, - Device->ZeDeviceComputeProperties->maxGroupCountZ)); - } - case UR_DEVICE_INFO_MAX_CLOCK_FREQUENCY: - return ReturnValue(uint32_t{Device->ZeDeviceProperties->coreClockRate}); - case UR_DEVICE_INFO_ADDRESS_BITS: { - // TODO: To confirm with spec. - return ReturnValue(uint32_t{64}); - } - case UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE: - // if the user wishes to allocate large allocations on a system that usually - // does not allow that allocation size, then we return the max global mem - // size as the limit. - if (Device->useRelaxedAllocationLimits()) { - return ReturnValue(uint64_t{calculateGlobalMemSize(Device)}); - } else { - return ReturnValue(uint64_t{Device->ZeDeviceProperties->maxMemAllocSize}); - } - case UR_DEVICE_INFO_GLOBAL_MEM_SIZE: { - // Support to read physicalSize depends on kernel, - // so fallback into reading totalSize if physicalSize - // is not available. - uint64_t GlobalMemSize = calculateGlobalMemSize(Device); - return ReturnValue(uint64_t{GlobalMemSize}); - } - case UR_DEVICE_INFO_LOCAL_MEM_SIZE: - return ReturnValue( - uint64_t{Device->ZeDeviceComputeProperties->maxSharedLocalMemory}); - case UR_DEVICE_INFO_IMAGE_SUPPORT: - return ReturnValue(Device->ZeDeviceImageProperties->maxImageDims1D > 0); - case UR_DEVICE_INFO_HOST_UNIFIED_MEMORY: - return ReturnValue( - static_cast((Device->ZeDeviceProperties->flags & - ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) != 0)); - case UR_DEVICE_INFO_AVAILABLE: - return ReturnValue(static_cast(ZeDevice ? true : false)); - case UR_DEVICE_INFO_VENDOR: - // TODO: Level-Zero does not return vendor's name at the moment - // only the ID. - return ReturnValue("Intel(R) Corporation"); - case UR_DEVICE_INFO_DRIVER_VERSION: - case UR_DEVICE_INFO_BACKEND_RUNTIME_VERSION: - return ReturnValue(Device->Platform->ZeDriverVersion.c_str()); - case UR_DEVICE_INFO_VERSION: { - // from compute-runtime/shared/source/helpers/hw_ip_version.h - typedef struct { - uint32_t revision : 6; - uint32_t reserved : 8; - uint32_t release : 8; - uint32_t architecture : 10; - } version_components_t; - typedef struct { - union { - uint32_t value; - version_components_t components; - }; - } ipVersion_t; - ipVersion_t IpVersion; - IpVersion.value = Device->ZeDeviceIpVersionExt->ipVersion; - std::stringstream S; - S << IpVersion.components.architecture << "." - << IpVersion.components.release << "." << IpVersion.components.revision; - return ReturnValue(S.str().c_str()); - } - case UR_DEVICE_INFO_PARTITION_MAX_SUB_DEVICES: { - auto Res = Device->Platform->populateDeviceCacheIfNeeded(); - if (Res != UR_RESULT_SUCCESS) { - return Res; - } - return ReturnValue((uint32_t)Device->SubDevices.size()); - } - case UR_DEVICE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Device->RefCount.getCount()}); - case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: { - // SYCL spec says: if this SYCL device cannot be partitioned into at least - // two sub devices then the returned vector must be empty. - auto Res = Device->Platform->populateDeviceCacheIfNeeded(); - if (Res != UR_RESULT_SUCCESS) { - return Res; - } - - uint32_t ZeSubDeviceCount = Device->SubDevices.size(); - if (pSize && ZeSubDeviceCount < 2) { - *pSize = 0; - return UR_RESULT_SUCCESS; - } - bool PartitionedByCSlice = Device->SubDevices[0]->isCCS(); - - auto ReturnHelper = [&](auto... Partitions) { - struct { - ur_device_partition_t Arr[sizeof...(Partitions)]; - } PartitionProperties = {{Partitions...}}; - return ReturnValue(PartitionProperties); - }; - - if (ExposeCSliceInAffinityPartitioning) { - if (PartitionedByCSlice) - return ReturnHelper(UR_DEVICE_PARTITION_BY_CSLICE, - UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); - - else - return ReturnHelper(UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); - } else { - return ReturnHelper(PartitionedByCSlice - ? UR_DEVICE_PARTITION_BY_CSLICE - : UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN); - } - break; - } - case UR_DEVICE_INFO_PARTITION_AFFINITY_DOMAIN: - return ReturnValue(ur_device_affinity_domain_flag_t( - UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA | - UR_DEVICE_AFFINITY_DOMAIN_FLAG_NEXT_PARTITIONABLE)); - case UR_DEVICE_INFO_PARTITION_TYPE: { - // For root-device there is no partitioning to report. - if (Device->SubDeviceCreationProperty == std::nullopt || - !Device->isSubDevice()) { - if (pSize) - *pSize = 0; - return UR_RESULT_SUCCESS; - } - - if (Device->isCCS()) { - ur_device_partition_property_t cslice{}; - cslice.type = UR_DEVICE_PARTITION_BY_CSLICE; - - return ReturnValue(cslice); - } - - return ReturnValue(*Device->SubDeviceCreationProperty); - } - // Everything under here is not supported yet - case UR_DEVICE_INFO_PREFERRED_INTEROP_USER_SYNC: - return ReturnValue(static_cast(true)); - case UR_DEVICE_INFO_PRINTF_BUFFER_SIZE: - return ReturnValue( - size_t{Device->ZeDeviceModuleProperties->printfBufferSize}); - case UR_DEVICE_INFO_PROFILE: - return ReturnValue("FULL_PROFILE"); - case UR_DEVICE_INFO_BUILT_IN_KERNELS: - // TODO: To find out correct value - return ReturnValue(""); - case UR_DEVICE_INFO_LOW_POWER_EVENTS_SUPPORT_EXP: - return ReturnValue(static_cast(true)); - case UR_DEVICE_INFO_REUSABLE_EVENTS_SUPPORT_EXP: -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - return ReturnValue(static_cast(true)); -#else - return ReturnValue(static_cast(false)); -#endif - case UR_DEVICE_INFO_QUEUE_PROPERTIES: - return ReturnValue( - ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE | - UR_QUEUE_FLAG_PROFILING_ENABLE)); - case UR_DEVICE_INFO_EXECUTION_CAPABILITIES: - return ReturnValue(ur_device_exec_capability_flag_t{ - UR_DEVICE_EXEC_CAPABILITY_FLAG_NATIVE_KERNEL}); - case UR_DEVICE_INFO_ENDIAN_LITTLE: - return ReturnValue(static_cast(true)); - case UR_DEVICE_INFO_ERROR_CORRECTION_SUPPORT: - return ReturnValue(static_cast( - Device->ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_ECC)); - case UR_DEVICE_INFO_PROFILING_TIMER_RESOLUTION: - return ReturnValue(static_cast(Device->getTimerResolution())); - case UR_DEVICE_INFO_LOCAL_MEM_TYPE: - return ReturnValue(UR_DEVICE_LOCAL_MEM_TYPE_LOCAL); - case UR_DEVICE_INFO_MAX_CONSTANT_ARGS: - return ReturnValue(uint32_t{64}); - case UR_DEVICE_INFO_MAX_CONSTANT_BUFFER_SIZE: - return ReturnValue( - uint64_t{Device->ZeDeviceImageProperties->maxImageBufferSize}); - case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_TYPE: - return ReturnValue(UR_DEVICE_MEM_CACHE_TYPE_READ_WRITE_CACHE); - case UR_DEVICE_INFO_GLOBAL_MEM_CACHELINE_SIZE: - if (Device->Platform->zeDriverExtensionMap.count( - ZE_CACHELINE_SIZE_EXT_NAME)) { - return ReturnValue(uint32_t{static_cast( - Device->ZeDeviceCacheLinePropertiesExt->cacheLineSize)}); - } else { - return ReturnValue(uint32_t{1}); - } - case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_SIZE: - return ReturnValue(uint64_t{Device->ZeDeviceCacheProperties->cacheSize}); - case UR_DEVICE_INFO_IP_VERSION: - return ReturnValue(uint32_t{Device->ZeDeviceIpVersionExt->ipVersion}); - case UR_DEVICE_INFO_MAX_PARAMETER_SIZE: - return ReturnValue( - size_t{Device->ZeDeviceModuleProperties->maxArgumentsSize}); - case UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN: - // SYCL/OpenCL spec is vague on what this means exactly, but seems to - // be for "alignment requirement (in bits) for sub-buffer offsets." - // An OpenCL implementation returns 8*128, but Level Zero can do just 8, - // meaning unaligned access for values of types larger than 8 bits. - return ReturnValue(uint32_t{8}); - case UR_DEVICE_INFO_MAX_SAMPLERS: - return ReturnValue(uint32_t{Device->ZeDeviceImageProperties->maxSamplers}); - case UR_DEVICE_INFO_MAX_READ_IMAGE_ARGS: - return ReturnValue( - uint32_t{Device->ZeDeviceImageProperties->maxReadImageArgs}); - case UR_DEVICE_INFO_MAX_WRITE_IMAGE_ARGS: - return ReturnValue( - uint32_t{Device->ZeDeviceImageProperties->maxWriteImageArgs}); - case UR_DEVICE_INFO_SINGLE_FP_CONFIG: { - ur_device_fp_capability_flags_t SingleFPValue = 0; - ze_device_fp_flags_t ZeSingleFPCapabilities = - Device->ZeDeviceModuleProperties->fp32flags; - if (ZE_DEVICE_FP_FLAG_DENORM & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; - } - if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; - } - if (ZE_DEVICE_FP_FLAG_FMA & ZeSingleFPCapabilities) { - SingleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; - } - if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeSingleFPCapabilities) { - SingleFPValue |= - UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; - } - return ReturnValue(SingleFPValue); - } - case UR_DEVICE_INFO_HALF_FP_CONFIG: { - ur_device_fp_capability_flags_t HalfFPValue = 0; - ze_device_fp_flags_t ZeHalfFPCapabilities = - Device->ZeDeviceModuleProperties->fp16flags; - if (ZE_DEVICE_FP_FLAG_DENORM & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; - } - if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; - } - if (ZE_DEVICE_FP_FLAG_FMA & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; - } - if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeHalfFPCapabilities) { - HalfFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; - } - return ReturnValue(HalfFPValue); - } - case UR_DEVICE_INFO_DOUBLE_FP_CONFIG: { - ur_device_fp_capability_flags_t DoubleFPValue = 0; - ze_device_fp_flags_t ZeDoubleFPCapabilities = - Device->ZeDeviceModuleProperties->fp64flags; - if (ZE_DEVICE_FP_FLAG_DENORM & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_DENORM; - } - if (ZE_DEVICE_FP_FLAG_INF_NAN & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_INF_NAN; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO; - } - if (ZE_DEVICE_FP_FLAG_ROUND_TO_INF & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF; - } - if (ZE_DEVICE_FP_FLAG_FMA & ZeDoubleFPCapabilities) { - DoubleFPValue |= UR_DEVICE_FP_CAPABILITY_FLAG_FMA; - } - if (ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT & ZeDoubleFPCapabilities) { - DoubleFPValue |= - UR_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT; - } - return ReturnValue(DoubleFPValue); - } - case UR_DEVICE_INFO_IMAGE2D_MAX_WIDTH: - return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims2D}); - case UR_DEVICE_INFO_IMAGE2D_MAX_HEIGHT: - return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims2D}); - case UR_DEVICE_INFO_IMAGE3D_MAX_WIDTH: - return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); - case UR_DEVICE_INFO_IMAGE3D_MAX_HEIGHT: - return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); - case UR_DEVICE_INFO_IMAGE3D_MAX_DEPTH: - return ReturnValue(size_t{Device->ZeDeviceImageProperties->maxImageDims3D}); - case UR_DEVICE_INFO_IMAGE_MAX_BUFFER_SIZE: - return ReturnValue( - size_t{Device->ZeDeviceImageProperties->maxImageBufferSize}); - case UR_DEVICE_INFO_IMAGE_MAX_ARRAY_SIZE: - return ReturnValue( - size_t{Device->ZeDeviceImageProperties->maxImageArraySlices}); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_char); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_CHAR: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_char); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_short); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_SHORT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_short); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_int); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_INT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_int); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG: - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG_LONG: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_long); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG: - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG_LONG: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_long); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_float); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_FLOAT: - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_float); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE: - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_DOUBLE: - // Must return 0 for *vector_width_double* if the device does not have fp64. - if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP64)) - return ReturnValue(uint32_t{0}); - return ReturnValue(uint32_t{1}); - case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF: - // Must return 0 for *vector_width_half* if the device does not have fp16. - if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16)) - return ReturnValue(uint32_t{0}); - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->native_vector_width_half); - case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_HALF: - // Must return 0 for *vector_width_half* if the device does not have fp16. - if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16)) - return ReturnValue(uint32_t{0}); - return ReturnValue( - Device->ZeDeviceVectorWidthPropertiesExt->preferred_vector_width_half); - case UR_DEVICE_INFO_MAX_NUM_SUB_GROUPS: { - // Max_num_sub_Groups = maxTotalGroupSize/min(set of subGroupSizes); - uint32_t MinSubGroupSize = - Device->ZeDeviceComputeProperties->subGroupSizes[0]; - for (uint32_t I = 1; - I < Device->ZeDeviceComputeProperties->numSubGroupSizes; I++) { - if (MinSubGroupSize > Device->ZeDeviceComputeProperties->subGroupSizes[I]) - MinSubGroupSize = Device->ZeDeviceComputeProperties->subGroupSizes[I]; - } - return ReturnValue(Device->ZeDeviceComputeProperties->maxTotalGroupSize / - MinSubGroupSize); - } - case UR_DEVICE_INFO_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS: { - // TODO: Not supported yet. Needs to be updated after support is added. - return ReturnValue(static_cast(false)); - } - case UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL: { - return ReturnValue(Device->ZeDeviceComputeProperties->subGroupSizes, - Device->ZeDeviceComputeProperties->numSubGroupSizes); - } - case UR_DEVICE_INFO_IL_VERSION: { - // Set to a space separated list of IL version strings of the form - // _.. - // "SPIR-V" is a required IL prefix when cl_khr_il_progam extension is - // reported. - uint32_t SpirvVersion = - Device->ZeDeviceModuleProperties->spirvVersionSupported; - uint32_t SpirvVersionMajor = ZE_MAJOR_VERSION(SpirvVersion); - uint32_t SpirvVersionMinor = ZE_MINOR_VERSION(SpirvVersion); - - char SpirvVersionString[50]; - int Len = sprintf(SpirvVersionString, "SPIR-V_%d.%d ", SpirvVersionMajor, - SpirvVersionMinor); - // returned string to contain only len number of characters. - std::string ILVersion(SpirvVersionString, Len); - return ReturnValue(ILVersion.c_str()); - } - case UR_DEVICE_INFO_USM_HOST_SUPPORT: - case UR_DEVICE_INFO_USM_DEVICE_SUPPORT: - case UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT: - case UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT: - case UR_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT: { - auto MapCaps = [](const ze_memory_access_cap_flags_t &ZeCapabilities) { - ur_device_usm_access_capability_flags_t Capabilities = 0; - if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_RW) - Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ACCESS; - if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_ATOMIC) - Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ATOMIC_ACCESS; - if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT) - Capabilities |= UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_CONCURRENT_ACCESS; - if (ZeCapabilities & ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT_ATOMIC) - Capabilities |= - UR_DEVICE_USM_ACCESS_CAPABILITY_FLAG_ATOMIC_CONCURRENT_ACCESS; - return Capabilities; - }; - auto &Props = Device->ZeDeviceMemoryAccessProperties; - switch (ParamName) { - case UR_DEVICE_INFO_USM_HOST_SUPPORT: - return ReturnValue(MapCaps(Props->hostAllocCapabilities)); - case UR_DEVICE_INFO_USM_DEVICE_SUPPORT: - return ReturnValue(MapCaps(Props->deviceAllocCapabilities)); - case UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT: - return ReturnValue(MapCaps(Props->sharedSingleDeviceAllocCapabilities)); - case UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT: - return ReturnValue(MapCaps(Props->sharedCrossDeviceAllocCapabilities)); - case UR_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT: - return ReturnValue(MapCaps(Props->sharedSystemAllocCapabilities)); - default: - die("urDeviceGetInfo: unexpected ParamName."); - } - } - - // intel extensions for GPU information - case UR_DEVICE_INFO_DEVICE_ID: - return ReturnValue(uint32_t{Device->ZeDeviceProperties->deviceId}); - case UR_DEVICE_INFO_PCI_ADDRESS: { - ze_pci_address_ext_t PciAddr{}; - ZeStruct ZeDevicePciProperties; - ZeDevicePciProperties.address = PciAddr; - ZE2UR_CALL(zeDevicePciGetPropertiesExt, (ZeDevice, &ZeDevicePciProperties)); - constexpr size_t AddressBufferSize = 13; - char AddressBuffer[AddressBufferSize]; - std::snprintf(AddressBuffer, AddressBufferSize, "%04x:%02x:%02x.%01x", - ZeDevicePciProperties.address.domain, - ZeDevicePciProperties.address.bus, - ZeDevicePciProperties.address.device, - ZeDevicePciProperties.address.function); - return ReturnValue(AddressBuffer); - } - - case UR_DEVICE_INFO_GLOBAL_MEM_FREE: { - if (!ParamValue && pSize) { - if (supportsDeviceUsableMemSizeExtension(Device->Platform)) { - return ReturnValue(uint64_t{0}); - } - - auto [ZesDevice, ZesDeviceData, Result] = getZesDeviceData(Device); - (void)ZesDevice; - (void)ZesDeviceData; - if (Result != UR_RESULT_SUCCESS) { - return Result; - } - - return ReturnValue(uint64_t{0}); - } - - // Calculate the global memory size as the max limit that can be reported as - // "free" memory for the user to allocate. - uint64_t GlobalMemSize = calculateGlobalMemSize(Device); - // Only report device memory which zeMemAllocDevice can allocate from. - // Currently this is only the one enumerated with ordinal 0. - uint64_t FreeMemory = 0; - uint32_t MemCount = 0; - - if (auto CoreUsableMemSize = getDeviceUsableMemSizeFromCore(Device)) { - return ReturnValue(std::min(GlobalMemSize, *CoreUsableMemSize)); - } - - auto [ZesDevice, ZesDeviceData, Result] = getZesDeviceData(Device); - if (Result != UR_RESULT_SUCCESS) - return Result; - - ZE2UR_CALL(zesDeviceEnumMemoryModules, (ZesDevice, &MemCount, nullptr)); - if (MemCount != 0) { - std::vector ZesMemHandles(MemCount); - ZE2UR_CALL(zesDeviceEnumMemoryModules, - (ZesDevice, &MemCount, ZesMemHandles.data())); - for (auto &ZesMemHandle : ZesMemHandles) { - ZesStruct ZesMemProperties; - ZE2UR_CALL(zesMemoryGetProperties, (ZesMemHandle, &ZesMemProperties)); - // For root-device report memory from all memory modules since that - // is what totally available in the default implicit scaling mode. - // For sub-devices only report memory local to them. - if (ZesDeviceData.SubDeviceId == ZesMemProperties.subdeviceId || - !ZesDeviceData.SubDevice) { - ZesStruct ZesMemState; - ZE2UR_CALL(zesMemoryGetState, (ZesMemHandle, &ZesMemState)); - FreeMemory += ZesMemState.free; - } - } - } - if (MemCount > 0) { - return ReturnValue(std::min(GlobalMemSize, FreeMemory)); - } else { - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - } - } - case UR_DEVICE_INFO_MEMORY_CLOCK_RATE: { - // If there are not any memory modules then return 0. - if (Device->ZeDeviceMemoryProperties->first.empty()) - return ReturnValue(uint32_t{0}); - - // If there are multiple memory modules on the device then we have to report - // the value of the slowest memory. - auto Comp = [](const ze_device_memory_properties_t &A, - const ze_device_memory_properties_t &B) -> bool { - return A.maxClockRate < B.maxClockRate; - }; - auto MinIt = - std::min_element(Device->ZeDeviceMemoryProperties->first.begin(), - Device->ZeDeviceMemoryProperties->first.end(), Comp); - return ReturnValue(uint32_t{MinIt->maxClockRate}); - } - case UR_DEVICE_INFO_MEMORY_BUS_WIDTH: { - // If there are not any memory modules then return 0. - if (Device->ZeDeviceMemoryProperties->first.empty()) - return ReturnValue(uint32_t{0}); - - // If there are multiple memory modules on the device then we have to report - // the value of the slowest memory. - auto Comp = [](const ze_device_memory_properties_t &A, - const ze_device_memory_properties_t &B) -> bool { - return A.maxBusWidth < B.maxBusWidth; - }; - auto MinIt = - std::min_element(Device->ZeDeviceMemoryProperties->first.begin(), - Device->ZeDeviceMemoryProperties->first.end(), Comp); - return ReturnValue(uint32_t{MinIt->maxBusWidth}); - } - case UR_DEVICE_INFO_MAX_COMPUTE_QUEUE_INDICES: { - if (Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] - .ZeIndex >= 0) - // Sub-sub-device represents a particular compute index already. - return ReturnValue(int32_t{1}); - - auto ZeDeviceNumIndices = - Device->QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] - .ZeProperties.numQueues; - return ReturnValue(int32_t(ZeDeviceNumIndices)); - } break; - case UR_DEVICE_INFO_GPU_EU_COUNT: { - if (Device->Platform->ZeDriverEuCountExtensionFound) { - ze_device_properties_t DeviceProp = {}; - DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - ze_eu_count_ext_t EuCountDesc = {}; - EuCountDesc.stype = ZE_STRUCTURE_TYPE_EU_COUNT_EXT; - DeviceProp.pNext = (void *)&EuCountDesc; - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); - if (EuCountDesc.numTotalEUs > 0) { - return ReturnValue(uint32_t{EuCountDesc.numTotalEUs}); - } - } - - uint32_t count = Device->ZeDeviceProperties->numEUsPerSubslice * - Device->ZeDeviceProperties->numSubslicesPerSlice * - Device->ZeDeviceProperties->numSlices; - return ReturnValue(uint32_t{count}); - } - case UR_DEVICE_INFO_GPU_EU_SLICES: { - return ReturnValue(uint32_t{Device->ZeDeviceProperties->numSlices}); - } - case UR_DEVICE_INFO_GPU_EU_SIMD_WIDTH: - return ReturnValue( - uint32_t{Device->ZeDeviceProperties->physicalEUSimdWidth}); - case UR_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE: - return ReturnValue( - uint32_t{Device->ZeDeviceProperties->numSubslicesPerSlice}); - case UR_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE: - return ReturnValue(uint32_t{Device->ZeDeviceProperties->numEUsPerSubslice}); - case UR_DEVICE_INFO_GPU_HW_THREADS_PER_EU: - return ReturnValue(uint32_t{Device->ZeDeviceProperties->numThreadsPerEU}); - case UR_DEVICE_INFO_ATOMIC_MEMORY_SCOPE_CAPABILITIES: { - // There are no explicit restrictions in L0 programming guide, so assume all - // are supported - ur_memory_scope_capability_flags_t result = - UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_ITEM | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_SUB_GROUP | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_DEVICE | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_SYSTEM; - - return ReturnValue(result); - } - case UR_DEVICE_INFO_ATOMIC_FENCE_ORDER_CAPABILITIES: { - // There are no explicit restrictions in L0 programming guide, so assume all - // are supported - ur_memory_order_capability_flags_t result = - UR_MEMORY_ORDER_CAPABILITY_FLAG_RELAXED | - UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQUIRE | - UR_MEMORY_ORDER_CAPABILITY_FLAG_RELEASE | - UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQ_REL | - UR_MEMORY_ORDER_CAPABILITY_FLAG_SEQ_CST; - - return ReturnValue(result); - } - case UR_DEVICE_INFO_ATOMIC_FENCE_SCOPE_CAPABILITIES: { - // There are no explicit restrictions in L0 programming guide, so assume all - // are supported - ur_memory_scope_capability_flags_t result = - UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_ITEM | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_SUB_GROUP | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_DEVICE | - UR_MEMORY_SCOPE_CAPABILITY_FLAG_SYSTEM; - - return ReturnValue(result); - } - - case UR_DEVICE_INFO_ATOMIC_MEMORY_ORDER_CAPABILITIES: { - ur_memory_order_capability_flags_t capabilities = - UR_MEMORY_ORDER_CAPABILITY_FLAG_RELAXED | - UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQUIRE | - UR_MEMORY_ORDER_CAPABILITY_FLAG_RELEASE | - UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQ_REL | - UR_MEMORY_ORDER_CAPABILITY_FLAG_SEQ_CST; - return ReturnValue(capabilities); - } - case UR_DEVICE_INFO_MEM_CHANNEL_SUPPORT: - return ReturnValue(ur_bool_t{false}); - case UR_DEVICE_INFO_IMAGE_SRGB: - return ReturnValue(ur_bool_t{false}); - - case UR_DEVICE_INFO_QUEUE_ON_DEVICE_PROPERTIES: - case UR_DEVICE_INFO_QUEUE_ON_HOST_PROPERTIES: { - ur_queue_flags_t queue_flags = 0; - return ReturnValue(queue_flags); - } - case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS: { - return ReturnValue(static_cast( - 0)); //__read_write attribute currently undefinde in opencl - } - case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: { - return ReturnValue(static_cast(true)); - } - case UR_DEVICE_INFO_TIMESTAMP_RECORDING_SUPPORT_EXP: { - return ReturnValue(static_cast(true)); - } - case UR_DEVICE_INFO_PER_EVENT_PROFILING_SUPPORT_EXP: { -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - return ReturnValue(true); -#else - return ReturnValue(false); -#endif - } - case UR_DEVICE_INFO_ENQUEUE_NATIVE_COMMAND_SUPPORT_EXP: { - // L0 doesn't support enqueueing native work through the urNativeEnqueueExp - return ReturnValue(static_cast(false)); - } - - case UR_DEVICE_INFO_ESIMD_SUPPORT: { - // ESIMD is only supported by Intel GPUs. - ur_bool_t result = Device->ZeDeviceProperties->type == ZE_DEVICE_TYPE_GPU && - Device->ZeDeviceProperties->vendorId == 0x8086; - return ReturnValue(result); - } - - case UR_DEVICE_INFO_COMPONENT_DEVICES: { - ze_device_handle_t DevHandle = Device->ZeDevice; - uint32_t SubDeviceCount = 0; - // First call to get SubDeviceCount. - ZE2UR_CALL(zeDeviceGetSubDevices, (DevHandle, &SubDeviceCount, nullptr)); - if (SubDeviceCount == 0) - return ReturnValue(std::nullopt); - - std::vector SubDevs(SubDeviceCount); - // Second call to get the actual list of devices. - ZE2UR_CALL(zeDeviceGetSubDevices, - (DevHandle, &SubDeviceCount, SubDevs.data())); - - size_t SubDeviceCount_s{SubDeviceCount}; - auto ResSize = - std::min(SubDeviceCount_s, propSize / sizeof(ur_device_handle_t)); - std::vector Res; - for (const auto &d : SubDevs) { - // We can only reach this code if ZE_FLAT_DEVICE_HIERARCHY != FLAT, - // because in flat mode we directly get tiles, and those don't have any - // further divisions, so zeDeviceGetSubDevices always will return an empty - // list. Thus, there's only two options left: (a) composite mode, and (b) - // combined mode. In (b), zeDeviceGet returns tiles as devices, and those - // are presented as root devices (i.e. isSubDevice() returns false). In - // contrast, in (a), zeDeviceGet returns cards as devices, so tiles are - // not root devices (i.e. isSubDevice() returns true). Since we only reach - // this code if there are tiles returned by zeDeviceGetSubDevices, we - // can know if we are in (a) or (b) by checking if a tile is root device - // or not. - ur_device_handle_t URDev = Device->Platform->getDeviceFromNativeHandle(d); - if (URDev->isSubDevice()) { - // We are in COMPOSITE mode, return an empty list. - if (pSize) { - *pSize = 0; - } - return UR_RESULT_SUCCESS; - } +#include "common/device.hpp" +#include "common/platform.hpp" +#include "ur_interface_loader.hpp" - Res.push_back(URDev); - } - if (pSize) - *pSize = SubDeviceCount * sizeof(ur_device_handle_t); - if (ParamValue) { - return ReturnValue(Res.data(), ResSize); - } - return UR_RESULT_SUCCESS; - } - case UR_DEVICE_INFO_COMPOSITE_DEVICE: { - ur_device_handle_t UrRootDev = nullptr; - ze_device_handle_t DevHandle = Device->ZeDevice; - ze_device_handle_t RootDev; - // Query Root Device. - auto errc = ZE_CALL_NOCHECK(zeDeviceGetRootDevice, (DevHandle, &RootDev)); - UrRootDev = Device->Platform->getDeviceFromNativeHandle(RootDev); - if (errc != ZE_RESULT_SUCCESS && - errc != ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) - return ze2urResult(errc); - return ReturnValue(UrRootDev); - } - case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { - bool Bfloat16ConversionSupport = - (Device->Platform->zeDriverExtensionMap.count( - ZE_BFLOAT16_CONVERSIONS_EXT_NAME)) || - ((Device->ZeDeviceProperties->deviceId & 0xfff) == 0x201 || - (Device->ZeDeviceProperties->deviceId & 0xff0) == 0xbd0); - return ReturnValue(Bfloat16ConversionSupport); - } - case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP: { - const auto ZeMutableCommandFlags = - Device->ZeDeviceMutableCmdListsProperties->mutableCommandFlags; +namespace ur::level_zero::v1 { - auto supportsFlags = [&](ze_mutable_command_exp_flags_t RequiredFlags) { - return (ZeMutableCommandFlags & RequiredFlags) == RequiredFlags; - }; +ur_result_t urDeviceGetInfo(::ur_device_handle_t hDeviceOpque, + ::ur_device_info_t propName, size_t propSize, + void *propValue, size_t *propSizeRet) { + UrReturnHelper ReturnValue(propSize, propValue, propSizeRet); - ur_device_command_buffer_update_capability_flags_t UpdateCapabilities = 0; - if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_ARGUMENTS)) { - UpdateCapabilities |= - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; - } - /* These capabilities are bundled together because, when the user updates - * the global work-size, the implementation might have to generate a new - * local work-size. This would require both mutable command flags to be set - * even though only the global work-size was explicitly updated. */ - if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_COUNT | - ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_SIZE)) { - UpdateCapabilities |= - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE | - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; - } - if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_GLOBAL_OFFSET)) { - UpdateCapabilities |= - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET; - } - if (supportsFlags(ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_INSTRUCTION)) { - UpdateCapabilities |= - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE; - } - return ReturnValue(UpdateCapabilities); - } - case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP: - return ReturnValue(false); - case UR_DEVICE_INFO_COMMAND_BUFFER_SUBGRAPH_SUPPORT_EXP: - return ReturnValue(false); - case UR_DEVICE_INFO_BINDLESS_IMAGES_SUPPORT_EXP: { - return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported); - } - case UR_DEVICE_INFO_BINDLESS_IMAGES_SHARED_USM_SUPPORT_EXP: { - // On L0 bindless images can not be backed by shared (managed) USM. - return ReturnValue(false); - } - case UR_DEVICE_INFO_BINDLESS_IMAGES_1D_USM_SUPPORT_EXP: { - return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported && - Device->ZeDeviceImageProperties->maxImageDims1D > 0); - } - case UR_DEVICE_INFO_BINDLESS_IMAGES_2D_USM_SUPPORT_EXP: { - return ReturnValue(Device->Platform->ZeBindlessImagesExtensionSupported && - Device->ZeDeviceImageProperties->maxImageDims2D > 0); - } - case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_WIDTH_EXP: { - return ReturnValue( - Device->ZeDevicePitchedAllocProperties->AllocProps.maxImageLinearWidth); - } - case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_HEIGHT_EXP: { - return ReturnValue(Device->ZeDevicePitchedAllocProperties->AllocProps - .maxImageLinearHeight); - } - case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP: { - return ReturnValue(static_cast( - Device->ZeDevicePitchedAllocProperties->PitchInfo.pitchAlign)); - } - case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_PITCH_EXP: { - return ReturnValue( - Device->ZeDevicePitchedAllocProperties->PitchInfo.maxSupportedPitch); - } - case UR_DEVICE_INFO_MIPMAP_SUPPORT_EXP: { - // L0 does not support mipmaps. - return ReturnValue(false); - } - case UR_DEVICE_INFO_MIPMAP_ANISOTROPY_SUPPORT_EXP: { - // L0 does not support anisotropic filtering. - return ReturnValue(false); - } - case UR_DEVICE_INFO_MIPMAP_MAX_ANISOTROPY_EXP: - UR_LOG(ERR, "Unsupported ParamName in urGetDeviceInfo"); - UR_LOG(ERR, "ParamName=%{}(0x{})", ParamName, logger::toHex(ParamName)); - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - case UR_DEVICE_INFO_MIPMAP_LEVEL_REFERENCE_SUPPORT_EXP: { - // L0 does not support creation of images from individual mipmap levels. - return ReturnValue(false); - } - case UR_DEVICE_INFO_EXTERNAL_MEMORY_IMPORT_SUPPORT_EXP: { - // L0 supports importing external memory. - return ReturnValue(true); - } - case UR_DEVICE_INFO_USM_HOST_ALLOC_REGISTER_SUPPORT_EXP: { -#if defined(UR_ADAPTER_LEVEL_ZERO_V2) - // Registering existing host memory as a USM host allocation relies on the - // external system memory mapping extension being supported by the driver. - return ReturnValue( - Device->Platform->ZeExternalMemoryMappingExtensionSupported); -#else - return ReturnValue(false); -#endif - } - case UR_DEVICE_INFO_EXTERNAL_SEMAPHORE_IMPORT_SUPPORT_EXP: { - return ReturnValue(Device->Platform->ZeExternalSemaphoreExt.Supported); - } - case UR_DEVICE_INFO_CUBEMAP_SUPPORT_EXP: { - // L0 does not support cubemaps. - return ReturnValue(false); - } - case UR_DEVICE_INFO_CUBEMAP_SEAMLESS_FILTERING_SUPPORT_EXP: { - // L0 does not support cubemap seamless filtering. - return ReturnValue(false); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_1D_USM_SUPPORT_EXP: { - // L0 does support fetching 1D USM sampled image data. - return ReturnValue(true); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_1D_SUPPORT_EXP: { - // L0 does support fetching 1D non-USM sampled image data. - return ReturnValue(true); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_2D_USM_SUPPORT_EXP: { - // L0 does support fetching 2D USM sampled image data. - return ReturnValue(true); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_2D_SUPPORT_EXP: { - // L0 does support fetching 2D non-USM sampled image data. - return ReturnValue(true); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLED_IMAGE_FETCH_3D_SUPPORT_EXP: { - // L0 does support fetching 3D non-USM sampled image data. - return ReturnValue(true); - } - case UR_DEVICE_INFO_IMAGE_ARRAY_SUPPORT_EXP: { - // L0 does support image arrays - return ReturnValue(true); - } - case UR_DEVICE_INFO_BINDLESS_UNIQUE_ADDRESSING_PER_DIM_SUPPORT_EXP: { - // L0 does not support unique addressing per dimension - return ReturnValue(false); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLE_1D_USM_SUPPORT_EXP: { - // L0 does not support sampling 1D USM sampled image data. + switch (propName) { + case UR_DEVICE_INFO_GRAPH_RECORD_AND_REPLAY_SUPPORT_EXP: return ReturnValue(false); - } - case UR_DEVICE_INFO_BINDLESS_SAMPLE_2D_USM_SUPPORT_EXP: { - // L0 does not support sampling 2D USM sampled image data. + case UR_DEVICE_INFO_ENQUEUE_HOST_TASK_SUPPORT_EXP: return ReturnValue(false); - } - case UR_DEVICE_INFO_BINDLESS_IMAGES_GATHER_SUPPORT_EXP: { - // L0 doesn't support sampled image gather. + case UR_DEVICE_INFO_REUSABLE_EVENTS_SUPPORT_EXP: return ReturnValue(static_cast(false)); - } - case UR_DEVICE_INFO_PROGRAM_SET_SPECIALIZATION_CONSTANTS: - return ReturnValue(true); - case UR_DEVICE_INFO_KERNEL_SET_SPECIALIZATION_CONSTANTS: + case UR_DEVICE_INFO_PER_EVENT_PROFILING_SUPPORT_EXP: return ReturnValue(false); - case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT: - return ReturnValue(true); - case UR_DEVICE_INFO_USM_POOL_SUPPORT: - return ReturnValue(true); - case UR_DEVICE_INFO_2D_BLOCK_ARRAY_CAPABILITIES_EXP: { -#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME - const auto ZeDeviceBlockArrayFlags = - Device->ZeDeviceBlockArrayProperties->flags; - - auto supportsFlags = - [&](ze_intel_device_block_array_exp_flags_t RequiredFlags) { - return (ZeDeviceBlockArrayFlags & RequiredFlags) == RequiredFlags; - }; - - ur_exp_device_2d_block_array_capability_flags_t BlockArrayCapabilities = 0; - if (supportsFlags(ZE_INTEL_DEVICE_EXP_FLAG_2D_BLOCK_LOAD)) { - BlockArrayCapabilities |= - UR_EXP_DEVICE_2D_BLOCK_ARRAY_CAPABILITY_FLAG_LOAD; - } - if (supportsFlags(ZE_INTEL_DEVICE_EXP_FLAG_2D_BLOCK_STORE)) { - BlockArrayCapabilities |= - UR_EXP_DEVICE_2D_BLOCK_ARRAY_CAPABILITY_FLAG_STORE; - } - return ReturnValue(BlockArrayCapabilities); -#else - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; -#endif - } - case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: -#ifdef _WIN32 - // TODO: Remove when IPC memory works in UMF on Windows. + case UR_DEVICE_INFO_USM_HOST_ALLOC_REGISTER_SUPPORT_EXP: return ReturnValue(false); -#else - return ReturnValue(true); -#endif case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: -#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__) - return ReturnValue(true); -#else - return ReturnValue(false); -#endif - case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP: { -#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__) - constexpr uint32_t MinDriverBuild = 38646; - ZeStruct ZeDriverProperties; - ZE2UR_CALL(zeDriverGetProperties, - (Device->Platform->ZeDriver, &ZeDriverProperties)); - const uint32_t DriverBuild = ZeDriverProperties.driverVersion & 0xFFFF; - return ReturnValue(static_cast(Device->isBMGOrNewer() && - DriverBuild >= MinDriverBuild)); -#else - return ReturnValue(false); -#endif - } - case UR_DEVICE_INFO_ASYNC_BARRIER: return ReturnValue(false); - case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORT: - return ReturnValue(false); - case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_USE_NATIVE_ASSERT: - return ReturnValue(true); - case UR_DEVICE_INFO_USM_P2P_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_MULTI_DEVICE_COMPILE_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_DEVICE_WAIT_SUPPORT_EXP: { - auto Supported = Device->Platform->ZeDeviceSynchronizeSupported; - return ReturnValue(Supported); - } - case UR_DEVICE_INFO_DYNAMIC_LINK_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_CURRENT_CLOCK_THROTTLE_REASONS: { - ur_device_throttle_reasons_flags_t ThrottleReasons = 0; - if (!ParamValue) { - // If ParamValue is nullptr, then we are only interested in the size of - // the value. - return ReturnValue(ThrottleReasons); - } - [[maybe_unused]] auto [ZesDevice, Ignored, Result] = - getZesDeviceData(Device); - if (Result != UR_RESULT_SUCCESS) - return Result; - uint32_t FreqCount = 0; - ZE2UR_CALL(zesDeviceEnumFrequencyDomains, (ZesDevice, &FreqCount, nullptr)); - if (FreqCount != 0) { - std::vector ZesFreqHandles(FreqCount); - ZE2UR_CALL(zesDeviceEnumFrequencyDomains, - (ZesDevice, &FreqCount, ZesFreqHandles.data())); - for (auto &ZesFreqHandle : ZesFreqHandles) { - ZesStruct FreqProperties; - ZE2UR_CALL(zesFrequencyGetProperties, (ZesFreqHandle, &FreqProperties)); - if (FreqProperties.type != ZES_FREQ_DOMAIN_GPU) { - continue; - } - zes_freq_state_t State; - zesFrequencyGetState(ZesFreqHandle, &State); - constexpr zes_freq_throttle_reason_flags_t ZeThrottleFlags[] = { - ZES_FREQ_THROTTLE_REASON_FLAG_AVE_PWR_CAP, - ZES_FREQ_THROTTLE_REASON_FLAG_CURRENT_LIMIT, - ZES_FREQ_THROTTLE_REASON_FLAG_THERMAL_LIMIT, - ZES_FREQ_THROTTLE_REASON_FLAG_PSU_ALERT, - ZES_FREQ_THROTTLE_REASON_FLAG_SW_RANGE, - ZES_FREQ_THROTTLE_REASON_FLAG_HW_RANGE}; - - constexpr ur_device_throttle_reasons_flags_t UrThrottleFlags[] = { - UR_DEVICE_THROTTLE_REASONS_FLAG_POWER_CAP, - UR_DEVICE_THROTTLE_REASONS_FLAG_CURRENT_LIMIT, - UR_DEVICE_THROTTLE_REASONS_FLAG_THERMAL_LIMIT, - UR_DEVICE_THROTTLE_REASONS_FLAG_PSU_ALERT, - UR_DEVICE_THROTTLE_REASONS_FLAG_SW_RANGE, - UR_DEVICE_THROTTLE_REASONS_FLAG_HW_RANGE}; - - for (size_t i = 0; - i < sizeof(ZeThrottleFlags) / sizeof(ZeThrottleFlags[0]); ++i) { - if (State.throttleReasons & ZeThrottleFlags[i]) { - ThrottleReasons |= UrThrottleFlags[i]; - State.throttleReasons &= ~ZeThrottleFlags[i]; - } - } - - if (State.throttleReasons) { - ThrottleReasons |= UR_DEVICE_THROTTLE_REASONS_FLAG_OTHER; - } - } - } - return ReturnValue(ThrottleReasons); - } - case UR_DEVICE_INFO_FAN_SPEED: { - [[maybe_unused]] auto [ZesDevice, Ignored, Result] = - getZesDeviceData(Device); - if (Result != UR_RESULT_SUCCESS) - return Result; - - uint32_t FanCount = 0; - ZE2UR_CALL(zesDeviceEnumFans, (ZesDevice, &FanCount, nullptr)); - // If there are no fans, then report speed query as unsupported. - if (FanCount == 0) - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - - if (!ParamValue) { - // If ParamValue is nullptr, then we are only interested in the size of - // the value. - return ReturnValue(int32_t{0}); - } - - std::vector ZeFanHandles(FanCount); - ZE2UR_CALL(zesDeviceEnumFans, (ZesDevice, &FanCount, ZeFanHandles.data())); - int32_t Speed = -1; - for (auto Fan : ZeFanHandles) { - int32_t CurSpeed; - ZE2UR_CALL(zesFanGetState, (Fan, ZES_FAN_SPEED_UNITS_PERCENT, &CurSpeed)); - Speed = std::max(Speed, CurSpeed); - } - return ReturnValue(Speed); - } - case UR_DEVICE_INFO_MIN_POWER_LIMIT: - case UR_DEVICE_INFO_MAX_POWER_LIMIT: { - [[maybe_unused]] auto [ZesDevice, Ignored, Result] = - getZesDeviceData(Device); - if (Result != UR_RESULT_SUCCESS) - return Result; - - zes_pwr_handle_t ZesPwrHandle = nullptr; - auto DomainResult = zesDeviceGetCardPowerDomain(ZesDevice, &ZesPwrHandle); - if (DomainResult == ZE_RESULT_ERROR_UNSUPPORTED_FEATURE) { - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - } else if (DomainResult != ZE_RESULT_SUCCESS) { - return ze2urResult(DomainResult); - } - - if (!ParamValue) { - // If ParamValue is nullptr, then we are only interested in the size of - // the value. - // Do this after calling getCardPowerDomain so that UNSUPPORTED is - // returned correctly if required - return ReturnValue(int32_t{0}); - } - - ZesStruct PowerProperties; - ZE2UR_CALL(zesPowerGetProperties, (ZesPwrHandle, &PowerProperties)); - - if (ParamName == UR_DEVICE_INFO_MIN_POWER_LIMIT) { - return ReturnValue(int32_t{PowerProperties.minLimit}); - } else { - return ReturnValue(int32_t{PowerProperties.maxLimit}); - } - } - case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: - return ReturnValue(UR_KERNEL_LAUNCH_PROPERTIES_FLAG_COOPERATIVE); - case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: - return ReturnValue(true); - case UR_DEVICE_INFO_LUID: { - // LUID is only available on Windows. - // Intel extension for device LUID. This returns the LUID as - // std::array. For details about this extension, - // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. - if (Device->Platform->ZeLUIDSupported) { - ze_device_properties_t DeviceProp = {}; - DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - ze_device_luid_ext_properties_t LuidDesc = {}; - LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; - DeviceProp.pNext = (void *)&LuidDesc; - - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); - - const auto &LUID = LuidDesc.luid.id; - return ReturnValue(LUID, sizeof(LUID)); - } else { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - } - case UR_DEVICE_INFO_NODE_MASK: { - // Device node mask is only available on Windows. - // Intel extension for device node mask. This returns the node mask as - // uint32_t. For details about this extension, - // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. - - // Node mask is provided through the L0 LUID extension so support for this - // extension must be checked. - if (Device->Platform->ZeLUIDSupported) { - ze_device_properties_t DeviceProp = {}; - DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - ze_device_luid_ext_properties_t LuidDesc = {}; - LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; - DeviceProp.pNext = (void *)&LuidDesc; - - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); - - return ReturnValue(LuidDesc.nodeMask); - } else { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - } - case UR_DEVICE_INFO_CLOCK_SUB_GROUP_SUPPORT_EXP: { - // IGC supports this since v.2.16.0 - return ReturnValue( - Device->Platform->isDriverVersionNewerOrSimilar(1, 6, 34666)); - } - case UR_DEVICE_INFO_CLOCK_WORK_GROUP_SUPPORT_EXP: - case UR_DEVICE_INFO_CLOCK_DEVICE_SUPPORT_EXP: - // Currently GPUs only support sub-group clock. - return ReturnValue(false); - case UR_DEVICE_INFO_IS_INTEGRATED_GPU: - return ReturnValue(static_cast(Device->isIntegrated() != 0)); - case UR_DEVICE_INFO_GRAPH_RECORD_AND_REPLAY_SUPPORT_EXP: -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - { - if (!Device->Platform->ZeGraphExt.Supported) { - return ReturnValue(false); - } - - // The experimental variant of the extension reports its capabilities - // through a structure with a different type value; an older driver would - // not recognize the stable one and would leave graphFlags unset. The - // structure layout (stype, pNext, graphFlags) is identical between the two - // variants, so the stable type can be reused with the experimental value. - constexpr ze_structure_type_t ZeStructTypeRecordReplayGraphExpProperties = - static_cast(0x00030029); - ze_record_replay_graph_ext_properties_t GraphProperties{}; - GraphProperties.stype = - Device->Platform->ZeGraphExt.UsesLegacyExperimentalApi - ? ZeStructTypeRecordReplayGraphExpProperties - : ZE_STRUCTURE_TYPE_RECORD_REPLAY_GRAPH_EXT_PROPERTIES; - GraphProperties.pNext = nullptr; - ZeStruct DeviceProperties; - DeviceProperties.pNext = &GraphProperties; - ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProperties)); - - constexpr ze_record_replay_graph_ext_flags_t GraphModeMask = - ZE_RECORD_REPLAY_GRAPH_EXT_FLAG_IMMUTABLE_GRAPH | - ZE_RECORD_REPLAY_GRAPH_EXT_FLAG_MUTABLE_GRAPH; - return ReturnValue(static_cast( - (GraphProperties.graphFlags & GraphModeMask) != 0)); - } -#else - return ReturnValue(false); -#endif - case UR_DEVICE_INFO_ENQUEUE_HOST_TASK_SUPPORT_EXP: -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - return ReturnValue(Device->Platform->ZeHostTaskExt.Supported); -#else + case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP: return ReturnValue(false); -#endif - case UR_DEVICE_INFO_XE_STACK_COUNT: - return ReturnValue(uint32_t{Device->ZeXEDeviceProperties->numXeStacks}); - case UR_DEVICE_INFO_XE_REGIONS_PER_STACK: - return ReturnValue( - uint32_t{Device->ZeXEDeviceProperties->numXeRegionsPerStack}); - case UR_DEVICE_INFO_XE_CLUSTERS_PER_REGION: - return ReturnValue( - uint32_t{Device->ZeXEDeviceProperties->numXeClustersPerRegion}); - case UR_DEVICE_INFO_XE_CORES_PER_CLUSTER: - return ReturnValue( - uint32_t{Device->ZeXEDeviceProperties->numXeCorePerCluster}); - case UR_DEVICE_INFO_EUS_PER_XE_CORE: - return ReturnValue( - uint32_t{Device->ZeXEDeviceProperties->numExecutionEnginesPerXeCore}); - case UR_DEVICE_INFO_MAX_LANES_PER_HW_THREAD: - return ReturnValue( - uint32_t{Device->ZeXEDeviceProperties->maxNumLanesPerHwThread}); default: - UR_LOG(ERR, "Unsupported ParamName in urGetDeviceInfo"); - UR_LOG(ERR, "ParamNameParamName={}(0x{})", ParamName, - logger::toHex(ParamName)); - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + return ur::level_zero::urDeviceGetInfo(hDeviceOpque, propName, propSize, + propValue, propSizeRet); } - - return UR_RESULT_SUCCESS; -} - -bool CopyEngineRequested(const ur_device_handle_t &Device) { - int LowerCopyQueueIndex = getRangeOfAllowedCopyEngines(Device).first; - int UpperCopyQueueIndex = getRangeOfAllowedCopyEngines(Device).second; - return ((LowerCopyQueueIndex != -1) || (UpperCopyQueueIndex != -1)); -} - -ur_result_t urDevicePartition( - /// [in] handle of the device to partition. - ur_device_handle_t Device, - /// [in] Device partition properties. - const ur_device_partition_properties_t *Properties, - /// [in] the number of sub-devices. - uint32_t NumDevices, - /// [out][optional][range(0, NumDevices)] array of handle of devices. If - /// NumDevices is less than the number of sub-devices available, then - /// the function shall only retrieve that number of sub-devices. - ur_device_handle_t *OutDevices, - /// [out][optional] pointer to the number of sub-devices the device can be - /// partitioned into according to the partitioning property. - uint32_t *NumDevicesRet) { - // Other partitioning ways are not supported by Level Zero - UR_ASSERT(Properties->PropCount == 1, UR_RESULT_ERROR_INVALID_VALUE); - if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN) { - if ((Properties->pProperties->value.affinity_domain != - UR_DEVICE_AFFINITY_DOMAIN_FLAG_NEXT_PARTITIONABLE && - Properties->pProperties->value.affinity_domain != - UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA)) { - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_CSLICE) { - if (Properties->pProperties->value.affinity_domain != 0) { - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else { - return UR_RESULT_ERROR_INVALID_VALUE; - } - - // Devices cache is normally created in piDevicesGet but still make - // sure that cache is populated. - // - auto Res = Device->Platform->populateDeviceCacheIfNeeded(); - if (Res != UR_RESULT_SUCCESS) { - return Res; - } - - auto EffectiveNumDevices = [&]() -> decltype(Device->SubDevices.size()) { - if (Device->SubDevices.size() == 0) - return 0; - - // Sub-Sub-Devices are partitioned by CSlices, not by affinity domain. - // However, if - // UR_L0_EXPOSE_CSLICE_IN_AFFINITY_PARTITIONING overrides that - // still expose CSlices in partitioning by affinity domain for compatibility - // reasons. - if (Properties->pProperties->type == - UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN && - !ExposeCSliceInAffinityPartitioning) { - if (Device->isSubDevice()) { - return 0; - } - } - if (Properties->pProperties->type == UR_DEVICE_PARTITION_BY_CSLICE) { - // Not a CSlice-based partitioning. - if (!Device->SubDevices[0]->isCCS()) { - return 0; - } - } - - return Device->SubDevices.size(); - }(); - - // TODO: Consider support for partitioning to <= total sub-devices. - // Currently supported partitioning (by affinity domain/numa) would always - // partition to all sub-devices. - // - if (NumDevices != 0) - UR_ASSERT(NumDevices == EffectiveNumDevices, UR_RESULT_ERROR_INVALID_VALUE); - - for (uint32_t I = 0; I < NumDevices; I++) { - auto prop = Properties->pProperties[0]; - if (prop.type == UR_DEVICE_PARTITION_BY_AFFINITY_DOMAIN) { - // In case the value is NEXT_PARTITIONABLE, we need to change it to the - // chosen domain. This will always be NUMA since that's the only domain - // supported by level zero. - prop.value.affinity_domain = UR_DEVICE_AFFINITY_DOMAIN_FLAG_NUMA; - } - Device->SubDevices[I]->SubDeviceCreationProperty = prop; - - OutDevices[I] = Device->SubDevices[I]; - // reusing the same pi_device needs to increment the reference count - ur::level_zero::urDeviceRetain(OutDevices[I]); - } - - if (NumDevicesRet) { - *NumDevicesRet = EffectiveNumDevices; - } - return UR_RESULT_SUCCESS; -} - -ur_result_t urDeviceSelectBinary( - /// [in] handle of the device to select binary for. - [[maybe_unused]] ur_device_handle_t Device, - /// [in] the array of binaries to select from. - const ur_device_binary_t *Binaries, - /// [in] the number of binaries passed in ppBinaries. Must greater than or - /// equal to zero otherwise ::UR_RESULT_ERROR_INVALID_VALUE is returned. - uint32_t NumBinaries, - /// [out] the index of the selected binary in the input array of - /// binaries. If a suitable binary was not found the function returns - /// ${X}_INVALID_BINARY. - uint32_t *SelectedBinary) { - // TODO: this is a bare-bones implementation for choosing a device image - // that would be compatible with the targeted device. An AOT-compiled - // image is preferred over SPIR-V for known devices (i.e. Intel devices) - // The implementation makes no effort to differentiate between multiple images - // for the given device, and simply picks the first one compatible. - // - // Real implementation will use the same mechanism OpenCL ICD dispatcher - // uses. Something like: - // PI_VALIDATE_HANDLE_RETURN_HANDLE(ctx, PI_ERROR_INVALID_CONTEXT); - // return context->dispatch->piextDeviceSelectIR( - // ctx, images, num_images, selected_image); - // where context->dispatch is set to the dispatch table provided by PI - // plugin for platform/device the ctx was created for. - - // Look for GEN binary, which we known can only be handled by Level-Zero now. - const char *BinaryTarget = - UR_DEVICE_BINARY_TARGET_SPIRV64_GEN; // UR_DEVICE_BINARY_TARGET_SPIRV64_GEN; - - uint32_t *SelectedBinaryInd = SelectedBinary; - - // Find the appropriate device image - // The order of elements is important, as it defines the priority: - std::vector FallbackTargets = {UR_DEVICE_BINARY_TARGET_SPIRV64}; - - constexpr uint32_t InvalidInd = std::numeric_limits::max(); - uint32_t FallbackInd = InvalidInd; - uint32_t FallbackPriority = InvalidInd; - - for (uint32_t i = 0; i < NumBinaries; ++i) { - if (strcmp(Binaries[i].pDeviceTargetSpec, BinaryTarget) == 0) { - *SelectedBinaryInd = i; - return UR_RESULT_SUCCESS; - } - for (uint32_t j = 0; j < FallbackTargets.size(); ++j) { - // We have a fall-back with the same or higher priority already - // no need to check the rest - if (FallbackPriority <= j) - break; - - if (strcmp(Binaries[i].pDeviceTargetSpec, FallbackTargets[j]) == 0) { - FallbackInd = i; - FallbackPriority = j; - break; - } - } - } - // We didn't find a primary target, try the highest-priority fall-back - if ((*SelectedBinaryInd = FallbackInd) != InvalidInd) - return UR_RESULT_SUCCESS; - - // No image can be loaded for the given device - return UR_RESULT_ERROR_INVALID_BINARY; -} - -ur_result_t urDeviceGetNativeHandle( - /// [in] handle of the device. - ur_device_handle_t Device, - /// [out] a pointer to the native handle of the device. - ur_native_handle_t *NativeDevice) { - *NativeDevice = reinterpret_cast(Device->ZeDevice); - return UR_RESULT_SUCCESS; -} - -ur_result_t urDeviceCreateWithNativeHandle( - /// [in] the native handle of the device. - ur_native_handle_t NativeDevice, - /// [in] handle of the platform instance - [[maybe_unused]] ur_adapter_handle_t Adapter, - /// [in][optional] pointer to native device properties struct. - [[maybe_unused]] const ur_device_native_properties_t *Properties, - /// [out] pointer to the handle of the device object created. - ur_device_handle_t *Device) { - auto ZeDevice = ur_cast(NativeDevice); - - // The SYCL spec requires that the set of devices must remain fixed for the - // duration of the application's execution. We assume that we found all of the - // Level Zero devices when we initialized the platforms/devices cache, so the - // "NativeHandle" must already be in the cache. If it is not, this must not be - // a valid Level Zero device. - - ur_device_handle_t Dev = nullptr; - for (const auto &p : GlobalAdapter->Platforms) { - Dev = p->getDeviceFromNativeHandle(ZeDevice); - } - - if (Dev == nullptr) - return UR_RESULT_ERROR_INVALID_VALUE; - - *Device = Dev; - return UR_RESULT_SUCCESS; } -ur_result_t urDeviceGetGlobalTimestamps( - /// [in] handle of the device instance - ur_device_handle_t Device, - /// [out][optional] pointer to the Device's global timestamp that correlates - /// with the Host's global timestamp value - uint64_t *DeviceTimestamp, - /// [out][optional] pointer to the Host's global timestamp that correlates - /// with the Device's global timestamp value - uint64_t *HostTimestamp) { - if (!DeviceTimestamp && HostTimestamp) { - // If only HostTimestamp is requested, we need to avoid making a call to - // zeDeviceGetGlobalTimestamps which has higher latency. This is a - // workaround for the fact that Level Zero does not provide a way to get the - // host timestamp directly. It is known that current implementation of L0 - // runtime uses CLOCK_MONOTONIC_RAW on Linux and QueryPerformanceCounter on - // Windows. -#if defined(__linux__) - timespec Monotonic; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &Monotonic) != 0) { - UR_LOG(ERR, "Failed to get CLOCK_MONOTONIC time"); - return UR_RESULT_ERROR_UNINITIALIZED; - } - *HostTimestamp = static_cast(Monotonic.tv_sec) * 1'000'000'000 + - static_cast(Monotonic.tv_nsec); - return UR_RESULT_SUCCESS; -#elif defined(_WIN32) - // Use QueryPerformanceCounter on Windows - uint64_t Counter; - if (!QueryPerformanceCounter((LARGE_INTEGER *)&Counter)) { - UR_LOG(ERR, "Failed to get performance counter"); - return UR_RESULT_ERROR_UNINITIALIZED; - } - LARGE_INTEGER Frequency; - if (!QueryPerformanceFrequency(&Frequency)) { - UR_LOG(ERR, "Failed to get performance frequency"); - return UR_RESULT_ERROR_UNINITIALIZED; - } - *HostTimestamp = static_cast( - (static_cast(Counter) * 1'000'000'000 / Frequency.QuadPart)); - return UR_RESULT_SUCCESS; -#endif - } - - const double ZeTimerResolution = Device->getTimerResolution(); - const uint64_t TimestampMaxCount = Device->getTimestampMask(); - uint64_t DeviceClockCount, Dummy; - - ZE2UR_CALL(zeDeviceGetGlobalTimestamps, - (Device->ZeDevice, - HostTimestamp == nullptr ? &Dummy : HostTimestamp, - &DeviceClockCount)); - - if (DeviceTimestamp != nullptr) { - *DeviceTimestamp = - (DeviceClockCount & TimestampMaxCount) * ZeTimerResolution; - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urDeviceRetain(ur_device_handle_t Device) { - // The root-device ref-count remains unchanged (always 1). - if (Device->isSubDevice()) { - Device->RefCount.retain(); - } - return UR_RESULT_SUCCESS; -} - -ur_result_t urDeviceRelease(ur_device_handle_t Device) { - // Root devices are destroyed during the piTearDown process. - if (Device->isSubDevice()) { - if (Device->RefCount.release()) { - delete Device; - } - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urDeviceWaitExp(ur_device_handle_t Device) { - ZE2UR_CALL(zeDeviceSynchronize, (Device->ZeDevice)); - return UR_RESULT_SUCCESS; -} -} // namespace ur::level_zero - -/** - * @brief Determines the mode of immediate command lists to be used. - * - * This function checks environment variables and device properties to decide - * the mode of immediate command lists. The mode can be influenced by the - * following environment variables: - * - `UR_L0_USE_IMMEDIATE_COMMANDLISTS` - * - `SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS` - * - * If neither environment variable is set, the function defaults to using the - * device's properties to determine the mode. - * - * @return The mode of immediate command lists, which can be one of the - * following: - * - `NotUsed`: Immediate command lists are not used. - * - `PerQueue`: Immediate command lists are used per queue. - * - `PerThreadPerQueue`: Immediate command lists are used per thread per queue. - * - * The decision process is as follows: - * 1. If the environment variables are not set, the function checks if the - * device is Intel DG2 or newer and if the driver version is supported. If both - * conditions are met, or if the device is PVC, it returns `PerQueue`. - * Otherwise, it returns `NotUsed`. - * 2. If the environment variable is set, it returns the corresponding mode: - * - `0`: `NotUsed` - * - `1`: `PerQueue` - * - `2`: `PerThreadPerQueue` - * - Any other value: `NotUsed` - */ -ur_device_handle_t_::ImmCmdlistMode -ur_device_handle_t_::useImmediateCommandLists() { - // If immediate commandlist setting is not explicitly set, then use the device - // default. - // TODO: confirm this is good once make_queue revert is added - static const int ImmediateCommandlistsSetting = [] { - const char *UrRet = std::getenv("UR_L0_USE_IMMEDIATE_COMMANDLISTS"); - const char *PiRet = - std::getenv("SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS"); - const char *ImmediateCommandlistsSettingStr = - UrRet ? UrRet : (PiRet ? PiRet : nullptr); - if (!ImmediateCommandlistsSettingStr) - return -1; - return std::atoi(ImmediateCommandlistsSettingStr); - }(); - - if (ImmediateCommandlistsSetting == -1) { - bool isDG2OrNewer = this->isIntelDG2OrNewer(); - bool isDG2SupportedDriver = - this->Platform->isDriverVersionNewerOrSimilar(1, 5, 30820); - bool isIntelMTLDevice = this->isIntelMTL(); - bool isIntelARLDevice = this->isIntelARL(); - // Disable immediate command lists for DG2 devices on Windows due to driver - // limitations. - bool isLinux = true; -#ifdef _WIN32 - isLinux = false; -#endif - // Disable immediate command lists for Intel MTL/ARL devices on Linux by - // default due to driver limitations. - if ((isIntelMTLDevice || isIntelARLDevice) && isLinux) { - return NotUsed; - } - if ((isDG2SupportedDriver && isDG2OrNewer && isLinux) || isPVC() || - isNewerThanIntelDG2()) { - return PerQueue; - } else { - return NotUsed; - } - } - - UR_LOG(INFO, "NOTE: L0 Immediate CommandList Setting: {}", - ImmediateCommandlistsSetting); - - switch (ImmediateCommandlistsSetting) { - case 0: - return NotUsed; - case 1: - return PerQueue; - case 2: - return PerThreadPerQueue; - default: - return NotUsed; - } -} - -bool ur_device_handle_t_::useRelaxedAllocationLimits() { - static const bool EnableRelaxedAllocationLimits = [] { - auto UrRet = ur_getenv("UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS"); - const bool RetVal = UrRet ? std::stoi(*UrRet) : 0; - return RetVal; - }(); - - return EnableRelaxedAllocationLimits; -} - -bool ur_device_handle_t_::useDriverCounterBasedEvents() { - // Use counter-based events implementation from L0 driver. - - static const bool DriverCounterBasedEventsEnabled = [] { - const char *UrRet = std::getenv("UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS"); - if (!UrRet) { - return true; - } - return std::atoi(UrRet) != 0; - }(); - - return DriverCounterBasedEventsEnabled; -} - -ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal, - int SubSubDeviceIndex) { - // Maintain various device properties cache. - // Note that we just describe here how to compute the data. - // The real initialization is upon first access. - // - auto ZeDevice = this->ZeDevice; - ZeDeviceProperties.Compute = [ZeDevice](ze_device_properties_t &Properties) { - ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &Properties)); - }; - - ZeDeviceComputeProperties.Compute = - [ZeDevice](ze_device_compute_properties_t &Properties) { - ZE_CALL_NOCHECK(zeDeviceGetComputeProperties, (ZeDevice, &Properties)); - }; - - ZeDeviceIpVersionExt.Compute = - [ZeDevice](ze_device_ip_version_ext_t &Properties) { - ze_device_properties_t P; - P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - P.pNext = (void *)&Properties; - ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); - }; - - ZeDeviceImageProperties.Compute = - [ZeDevice](ze_device_image_properties_t &Properties) { - ZE_CALL_NOCHECK(zeDeviceGetImageProperties, (ZeDevice, &Properties)); - }; - - ZeDeviceModuleProperties.Compute = - [ZeDevice](ze_device_module_properties_t &Properties) { - ZE_CALL_NOCHECK(zeDeviceGetModuleProperties, (ZeDevice, &Properties)); - }; - - ZeDeviceMemoryProperties.Compute = - [ZeDevice]( - std::pair>, - std::vector>> - &Properties) { - uint32_t Count = 0; - ZE_CALL_NOCHECK(zeDeviceGetMemoryProperties, - (ZeDevice, &Count, nullptr)); - - auto &PropertiesVector = Properties.first; - auto &PropertiesExtVector = Properties.second; - - PropertiesVector.resize(Count); - PropertiesExtVector.resize(Count); - // Request for extended memory properties be read in - for (uint32_t I = 0; I < Count; ++I) - PropertiesVector[I].pNext = (void *)&PropertiesExtVector[I]; - - ZE_CALL_NOCHECK(zeDeviceGetMemoryProperties, - (ZeDevice, &Count, PropertiesVector.data())); - }; - - ZeDeviceMemoryAccessProperties.Compute = - [ZeDevice](ze_device_memory_access_properties_t &Properties) { - ZE_CALL_NOCHECK(zeDeviceGetMemoryAccessProperties, - (ZeDevice, &Properties)); - }; - - ZeDeviceCacheProperties.Compute = - [ZeDevice](ze_device_cache_properties_t &Properties) { - // TODO: Since v1.0 there can be multiple cache properties. - // For now remember the first one, if any. - uint32_t Count = 0; - ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, - (ZeDevice, &Count, nullptr)); - if (Count > 0) - Count = 1; - ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, - (ZeDevice, &Count, &Properties)); - }; - - if (this->Platform->zeDriverExtensionMap.count(ZE_CACHELINE_SIZE_EXT_NAME)) { - ZeDeviceCacheLinePropertiesExt.Compute = - [ZeDevice](ze_device_cache_line_size_ext_t &Properties) { - // TODO: Since v1.0 there can be multiple cache properties. - // For now remember the first one, if any. - uint32_t Count = 0; - ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, - (ZeDevice, &Count, nullptr)); - if (Count > 0) - Count = 1; - ze_device_cache_properties_t P; - P.stype = ZE_STRUCTURE_TYPE_DEVICE_CACHE_PROPERTIES; - P.pNext = &Properties; - ZE_CALL_NOCHECK(zeDeviceGetCacheProperties, (ZeDevice, &Count, &P)); - if (Properties.cacheLineSize == 0) { - // If cache line size is not set, use the default value. - Properties.cacheLineSize = - 1; // Default cache line size property value. - } - }; - } - - ZeDeviceMutableCmdListsProperties.Compute = - [ZeDevice]( - ZeStruct &Properties) { - ze_device_properties_t P; - P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - P.pNext = &Properties; - ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); - }; - -#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME - ZeDeviceBlockArrayProperties.Compute = - [ZeDevice]( - ZexStruct &Properties) { - ze_device_properties_t P; - P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - P.pNext = &Properties; - ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); - }; -#endif // ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME - - auto UrPlatform = this->Platform; - ZeDeviceVectorWidthPropertiesExt.Compute = - [ZeDevice, UrPlatform]( - ZeStruct &Properties) { - // Set default vector width properties - Properties.preferred_vector_width_char = 16u; - Properties.preferred_vector_width_short = 8u; - Properties.preferred_vector_width_int = 4u; - Properties.preferred_vector_width_long = 1u; - Properties.preferred_vector_width_float = 1u; - Properties.preferred_vector_width_half = 8u; - Properties.native_vector_width_char = 16u; - Properties.native_vector_width_short = 8u; - Properties.native_vector_width_int = 4u; - Properties.native_vector_width_long = 1u; - Properties.native_vector_width_float = 1u; - Properties.native_vector_width_half = 8u; - - if (UrPlatform->zeDriverExtensionMap.count( - ZE_DEVICE_VECTOR_SIZES_EXT_NAME)) { - uint32_t Count = 0; - ZE_CALL_NOCHECK(zeDeviceGetVectorWidthPropertiesExt, - (ZeDevice, &Count, nullptr)); - - std::vector> - PropertiesVector; - PropertiesVector.resize(Count); - - ZeStruct - MaxVectorWidthProperties; - - ZE_CALL_NOCHECK(zeDeviceGetVectorWidthPropertiesExt, - (ZeDevice, &Count, PropertiesVector.data())); - if (!PropertiesVector.empty()) { - // Find the largest vector_width_size property - uint32_t max_vector_width_size = 0; - for (const auto &prop : PropertiesVector) { - if (!max_vector_width_size) { - max_vector_width_size = prop.vector_width_size; - MaxVectorWidthProperties = prop; - } else if (prop.vector_width_size > max_vector_width_size) { - max_vector_width_size = prop.vector_width_size; - MaxVectorWidthProperties = prop; - } - } - Properties = MaxVectorWidthProperties; - // If the environment variable is set, use the specified vector - // width if it exists - if (UrL0VectorWidth) { - for (const auto &prop : PropertiesVector) { - if (prop.vector_width_size == - static_cast(UrL0VectorWidth)) { - Properties = prop; - break; - } - } - } - } - } - }; - - ZeXEDeviceProperties.Compute = - [ZeDevice](ze_intel_xe_device_exp_properties_t &Properties) { - // Set up default values of 0 - Properties.numXeStacks = 0; - Properties.numXeRegionsPerStack = 0; - Properties.numXeClustersPerRegion = 0; - Properties.numXeCorePerCluster = 0; - Properties.numExecutionEnginesPerXeCore = 0; - Properties.maxNumHwThreadsPerExecutionEngine = 0; - Properties.maxNumLanesPerHwThread = 0; - - ze_device_properties_t P; - P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - P.pNext = (void *)&Properties; - ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P)); - }; - - ZeDevicePitchedAllocProperties.Compute = - [ZeDevice](ZeDevicePitchedAllocInfo &Properties) { - ze_device_image_properties_t ImageProps = {}; - ImageProps.stype = ZE_STRUCTURE_TYPE_DEVICE_IMAGE_PROPERTIES; - ImageProps.pNext = &Properties.AllocProps; - Properties.AllocProps.pNext = &Properties.PitchInfo; - ZE_CALL_NOCHECK(zeDeviceGetImageProperties, (ZeDevice, &ImageProps)); - }; - - ImmCommandListUsed = this->useImmediateCommandLists(); - - uint32_t numQueueGroups = 0; - ZE2UR_CALL(zeDeviceGetCommandQueueGroupProperties, - (ZeDevice, &numQueueGroups, nullptr)); - if (numQueueGroups == 0) { - return UR_RESULT_ERROR_UNKNOWN; - } - UR_LOG_LEGACY(INFO, - logger::LegacyMessage("NOTE: Number of queue groups = {}"), - "Number of queue groups = {}", numQueueGroups); - - std::vector> - QueueGroupProperties(numQueueGroups); - ZE2UR_CALL(zeDeviceGetCommandQueueGroupProperties, - (ZeDevice, &numQueueGroups, QueueGroupProperties.data())); - - // Initialize ordinal and compute queue group properties - for (uint32_t i = 0; i < numQueueGroups; i++) { - if (QueueGroupProperties[i].flags & - ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) { - QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal = - i; - QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute] - .ZeProperties = QueueGroupProperties[i]; - break; - } - } - - // Reinitialize a sub-sub-device with its own ordinal, index. - // Our sub-sub-device representation is currently [Level-Zero sub-device - // handle + Level-Zero compute group/engine index]. Only the specified - // index queue will be used to submit work to the sub-sub-device. - if (SubSubDeviceOrdinal >= 0) { - QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal = - SubSubDeviceOrdinal; - QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeIndex = - SubSubDeviceIndex; - } else { // Proceed with initialization for root and sub-device - // How is it possible that there are no "compute" capabilities? - if (QueueGroup[ur_device_handle_t_::queue_group_info_t::Compute].ZeOrdinal < - 0) { - return UR_RESULT_ERROR_UNKNOWN; - } - - if (ur::level_zero::CopyEngineRequested((ur_device_handle_t)this)) { - for (uint32_t i = 0; i < numQueueGroups; i++) { - if (((QueueGroupProperties[i].flags & - ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) == 0) && - (QueueGroupProperties[i].flags & - ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY)) { - if (QueueGroupProperties[i].numQueues == 1) { - QueueGroup[queue_group_info_t::MainCopy].ZeOrdinal = i; - QueueGroup[queue_group_info_t::MainCopy].ZeProperties = - QueueGroupProperties[i]; - } else { - QueueGroup[queue_group_info_t::LinkCopy].ZeOrdinal = i; - QueueGroup[queue_group_info_t::LinkCopy].ZeProperties = - QueueGroupProperties[i]; - break; - } - } - } - if (QueueGroup[queue_group_info_t::MainCopy].ZeOrdinal < 0) - UR_LOG_LEGACY(INFO, - logger::LegacyMessage( - "NOTE: main blitter/copy engine is not available"), - "main blitter/copy engine is not available") - else - UR_LOG_LEGACY(INFO, - logger::LegacyMessage( - "NOTE: main blitter/copy engine is available"), - "main blitter/copy engine is available") - - if (QueueGroup[queue_group_info_t::LinkCopy].ZeOrdinal < 0) - UR_LOG_LEGACY(INFO, - logger::LegacyMessage( - "NOTE: link blitter/copy engines are not available"), - "link blitter/copy engines are not available") - else - UR_LOG_LEGACY(INFO, - logger::LegacyMessage( - "NOTE: link blitter/copy engines are available"), - "link blitter/copy engines are available") - } - } - - return UR_RESULT_SUCCESS; -} - -void ZeDriverVersionStringExtension::setZeDriverVersionString( - ur_platform_handle_t_ *Platform) { - // Check if Intel Driver Version String is available. If yes, save the API - // pointer. The pointer will be used when reading the Driver Version for - // users. - ze_driver_handle_t DriverHandle = Platform->ZeDriver; - if (auto extension = Platform->zeDriverExtensionMap.find( - "ZE_intel_get_driver_version_string"); - extension != Platform->zeDriverExtensionMap.end()) { - if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress, - (DriverHandle, "zeIntelGetDriverVersionString", - reinterpret_cast( - &zeIntelGetDriverVersionStringPointer))) == 0) { - // Intel Driver Version String is Supported by this Driver. - Supported = true; - } - } -} - -void ZeDriverVersionStringExtension::getDriverVersionString( - ze_driver_handle_t DriverHandle, char *pDriverVersion, - size_t *pVersionSize) { - ZE_CALL_NOCHECK(zeIntelGetDriverVersionStringPointer, - (DriverHandle, pDriverVersion, pVersionSize)); -} - -void ZeUSMImportExtension::setZeUSMImport(ur_platform_handle_t_ *Platform) { - // Check if USM hostptr import feature is available. If yes, save the API - // pointers. The pointers will be used for both import/release of SYCL buffer - // host ptr and the SYCL experimental APIs, prepare_for_device_copy and - // release_from_device_copy. - ze_driver_handle_t DriverHandle = Platform->ZeDriver; - if (ZE_CALL_NOCHECK( - zeDriverGetExtensionFunctionAddress, - (DriverHandle, "zexDriverImportExternalPointer", - reinterpret_cast(&zexDriverImportExternalPointer))) == 0) { - ZE_CALL_NOCHECK( - zeDriverGetExtensionFunctionAddress, - (DriverHandle, "zexDriverReleaseImportedPointer", - reinterpret_cast(&zexDriverReleaseImportedPointer))); - // Hostptr import/release is supported by this platform. - Supported = true; - - // Check if env var SYCL_USM_HOSTPTR_IMPORT has been set requesting - // host ptr import during buffer creation. - const char *USMHostPtrImportStr = std::getenv("SYCL_USM_HOSTPTR_IMPORT"); - if (!USMHostPtrImportStr || std::atoi(USMHostPtrImportStr) == 0) - return; - - // Hostptr import/release is turned on because it has been requested - // by the env var, and this platform supports the APIs. - Enabled = true; - // Hostptr import is only possible if piMemBufferCreate receives a - // hostptr as an argument. The SYCL runtime passes a host ptr - // only when SYCL_HOST_UNIFIED_MEMORY is enabled. Therefore we turn it on. - setEnvVar("SYCL_HOST_UNIFIED_MEMORY", "1"); - } -} -void ZeUSMImportExtension::doZeUSMImport(ze_driver_handle_t DriverHandle, - void *HostPtr, size_t Size) { - ZE_CALL_NOCHECK(zexDriverImportExternalPointer, - (DriverHandle, HostPtr, Size)); -} -void ZeUSMImportExtension::doZeUSMRelease(ze_driver_handle_t DriverHandle, - void *HostPtr) { - ZE_CALL_NOCHECK(zexDriverReleaseImportedPointer, (DriverHandle, HostPtr)); -} - -std::ostream &operator<<(std::ostream &os, - ur_device_handle_t_ const &device_handle) { - if (device_handle.Id.has_value()) { - return os << device_handle.Id.value(); - } - return os << "NONE"; -} - -std::ostream &operator<<(std::ostream &os, - ur_device_handle_t_::PeerStatus peer_status) { - switch (peer_status) { - case ur_device_handle_t_::PeerStatus::DISABLED: - return os << "DISABLED"; - case ur_device_handle_t_::PeerStatus::ENABLED: - return os << "ENABLED"; - case ur_device_handle_t_::PeerStatus::NO_CONNECTION: - return os << "NO_CONNECTION"; - } - return os << "UNKNOWN"; -} +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/enqueue_native.cpp b/unified-runtime/source/adapters/level_zero/enqueue_native.cpp index 58b23047cd122..b3089d73c9154 100644 --- a/unified-runtime/source/adapters/level_zero/enqueue_native.cpp +++ b/unified-runtime/source/adapters/level_zero/enqueue_native.cpp @@ -9,19 +9,19 @@ #include -namespace ur::level_zero { +namespace ur::level_zero::v1 { ur_result_t urEnqueueNativeCommandExp( - ur_queue_handle_t /*hQueue*/, + ::ur_queue_handle_t /*hQueue*/, ur_exp_enqueue_native_command_function_t /*pfnNativeEnqueue*/, void * /*data*/, uint32_t /*numMemsInMemList*/, - const ur_mem_handle_t * /*phMemList*/, + const ::ur_mem_handle_t * /*phMemList*/, const ur_exp_enqueue_native_command_properties_t * /*pProperties*/, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, - ur_event_handle_t * /*phEvent*/) { + const ::ur_event_handle_t * /*phEventWaitList*/, + ::ur_event_handle_t * /*phEvent*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index 83f9b347ff2fd..51bb588bbb9ec 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -15,11 +15,14 @@ #include "command_buffer.hpp" #include "common.hpp" +#include "common/device.hpp" #include "event.hpp" #include "logger/ur_logger.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" +namespace ur::level_zero::v1 { + void printZeEventList(const ur_ze_event_list_t &UrZeEventList) { if (UrL0Debug & UR_L0_DEBUG_BASIC) { std::stringstream ss; @@ -59,22 +62,23 @@ bool WaitListEmptyOrAllEventsFromSameQueue( return true; } -namespace ur::level_zero { - ur_result_t urEnqueueEventsWait( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] size of the event wait list uint32_t NumEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that all /// previously enqueued commands must be complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { - if (EventWaitList) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); + if (EventWaitListInternal) { bool UseCopyEngine = false; // Lock automatically releases when this goes out of scope. @@ -82,18 +86,20 @@ ur_result_t urEnqueueEventsWait( ur_ze_event_list_t TmpWaitList = {}; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, - false /*AllowBatching*/, nullptr /*ForceCmdQueue*/)); + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, false /*AllowBatching*/, + nullptr /*ForceCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_EVENTS_WAIT, CommandList, IsInternal, false)); @@ -121,8 +127,8 @@ ur_result_t urEnqueueEventsWait( // Lock automatically releases when this goes out of scope. std::scoped_lock lock(Queue->Mutex); - if (OutEvent) { - UR_CALL(createEventAndAssociateQueue(Queue, OutEvent, + if (OutEventInternal) { + UR_CALL(createEventAndAssociateQueue(Queue, OutEventInternal, UR_COMMAND_EVENTS_WAIT, Queue->CommandListMap.end(), false, /* IsInternal */ false)); @@ -131,12 +137,13 @@ ur_result_t urEnqueueEventsWait( UR_CALL(Queue->executeAllOpenCommandLists()); UR_CALL(Queue->synchronize()); - if (OutEvent) { - Queue->LastCommandEvent = reinterpret_cast(*OutEvent); + if (OutEventInternal) { + auto OutEventLocal = *OutEventInternal; + Queue->LastCommandEvent = OutEventLocal; - if (!(*OutEvent)->CounterBasedEventsEnabled) - ZE2UR_CALL(zeEventHostSignal, ((*OutEvent)->ZeEvent)); - (*OutEvent)->Completed = true; + if (!OutEventLocal->CounterBasedEventsEnabled) + ZE2UR_CALL(zeEventHostSignal, (OutEventLocal->ZeEvent)); + OutEventLocal->Completed = true; } if (!Queue->UsingImmCmdLists) { @@ -155,24 +162,24 @@ static const bool InOrderBarrierBySignal = [] { ur_result_t urEnqueueEventsWaitWithBarrier( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] size of the event wait list uint32_t NumEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that all /// previously enqueued commands must be complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { - return ur::level_zero::urEnqueueEventsWaitWithBarrierExt( - Queue, nullptr, NumEventsInWaitList, EventWaitList, OutEvent); + ::ur_event_handle_t *OutEvent) { + return ur::level_zero::v1::urEnqueueEventsWaitWithBarrierExt( + QueueOpque, nullptr, NumEventsInWaitList, EventWaitList, OutEvent); } ur_result_t urEnqueueEventsWaitWithBarrierExt( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in][optional] pointer to the extended enqueue const ur_exp_enqueue_ext_properties_t *EnqueueExtProp, /// [in] size of the event wait list @@ -181,10 +188,13 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that all /// previously enqueued commands must be complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); bool InterruptBasedEventsEnabled = EnqueueExtProp ? (EnqueueExtProp->flags & UR_EXP_ENQUEUE_EXT_FLAG_LOW_POWER_EVENTS_SUPPORT) || @@ -241,12 +251,13 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( // a "barrier" event to be created. Or if we need to wait for events in // potentially different queues. // - if (Queue->isInOrderQueue() && NumEventsInWaitList == 0 && !OutEvent) { + if (Queue->isInOrderQueue() && NumEventsInWaitList == 0 && + !OutEventInternal) { return UR_RESULT_SUCCESS; } ur_event_handle_t ResultEvent = nullptr; - bool IsInternal = OutEvent == nullptr; + bool IsInternal = OutEventInternal == nullptr; // For in-order queue and wait-list which is empty or has events from // the same queue just use the last command event as the barrier event. @@ -254,12 +265,13 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( // accurate profiling values & the overhead that profiling incurs. if (Queue->isInOrderQueue() && !Queue->isProfilingEnabled() && WaitListEmptyOrAllEventsFromSameQueue(Queue, NumEventsInWaitList, - EventWaitList) && + EventWaitListInternal) && Queue->LastCommandEvent && !Queue->LastCommandEvent->IsDiscarded) { - UR_CALL(ur::level_zero::urEventRetain(Queue->LastCommandEvent)); + UR_CALL( + ur::level_zero::v1::urEventRetain(v1_cast(Queue->LastCommandEvent))); ResultEvent = Queue->LastCommandEvent; - if (OutEvent) { - *OutEvent = ResultEvent; + if (OutEventInternal) { + *OutEventInternal = ResultEvent; } return UR_RESULT_SUCCESS; } @@ -281,13 +293,14 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( // Retain the events as they will be owned by the result event. ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, false /*UseCopyEngine=*/)); + NumEventsInWaitList, EventWaitListInternal, Queue, + false /*UseCopyEngine=*/)); // Get an arbitrary command-list in the queue. ur_command_list_ptr_t CmdList; UR_CALL(Queue->Context->getAvailableCommandList( Queue, CmdList, false /*UseCopyEngine=*/, NumEventsInWaitList, - EventWaitList, OkToBatch, nullptr /*ForcedCmdQueue*/)); + EventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); // Insert the barrier into the command-list and execute. UR_CALL(insertBarrierIntoCmdList(CmdList, TmpWaitList, ResultEvent, @@ -306,8 +319,8 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( Queue->ActiveBarriers.add(UREvent); } - if (OutEvent) { - *OutEvent = ResultEvent; + if (OutEventInternal) { + *OutEventInternal = ResultEvent; } return UR_RESULT_SUCCESS; } @@ -339,7 +352,8 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( {Queue->ComputeQueueGroupsByTID, Queue->CopyQueueGroupsByTID}) for (auto &QueueGroup : QueueMap) { bool UseCopyEngine = - QueueGroup.second.Type != ur_queue_handle_t_::queue_type::Compute; + QueueGroup.second.Type != + ur::level_zero::v1::ur_queue_handle_t_::queue_type::Compute; if (Queue->UsingImmCmdLists) { // If immediate command lists are being used, each will act as their own // queue, so we must insert a barrier into each. @@ -352,7 +366,7 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( ur_command_list_ptr_t CmdList; UR_CALL(Queue->Context->getAvailableCommandList( Queue, CmdList, UseCopyEngine, NumEventsInWaitList, - EventWaitList, OkToBatch, &ZeQueue)); + EventWaitListInternal, OkToBatch, &ZeQueue)); CmdLists.push_back(CmdList); } } @@ -366,7 +380,7 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( ur_command_list_ptr_t CmdList; UR_CALL(Queue->Context->getAvailableCommandList( Queue, CmdList, false /*UseCopyEngine=*/, NumEventsInWaitList, - EventWaitList, OkToBatch, nullptr /*ForcedCmdQueue*/)); + EventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); CmdLists.push_back(CmdList); } @@ -411,8 +425,7 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( // Execute each command list so the barriers can be encountered. for (ur_command_list_ptr_t &CmdList : CmdLists) { - bool IsCopy = - CmdList->second.isCopy(reinterpret_cast(Queue)); + bool IsCopy = CmdList->second.isCopy(Queue); const auto &CommandBatch = (IsCopy) ? Queue->CopyCommandBatch : Queue->ComputeCommandBatch; // Only batch if the matching CmdList is already open. @@ -424,15 +437,15 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( UR_CALL(Queue->ActiveBarriers.clear()); Queue->ActiveBarriers.add(ResultEvent); - if (OutEvent) { - *OutEvent = ResultEvent; + if (OutEventInternal) { + *OutEventInternal = ResultEvent; } return UR_RESULT_SUCCESS; } ur_result_t urEventGetInfo( /// [in] handle of the event object - ur_event_handle_t Event, + ::ur_event_handle_t EventOpque, /// [in] the name of the event property to query ur_event_info_t PropName, /// [in] size in bytes of the event property value @@ -442,6 +455,7 @@ ur_result_t urEventGetInfo( size_t /// [out][optional] bytes returned in event property *PropValueSizeRet) { + auto Event = v1_cast(EventOpque); UrReturnHelper ReturnValue(PropValueSize, PropValue, PropValueSizeRet); switch (PropName) { @@ -518,7 +532,7 @@ ur_result_t urEventGetInfo( ur_result_t urEventGetProfilingInfo( /// [in] handle of the event object - ur_event_handle_t Event, + ::ur_event_handle_t EventOpque, /// [in] the name of the profiling property to query ur_profiling_info_t PropName, /// [in] size in bytes of the profiling property value @@ -528,6 +542,7 @@ ur_result_t urEventGetProfilingInfo( /// [out][optional] pointer to the actual size in bytes returned in /// propValue size_t *PropValueSizeRet) { + auto Event = v1_cast(EventOpque); std::shared_lock EventLock(Event->Mutex); // The event must either have profiling enabled or be recording timestamps. @@ -537,7 +552,7 @@ ur_result_t urEventGetProfilingInfo( } ur_device_handle_t Device = - Event->UrQueue ? Event->UrQueue->Device : Event->Context->Devices[0]; + Event->UrQueue ? Event->UrQueue->Device : Event->Context->getDevices()[0]; const double ZeTimerResolution = Device->getTimerResolution(); const uint64_t TimestampMaxValue = Device->getTimestampMask(); @@ -721,7 +736,7 @@ ur_result_t urEventGetProfilingInfo( ur_result_t urEnqueueTimestampRecordingExp( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] blocking or non-blocking enqueue bool Blocking, /// [in] size of the event wait list @@ -730,10 +745,13 @@ ur_result_t urEnqueueTimestampRecordingExp( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out] return an event object that identifies this particular /// command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternalPtr = v1_cast(OutEvent); // Lock automatically releases when this goes out of scope. std::scoped_lock lock(Queue->Mutex); @@ -742,37 +760,42 @@ ur_result_t urEnqueueTimestampRecordingExp( bool UseCopyEngine = false; ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, /* AllowBatching */ false, nullptr /*ForcedCmdQueue*/)); UR_CALL(createEventAndAssociateQueue( - Queue, OutEvent, UR_COMMAND_TIMESTAMP_RECORDING_EXP, CommandList, + Queue, OutEventInternalPtr, UR_COMMAND_TIMESTAMP_RECORDING_EXP, + CommandList, /* IsInternal */ false, /* HostVisible */ true)); - ze_event_handle_t ZeEvent = (*OutEvent)->ZeEvent; - (*OutEvent)->WaitList = TmpWaitList; + auto OutEventInternal = *OutEventInternalPtr; + ze_event_handle_t ZeEvent = OutEventInternal->ZeEvent; + OutEventInternal->WaitList = TmpWaitList; // Reset the end timestamp, in case it has been previously used. - (*OutEvent)->RecordEventEndTimestamp = 0; + OutEventInternal->RecordEventEndTimestamp = 0; uint64_t DeviceStartTimestamp = 0; UR_CALL(ur::level_zero::urDeviceGetGlobalTimestamps( - Device, &DeviceStartTimestamp, nullptr)); - (*OutEvent)->RecordEventStartTimestamp = DeviceStartTimestamp; + common_cast(Device), &DeviceStartTimestamp, nullptr)); + OutEventInternal->RecordEventStartTimestamp = DeviceStartTimestamp; // Mark this event as timestamped - (*OutEvent)->IsTimestamped = true; + OutEventInternal->IsTimestamped = true; // Create a new entry in the queue's recordings. - Queue->EndTimeRecordings[*OutEvent] = 0; + Queue->EndTimeRecordings[*OutEventInternalPtr] = 0; ZE2UR_CALL(zeCommandListAppendWriteGlobalTimestamp, - (CommandList->first, &Queue->EndTimeRecordings[*OutEvent], ZeEvent, - (*OutEvent)->WaitList.Length, (*OutEvent)->WaitList.ZeEventList)); + (CommandList->first, + &Queue->EndTimeRecordings[*OutEventInternalPtr], ZeEvent, + OutEventInternal->WaitList.Length, + OutEventInternal->WaitList.ZeEventList)); UR_CALL( Queue->executeCommandList(CommandList, Blocking, false /* OkToBatch */)); @@ -785,7 +808,8 @@ ur_result_t urEventWait(uint32_t NumEvents, /// [in][range(0, numEvents)] pointer to a /// list of events to wait for completion - const ur_event_handle_t *EventWaitList) { + const ::ur_event_handle_t *EventWaitListOpque) { + auto EventWaitList = v1_cast(EventWaitListOpque); for (uint32_t I = 0; I < NumEvents; I++) { auto e = EventWaitList[I]; auto UrQueue = e->UrQueue; @@ -794,7 +818,7 @@ urEventWait(uint32_t NumEvents, // This ensures that all signalling commands are submitted below and // thus proxy events can be waited without a deadlock. // - ur_event_handle_t_ *Event = ur_cast(e); + ur::level_zero::v1::ur_event_handle_t_ *Event = e; if (!Event->hasExternalRefs()) die("urEventWait must not be called for an internal event"); @@ -805,7 +829,7 @@ urEventWait(uint32_t NumEvents, } // Submit dependent open command lists for execution, if any for (uint32_t I = 0; I < NumEvents; I++) { - ur_event_handle_t_ *Event = ur_cast(EventWaitList[I]); + ur::level_zero::v1::ur_event_handle_t_ *Event = EventWaitList[I]; auto UrQueue = Event->UrQueue; if (UrQueue) { // Lock automatically releases when this goes out of scope. @@ -817,8 +841,7 @@ urEventWait(uint32_t NumEvents, std::unordered_set Queues; for (uint32_t I = 0; I < NumEvents; I++) { { - ur_event_handle_t_ *Event = - ur_cast(EventWaitList[I]); + ur::level_zero::v1::ur_event_handle_t_ *Event = EventWaitList[I]; { std::shared_lock EventLock(Event->Mutex); if (!Event->hasExternalRefs()) @@ -875,7 +898,9 @@ urEventWait(uint32_t NumEvents, ur_result_t /// [in] handle of the event object -urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { +urEventRetain( + /** [in] handle of the event object */ ::ur_event_handle_t EventOpque) { + auto Event = v1_cast(EventOpque); Event->RefCountExternal++; Event->RefCount.retain(); @@ -884,7 +909,9 @@ urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { ur_result_t -urEventRelease(/** [in] handle of the event object */ ur_event_handle_t Event) { +urEventRelease( + /** [in] handle of the event object */ ::ur_event_handle_t EventOpque) { + auto Event = v1_cast(EventOpque); Event->RefCountExternal--; bool isEventsWaitCompleted = (Event->CommandType == UR_COMMAND_EVENTS_WAIT || @@ -898,7 +925,7 @@ urEventRelease(/** [in] handle of the event object */ ur_event_handle_t Event) { // cleaned up (e.g. by CleanupEventListFromResetCmdList), preventing a // double-release of the internal reference count. if (isEventsWaitCompleted & !isEventDeleted) { - UR_CALL(CleanupCompletedEvent((Event), false, false)); + UR_CALL(CleanupCompletedEvent(Event, false, false)); } return UR_RESULT_SUCCESS; @@ -906,9 +933,10 @@ urEventRelease(/** [in] handle of the event object */ ur_event_handle_t Event) { ur_result_t urEventGetNativeHandle( /// [in] handle of the event. - ur_event_handle_t Event, + ::ur_event_handle_t EventOpque, /// [out] a pointer to the native handle of the event. - ur_native_handle_t *NativeEvent) { + ::ur_native_handle_t *NativeEvent) { + auto Event = v1_cast(EventOpque); { std::shared_lock Lock(Event->Mutex); auto *ZeEvent = ur_cast(NativeEvent); @@ -931,49 +959,54 @@ ur_result_t urEventGetNativeHandle( ur_result_t urExtEventCreate( /// [in] handle of the context object - ur_context_handle_t Context, - ur_event_handle_t + ::ur_context_handle_t ContextOpque, + ::ur_event_handle_t /// [out] pointer to the handle of the event object created. *Event) { + auto Context = v1_cast(ContextOpque); + auto EventInternal = v1_cast(Event); UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/, - true /*HostVisible*/, Event, + true /*HostVisible*/, EventInternal, false /*CounterBasedEventEnabled*/, false /*ForceDisableProfiling*/, false)); - (*Event)->RefCountExternal++; - if (!(*Event)->CounterBasedEventsEnabled) - ZE2UR_CALL(zeEventHostSignal, ((*Event)->ZeEvent)); + (*EventInternal)->RefCountExternal++; + if (!(*EventInternal)->CounterBasedEventsEnabled) + ZE2UR_CALL(zeEventHostSignal, ((*EventInternal)->ZeEvent)); return UR_RESULT_SUCCESS; } ur_result_t urEventCreateWithNativeHandle( /// [in] the native handle of the event. - ur_native_handle_t NativeEvent, + ::ur_native_handle_t NativeEvent, /// [in] handle of the context object - ur_context_handle_t Context, const ur_event_native_properties_t *Properties, + ::ur_context_handle_t ContextOpque, + const ur_event_native_properties_t *Properties, /// [out] pointer to the handle of the event object created. - ur_event_handle_t *Event) { + ::ur_event_handle_t *Event) { + auto Context = v1_cast(ContextOpque); + auto EventInternal = v1_cast(Event); // we dont have urEventCreate, so use this check for now to know that // the call comes from urEventCreate() if (reinterpret_cast(NativeEvent) == nullptr) { UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/, - true /*HostVisible*/, Event, + true /*HostVisible*/, EventInternal, false /*CounterBasedEventEnabled*/, false /*ForceDisableProfiling*/, false)); - (*Event)->RefCountExternal++; - if (!(*Event)->CounterBasedEventsEnabled) - ZE2UR_CALL(zeEventHostSignal, ((*Event)->ZeEvent)); + (*EventInternal)->RefCountExternal++; + if (!(*EventInternal)->CounterBasedEventsEnabled) + ZE2UR_CALL(zeEventHostSignal, ((*EventInternal)->ZeEvent)); return UR_RESULT_SUCCESS; } auto ZeEvent = ur_cast(NativeEvent); - ur_event_handle_t_ *UREvent{}; + ur::level_zero::v1::ur_event_handle_t_ *UREvent{}; try { - UREvent = new ur_event_handle_t_(ZeEvent, nullptr /* ZeEventPool */, - Context, UR_EXT_COMMAND_TYPE_USER, - Properties->isNativeHandleOwned); + UREvent = new ur::level_zero::v1::ur_event_handle_t_( + ZeEvent, nullptr /* ZeEventPool */, Context, UR_EXT_COMMAND_TYPE_USER, + Properties->isNativeHandleOwned); UREvent->RefCountExternal++; } catch (const std::bad_alloc &) { @@ -993,14 +1026,14 @@ ur_result_t urEventCreateWithNativeHandle( // made for waiting for event completion, but not this interop event. UREvent->CleanedUp = true; - *Event = reinterpret_cast(UREvent); + *EventInternal = UREvent; return UR_RESULT_SUCCESS; } ur_result_t urEventSetCallback( /// [in] handle of the event object - ur_event_handle_t /*Event*/, + ::ur_event_handle_t /*Event*/, /// [in] execution status of the event ur_execution_info_t /*ExecStatus*/, /// [in] execution status of the event @@ -1013,36 +1046,34 @@ ur_result_t urEventSetCallback( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urEventCreateExp(ur_context_handle_t /*hContext*/, - ur_device_handle_t /*hDevice*/, +ur_result_t urEventCreateExp(::ur_context_handle_t /*hContext*/, + ::ur_device_handle_t /*hDevice*/, const ur_exp_event_desc_t * /*pEventDesc*/, - ur_event_handle_t * /*phEvent*/) { + ::ur_event_handle_t * /*phEvent*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCGetEventHandleExp(ur_event_handle_t /*hEvent*/, +ur_result_t urIPCGetEventHandleExp(::ur_event_handle_t /*hEvent*/, void ** /*ppIPCEventHandleData*/, size_t * /*pIPCEventHandleDataSizeRet*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCPutEventHandleExp(ur_context_handle_t /*hContext*/, +ur_result_t urIPCPutEventHandleExp(::ur_context_handle_t /*hContext*/, void * /*pIPCEventHandleData*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t /*hContext*/, +ur_result_t urIPCOpenEventHandleExp(::ur_context_handle_t /*hContext*/, const void * /*pIPCEventHandleData*/, size_t /*ipcEventHandleDataSize*/, - ur_event_handle_t * /*phEvent*/) { + ::ur_event_handle_t * /*phEvent*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero - ur_result_t ur_event_handle_t_::getOrCreateHostVisibleEvent( ze_event_handle_t &ZeHostVisibleEvent) { auto UrQueue = this->UrQueue; @@ -1105,7 +1136,7 @@ ur_result_t ur_event_handle_t_::getOrCreateHostVisibleEvent( * This ensures that resources are properly released and avoids potential memory * leaks or resource mismanagement. */ -ur_event_handle_t_::~ur_event_handle_t_() { +ur::level_zero::v1::ur_event_handle_t_::~ur_event_handle_t_() { if (this->ZeEvent && this->Completed && checkL0LoaderTeardown()) { if (this->UrQueue && !this->UrQueue->isDiscardEvents()) ZE_CALL_NOCHECK(zeEventDestroy, (this->ZeEvent)); @@ -1284,7 +1315,8 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked, // some other thread) then release referenced memory allocations. As a // result, memory can be deallocated and context can be removed from // container in the platform. That's why we need to lock a mutex here. - ur_platform_handle_t Plt = Kernel->Program->Context->getPlatform(); + ur_platform_handle_t Plt = + v1_cast(Kernel->Program->Context)->getPlatform(); std::scoped_lock ContextsLock(Plt->ContextsMutex); if (--Kernel->SubmissionsCount == 0) { @@ -1303,7 +1335,7 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked, // We've reset event data members above, now cleanup resources. if (AssociatedKernel) { ReleaseIndirectMem(AssociatedKernel); - UR_CALL(ur::level_zero::urKernelRelease(AssociatedKernel)); + UR_CALL(ur::level_zero::v1::urKernelRelease(v1_cast(AssociatedKernel))); } if (AssociatedQueue) { @@ -1362,7 +1394,7 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked, } if (DepEventKernel) { ReleaseIndirectMem(DepEventKernel); - UR_CALL(ur::level_zero::urKernelRelease(DepEventKernel)); + UR_CALL(ur::level_zero::v1::urKernelRelease(v1_cast(DepEventKernel))); } UR_CALL(urEventReleaseInternal(DepEvent)); } @@ -1432,9 +1464,8 @@ EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue, ZE2UR_CALL(zeEventCreate, (ZeEventPool, &ZeEventDesc, &ZeEvent)); try { - *RetEvent = new ur_event_handle_t_( - ZeEvent, ZeEventPool, reinterpret_cast(Context), - UR_EXT_COMMAND_TYPE_USER, true); + *RetEvent = new ur::level_zero::v1::ur_event_handle_t_( + ZeEvent, ZeEventPool, Context, UR_EXT_COMMAND_TYPE_USER, true); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -1449,13 +1480,14 @@ EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue, return UR_RESULT_SUCCESS; } -ur_result_t ur_event_handle_t_::reset() { +ur_result_t ur::level_zero::v1::ur_event_handle_t_::reset() { // Clean up timestamp recording entry from queue if this was a timestamped // event if (IsTimestamped && UrQueue) { - auto Entry = UrQueue->EndTimeRecordings.find(this); - if (Entry != UrQueue->EndTimeRecordings.end()) { - UrQueue->EndTimeRecordings.erase(Entry); + auto *Queue = UrQueue; + auto Entry = Queue->EndTimeRecordings.find(this); + if (Entry != Queue->EndTimeRecordings.end()) { + Queue->EndTimeRecordings.erase(Entry); } } @@ -1801,7 +1833,9 @@ ur_result_t ur_ze_event_list_t::collectEventsForReleaseAndDestroyUrZeEventList( } // Tells if this event is with profiling capabilities. -bool ur_event_handle_t_::isProfilingEnabled() const { +bool ur::level_zero::v1::ur_event_handle_t_::isProfilingEnabled() const { return !UrQueue || // tentatively assume user events are profiling enabled (UrQueue->Properties & UR_QUEUE_FLAG_PROFILING_ENABLE) != 0; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/event.hpp b/unified-runtime/source/adapters/level_zero/event.hpp index 88a463b97efbe..9a977b8efa5e3 100644 --- a/unified-runtime/source/adapters/level_zero/event.hpp +++ b/unified-runtime/source/adapters/level_zero/event.hpp @@ -28,7 +28,8 @@ #include "queue.hpp" #include "unified-runtime/ur_api.h" -extern "C" { +namespace ur::level_zero::v1 { + ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted = nullptr); ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue, @@ -38,7 +39,6 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue, bool ForceDisableProfiling, bool InterruptBasedEventEnabled, std::optional IsInternal = std::nullopt); -} // extern "C" // This is an experimental option that allows to disable caching of events in // the context. @@ -126,7 +126,7 @@ struct ur_ze_event_list_t { void printZeEventList(const ur_ze_event_list_t &PiZeEventList); -struct ur_event_handle_t_ : ur_object { +struct ur_event_handle_t_ : ur_object_t { ur_event_handle_t_(ze_event_handle_t ZeEvent, ze_event_pool_handle_t ZeEventPool, ur_context_handle_t Context, ur_command_t CommandType, @@ -265,7 +265,6 @@ struct ur_event_handle_t_ : ur_object { ur::RefCount RefCount; }; - // Helper function to implement zeHostSynchronize. // The behavior is to avoid infinite wait during host sync under ZE_DEBUG. // This allows for a much more responsive debugging of hangs. @@ -313,3 +312,5 @@ static const EventsScope DeviceEventsSetting = [] { // with the modern GPU drivers. return AllHostVisible; }(); + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/graph.cpp b/unified-runtime/source/adapters/level_zero/graph.cpp index 3ab7e25a2cc59..a56c6beb9362a 100644 --- a/unified-runtime/source/adapters/level_zero/graph.cpp +++ b/unified-runtime/source/adapters/level_zero/graph.cpp @@ -11,17 +11,17 @@ #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v1 { -ur_result_t urGraphCreateExp(ur_context_handle_t /* hContext */, - ur_exp_graph_handle_t * /* phGraph */) { +ur_result_t urGraphCreateExp(::ur_context_handle_t /* hContext */, + ::ur_exp_graph_handle_t * /* phGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t /* hGraph */) { +ur_result_t urGraphDestroyExp(::ur_exp_graph_handle_t /* hGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -29,14 +29,14 @@ ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t /* hGraph */) { } ur_result_t urGraphExecutableGraphDestroyExp( - ur_exp_executable_graph_handle_t /* hExecutableGraph */) { + ::ur_exp_executable_graph_handle_t /* hExecutableGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t /* hGraph */, +ur_result_t urGraphIsEmptyExp(::ur_exp_graph_handle_t /* hGraph */, bool * /* pIsEmpty */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -44,7 +44,7 @@ ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t /* hGraph */, return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t /* hGraph */, +ur_result_t urGraphGetIdExp(::ur_exp_graph_handle_t /* hGraph */, uint64_t * /* pGraphId */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -53,7 +53,7 @@ ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t /* hGraph */, } ur_result_t urGraphSetDestructionCallbackExp( - ur_exp_graph_handle_t /* hGraph */, + ::ur_exp_graph_handle_t /* hGraph */, ur_exp_graph_destruction_callback_t /* pfnCallback */, void * /* pUserData */) { UR_LOG_LEGACY(ERR, @@ -62,7 +62,7 @@ ur_result_t urGraphSetDestructionCallbackExp( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t /* hGraph */, +ur_result_t urGraphDumpContentsExp(::ur_exp_graph_handle_t /* hGraph */, const char * /* pDotFilePath */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -71,8 +71,8 @@ ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t /* hGraph */, } ur_result_t urGraphInstantiateGraphExp( - ur_exp_graph_handle_t /* hGraph */, - ur_exp_executable_graph_handle_t * /* phExecutableGraph */) { + ::ur_exp_graph_handle_t /* hGraph */, + ::ur_exp_executable_graph_handle_t * /* phExecutableGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -80,7 +80,7 @@ ur_result_t urGraphInstantiateGraphExp( } ur_result_t -urGraphGetNativeHandleExp(ur_exp_graph_handle_t /* hGraph */, +urGraphGetNativeHandleExp(::ur_exp_graph_handle_t /* hGraph */, ur_native_handle_t * /* phNativeGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -89,7 +89,7 @@ urGraphGetNativeHandleExp(ur_exp_graph_handle_t /* hGraph */, } ur_result_t urGraphExecutableGraphGetNativeHandleExp( - ur_exp_executable_graph_handle_t /* hExecutableGraph */, + ::ur_exp_executable_graph_handle_t /* hExecutableGraph */, ur_native_handle_t * /* phNativeExecutableGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -97,4 +97,4 @@ ur_result_t urGraphExecutableGraphGetNativeHandleExp( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/image.cpp b/unified-runtime/source/adapters/level_zero/image.cpp index b21a61e678609..4002dbd5aec3f 100644 --- a/unified-runtime/source/adapters/level_zero/image.cpp +++ b/unified-runtime/source/adapters/level_zero/image.cpp @@ -8,32 +8,35 @@ //===----------------------------------------------------------------------===// #include "common.hpp" +#include "common/image_common.hpp" +#include "common/platform.hpp" +#include "common/sampler.hpp" #include "context.hpp" #include "event.hpp" -#include "image_common.hpp" #include "logger/ur_logger.hpp" #include "memory.hpp" -#include "sampler.hpp" #include "unified-runtime/ur_api.h" #include "ur_interface_loader.hpp" #include "loader/ze_loader.h" -namespace ur::level_zero { +namespace ur::level_zero::v1 { ur_result_t urBindlessImagesImageCopyExp( - ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + ::ur_queue_handle_t hQueueOpque, const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, const ur_image_format_t *pSrcImageFormat, const ur_image_format_t *pDstImageFormat, ur_exp_image_copy_region_t *pCopyRegion, ur_exp_image_copy_flags_t imageCopyFlags, ur_exp_image_copy_input_types_t imageCopyInputTypes, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) { + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitListOpque, + ::ur_event_handle_t *phEventOpque) { + auto hQueue = v1_cast(hQueueOpque); std::scoped_lock Lock(hQueue->Mutex); - UR_ASSERT(hQueue, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(hQueueOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(pDst && pSrc && pSrcImageFormat && pSrcImageDesc && pDstImageDesc && pCopyRegion, UR_RESULT_ERROR_INVALID_NULL_POINTER); @@ -52,9 +55,9 @@ ur_result_t urBindlessImagesImageCopyExp( // This is wild and the change is only added because we continue to test // both V1 and V2 L0 adapters for all HW, regardless of the default adapter // there. - bool UseCopyEngine = - hQueue->useCopyEngine(/*PreferCopyEngine*/ imageCopyInputTypes == - UR_EXP_IMAGE_COPY_INPUT_TYPES_MEM_TO_MEM); + bool UseCopyEngine = hQueue->useCopyEngine( + /*PreferCopyEngine*/ imageCopyInputTypes == + UR_EXP_IMAGE_COPY_INPUT_TYPES_MEM_TO_MEM); // Due to the limitation of the copy engine, disable usage of Copy Engine // Given 3 channel image if (is3ChannelOrder( @@ -64,9 +67,12 @@ ur_result_t urBindlessImagesImageCopyExp( UseCopyEngine = false; } + auto phEventWaitListInternal = v1_cast(phEventWaitListOpque); + auto phEventInternal = v1_cast(phEventOpque); + ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - numEventsInWaitList, phEventWaitList, hQueue, UseCopyEngine)); + numEventsInWaitList, phEventWaitListInternal, hQueue, UseCopyEngine)); bool Blocking = false; // We want to batch these commands to avoid extra submissions (costly) @@ -75,18 +81,18 @@ ur_result_t urBindlessImagesImageCopyExp( // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(hQueue->Context->getAvailableCommandList( - hQueue, CommandList, UseCopyEngine, numEventsInWaitList, phEventWaitList, - OkToBatch, nullptr /*ForcedCmdQueue*/)); + hQueue, CommandList, UseCopyEngine, numEventsInWaitList, + phEventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = phEvent == nullptr; - ur_event_handle_t *Event = phEvent ? phEvent : &InternalEvent; + bool IsInternal = phEventInternal == nullptr; + ur_event_handle_t *Event = phEventInternal ? phEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(hQueue, Event, UR_COMMAND_MEM_IMAGE_COPY, CommandList, IsInternal, /*IsMultiDevice*/ false)); UR_CALL(setSignalEvent(hQueue, UseCopyEngine, &ZeEvent, Event, - numEventsInWaitList, phEventWaitList, + numEventsInWaitList, phEventWaitListInternal, CommandList->second.ZeQueue)); (*Event)->WaitList = TmpWaitList; @@ -105,9 +111,12 @@ ur_result_t urBindlessImagesImageCopyExp( } ur_result_t urBindlessImagesWaitExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, - bool hasValue, uint64_t waitValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + ::ur_queue_handle_t hQueueOpque, + ::ur_exp_external_semaphore_handle_t hSemaphoreOpque, bool hasValue, + uint64_t waitValue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitListOpque, + ::ur_event_handle_t *phEventOpque) { + auto hQueue = v1_cast(hQueueOpque); auto UrPlatform = hQueue->Context->getPlatform(); if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { UR_LOG_LEGACY(ERR, @@ -121,26 +130,29 @@ ur_result_t urBindlessImagesWaitExternalSemaphoreExp( // We want to batch these commands to avoid extra submissions (costly) bool OkToBatch = true; + auto phEventWaitListInternal = v1_cast(phEventWaitListOpque); + auto phEventInternal = v1_cast(phEventOpque); + ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - numEventsInWaitList, phEventWaitList, hQueue, UseCopyEngine)); + numEventsInWaitList, phEventWaitListInternal, hQueue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(hQueue->Context->getAvailableCommandList( - hQueue, CommandList, UseCopyEngine, numEventsInWaitList, phEventWaitList, - OkToBatch, nullptr /*ForcedCmdQueue*/)); + hQueue, CommandList, UseCopyEngine, numEventsInWaitList, + phEventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = phEvent == nullptr; - ur_event_handle_t *Event = phEvent ? phEvent : &InternalEvent; + bool IsInternal = phEventInternal == nullptr; + ur_event_handle_t *Event = phEventInternal ? phEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(hQueue, Event, UR_COMMAND_EXTERNAL_SEMAPHORE_WAIT_EXP, CommandList, IsInternal, /*IsMultiDevice*/ false)); UR_CALL(setSignalEvent(hQueue, UseCopyEngine, &ZeEvent, Event, - numEventsInWaitList, phEventWaitList, + numEventsInWaitList, phEventWaitListInternal, CommandList->second.ZeQueue)); (*Event)->WaitList = TmpWaitList; @@ -151,7 +163,7 @@ ur_result_t urBindlessImagesWaitExternalSemaphoreExp( ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_WAIT_PARAMS_EXT, nullptr, 0}; WaitParams.value = hasValue ? waitValue : 0; ze_external_semaphore_ext_handle_t hExtSemaphore = - reinterpret_cast(hSemaphore); + reinterpret_cast(hSemaphoreOpque); ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt .zexCommandListAppendWaitExternalSemaphoresExp, (ZeCommandList, 1, &hExtSemaphore, &WaitParams, ZeEvent, @@ -163,9 +175,12 @@ ur_result_t urBindlessImagesWaitExternalSemaphoreExp( } ur_result_t urBindlessImagesSignalExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, - bool hasValue, uint64_t signalValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + ::ur_queue_handle_t hQueueOpque, + ::ur_exp_external_semaphore_handle_t hSemaphoreOpque, bool hasValue, + uint64_t signalValue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitListOpque, + ::ur_event_handle_t *phEventOpque) { + auto hQueue = v1_cast(hQueueOpque); auto UrPlatform = hQueue->Context->getPlatform(); if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { UR_LOG_LEGACY(ERR, @@ -179,26 +194,29 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp( // We want to batch these commands to avoid extra submissions (costly) bool OkToBatch = true; + auto phEventWaitListInternal = v1_cast(phEventWaitListOpque); + auto phEventInternal = v1_cast(phEventOpque); + ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - numEventsInWaitList, phEventWaitList, hQueue, UseCopyEngine)); + numEventsInWaitList, phEventWaitListInternal, hQueue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(hQueue->Context->getAvailableCommandList( - hQueue, CommandList, UseCopyEngine, numEventsInWaitList, phEventWaitList, - OkToBatch, nullptr /*ForcedCmdQueue*/)); + hQueue, CommandList, UseCopyEngine, numEventsInWaitList, + phEventWaitListInternal, OkToBatch, nullptr /*ForcedCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = phEvent == nullptr; - ur_event_handle_t *Event = phEvent ? phEvent : &InternalEvent; + bool IsInternal = phEventInternal == nullptr; + ur_event_handle_t *Event = phEventInternal ? phEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(hQueue, Event, UR_COMMAND_EXTERNAL_SEMAPHORE_SIGNAL_EXP, CommandList, IsInternal, /*IsMultiDevice*/ false)); UR_CALL(setSignalEvent(hQueue, UseCopyEngine, &ZeEvent, Event, - numEventsInWaitList, phEventWaitList, + numEventsInWaitList, phEventWaitListInternal, CommandList->second.ZeQueue)); (*Event)->WaitList = TmpWaitList; @@ -209,7 +227,7 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp( ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS_EXT, nullptr, 0}; SignalParams.value = hasValue ? signalValue : 0; ze_external_semaphore_ext_handle_t hExtSemaphore = - reinterpret_cast(hSemaphore); + reinterpret_cast(hSemaphoreOpque); ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt .zexCommandListAppendSignalExternalSemaphoresExp, @@ -221,4 +239,4 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp( return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index de24587cf5865..178f22a7947a4 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -12,7 +12,12 @@ #include "unified-runtime/ur_api.h" #include "ur_interface_loader.hpp" -#include "helpers/kernel_helpers.hpp" +#include "common/api.hpp" +#include "common/helpers/kernel_helpers.hpp" +#include "common/platform.hpp" +#include "common/sampler.hpp" + +namespace ur::level_zero::v1 { ur_result_t getZeKernel(ze_device_handle_t hDevice, ur_kernel_handle_t hKernel, ze_kernel_handle_t *phZeKernel) { @@ -30,12 +35,11 @@ ur_result_t getZeKernel(ze_device_handle_t hDevice, ur_kernel_handle_t hKernel, return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - ur_result_t urKernelGetSuggestedLocalWorkSize( - ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t workDim, - [[maybe_unused]] const size_t *pGlobalWorkOffset, + ::ur_kernel_handle_t hKernelOpque, ::ur_queue_handle_t hQueueOpque, + uint32_t workDim, [[maybe_unused]] const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, size_t *pSuggestedLocalWorkSize) { + auto hQueue = v1_cast(hQueueOpque); UR_ASSERT(workDim > 0, UR_RESULT_ERROR_INVALID_WORK_DIMENSION); UR_ASSERT(workDim < 4, UR_RESULT_ERROR_INVALID_WORK_DIMENSION); UR_ASSERT(pSuggestedLocalWorkSize != nullptr, @@ -46,7 +50,8 @@ ur_result_t urKernelGetSuggestedLocalWorkSize( std::copy(pGlobalWorkSize, pGlobalWorkSize + workDim, GlobalWorkSize3D); ze_kernel_handle_t ZeKernel{}; - UR_CALL(getZeKernel(hQueue->Device->ZeDevice, hKernel, &ZeKernel)); + UR_CALL( + getZeKernel(hQueue->Device->ZeDevice, v1_cast(hKernelOpque), &ZeKernel)); UR_CALL(getSuggestedLocalWorkSize(hQueue->Device, ZeKernel, GlobalWorkSize3D, LocalWorkSize)); @@ -56,13 +61,13 @@ ur_result_t urKernelGetSuggestedLocalWorkSize( } ur_result_t urKernelGetSuggestedLocalWorkSizeWithArgs( - ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t workDim, - const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, - [[maybe_unused]] uint32_t numArgs, + ::ur_kernel_handle_t hKernelOpque, ::ur_queue_handle_t hQueueOpque, + uint32_t workDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, [[maybe_unused]] uint32_t numArgs, [[maybe_unused]] const ur_exp_kernel_arg_properties_t *pArgs, size_t *pSuggestedLocalWorkSize) { - return ur::level_zero::urKernelGetSuggestedLocalWorkSize( - hKernel, hQueue, workDim, pGlobalWorkOffset, pGlobalWorkSize, + return ur::level_zero::v1::urKernelGetSuggestedLocalWorkSize( + hKernelOpque, hQueueOpque, workDim, pGlobalWorkOffset, pGlobalWorkSize, pSuggestedLocalWorkSize); } @@ -126,7 +131,7 @@ ur_result_t urKernelSetArgMemObjHelper( return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX; } - ur_mem_handle_t_ *UrMem = ur_cast(ArgValue); + auto *UrMem = ArgValue; ur_mem_handle_t_::access_mode_t UrAccessMode = ur_mem_handle_t_::read_write; if (Properties) { @@ -155,20 +160,19 @@ ur_result_t urKernelSetArgMemObjHelper( // forward declaration of the old enqueue function to call after setting all the // arguments -static ur_result_t -urEnqueueKernelLaunch(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel, - uint32_t WorkDim, const size_t *GlobalWorkOffset, - const size_t *GlobalWorkSize, const size_t *LocalWorkSize, - const ur_kernel_launch_ext_properties_t *LaunchPropList, - uint32_t NumEventsInWaitList, - const ur_event_handle_t *EventWaitList, - ur_event_handle_t *OutEvent); +static ur_result_t urEnqueueKernelLaunch( + ::ur_queue_handle_t QueueOpque, ::ur_kernel_handle_t KernelOpque, + uint32_t WorkDim, const size_t *GlobalWorkOffset, + const size_t *GlobalWorkSize, const size_t *LocalWorkSize, + const ur_kernel_launch_ext_properties_t *LaunchPropList, + uint32_t NumEventsInWaitList, const ::ur_event_handle_t *EventWaitListOpque, + ::ur_event_handle_t *OutEventOpque); ur_result_t urEnqueueKernelLaunchWithArgsExp( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] number of dimensions, from 1 to 3, to specify the global and /// work-group work-items uint32_t workDim, @@ -196,10 +200,11 @@ ur_result_t urEnqueueKernelLaunchWithArgsExp( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitListOpque, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEventOpque) { + auto Kernel = v1_cast(KernelOpque); { std::scoped_lock Guard(Kernel->Mutex); for (uint32_t i = 0; i < NumArgs; i++) { @@ -220,13 +225,15 @@ ur_result_t urEnqueueKernelLaunchWithArgsExp( ur_kernel_arg_mem_obj_properties_t Properties = { UR_STRUCTURE_TYPE_KERNEL_ARG_MEM_OBJ_PROPERTIES, nullptr, Args[i].value.memObjTuple.flags}; - UR_CALL(urKernelSetArgMemObjHelper(Kernel, Args[i].index, &Properties, - Args[i].value.memObjTuple.hMem)); + UR_CALL(urKernelSetArgMemObjHelper( + Kernel, Args[i].index, &Properties, + v1_cast(Args[i].value.memObjTuple.hMem))); break; } case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { - UR_CALL(urKernelSetArgValueHelper(Kernel, Args[i].index, Args[i].size, - &Args[i].value.sampler->ZeSampler)); + UR_CALL(urKernelSetArgValueHelper( + Kernel, Args[i].index, Args[i].size, + &ur::level_zero::common_cast(Args[i].value.sampler)->ZeSampler)); break; } default: @@ -235,16 +242,17 @@ ur_result_t urEnqueueKernelLaunchWithArgsExp( } } // Normalize so each dimension has at least one work item - return urEnqueueKernelLaunch(Queue, Kernel, workDim, GlobalWorkOffset, - GlobalWorkSize, LocalWorkSize, LaunchPropList, - NumEventsInWaitList, EventWaitList, OutEvent); + return urEnqueueKernelLaunch(QueueOpque, KernelOpque, workDim, + GlobalWorkOffset, GlobalWorkSize, LocalWorkSize, + LaunchPropList, NumEventsInWaitList, + EventWaitListOpque, OutEventOpque); } static ur_result_t urEnqueueKernelLaunch( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] number of dimensions, from 1 to 3, to specify the global and /// work-group work-items uint32_t WorkDim, @@ -268,10 +276,14 @@ static ur_result_t urEnqueueKernelLaunch( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitListOpque, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEventOpque) { + auto Queue = v1_cast(QueueOpque); + auto Kernel = v1_cast(KernelOpque); + auto EventWaitListInternal = v1_cast(EventWaitListOpque); + auto OutEventInternal = v1_cast(OutEventOpque); using ZeKernelLaunchFuncT = ze_result_t (*)( ze_command_list_handle_t, ze_kernel_handle_t, const ze_group_count_t *, ze_event_handle_t, uint32_t, ze_event_handle_t *); @@ -310,7 +322,7 @@ static ur_result_t urEnqueueKernelLaunch( std::scoped_lock Lock( Queue->Mutex, Kernel->Mutex, Kernel->Program->Mutex); if (GlobalWorkOffset != NULL) { - UR_CALL(setKernelGlobalOffset(Queue->Context, ZeKernel, WorkDim, + UR_CALL(setKernelGlobalOffset(v1_cast(Queue->Context), ZeKernel, WorkDim, GlobalWorkOffset)); } @@ -324,7 +336,7 @@ static ur_result_t urEnqueueKernelLaunch( ur_mem_handle_t MemObj = *MemObjPtr; if (MemObj) { UR_CALL(MemObj->getZeHandlePtr(ZeHandlePtr, Arg.AccessMode, - Queue->Device, EventWaitList, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); } } else { @@ -347,23 +359,25 @@ static ur_result_t urEnqueueKernelLaunch( bool UseCopyEngine = false; ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, - true /* AllowBatching */, nullptr /*ForcedCmdQueue*/)); + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, true /* AllowBatching */, + nullptr /*ForcedCmdQueue*/)); ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent{}; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_KERNEL_LAUNCH, CommandList, IsInternal, false)); UR_CALL(setSignalEvent(Queue, UseCopyEngine, &ZeEvent, Event, - NumEventsInWaitList, EventWaitList, + NumEventsInWaitList, EventWaitListInternal, CommandList->second.ZeQueue)); (*Event)->WaitList = TmpWaitList; @@ -375,7 +389,7 @@ static ur_result_t urEnqueueKernelLaunch( // is in use. Once the event has been signalled, the code in // CleanupCompletedEvent(Event) will do a urKernelRelease to update the // reference count on the kernel, using the kernel saved in CommandData. - UR_CALL(ur::level_zero::urKernelRetain(Kernel)); + UR_CALL(ur::level_zero::v1::urKernelRetain(KernelOpque)); // Add to list of kernels to be submitted if (IndirectAccessTrackingEnabled) @@ -423,9 +437,9 @@ static ur_result_t urEnqueueKernelLaunch( ur_result_t urEnqueueDeviceGlobalVariableWrite( /// [in] handle of the queue to submit to. - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the program containing the device global variable. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] the unique identifier for the device global variable. const char *Name, /// [in] indicates if this operation should block. @@ -442,16 +456,17 @@ ur_result_t urEnqueueDeviceGlobalVariableWrite( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitListOpque, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t *Event) { + ::ur_event_handle_t *EventOpque) { + auto Queue = v1_cast(QueueOpque); std::scoped_lock lock(Queue->Mutex); // Find global variable pointer size_t GlobalVarSize = 0; void *GlobalVarPtr = nullptr; ze_module_handle_t ZeModule = - Program->getZeModuleHandle(Queue->Device->ZeDevice); + common_cast(ProgramOpque)->getZeModuleHandle(Queue->Device->ZeDevice); ZE2UR_CALL(zeModuleGetGlobalPointer, (ZeModule, Name, &GlobalVarSize, &GlobalVarPtr)); if (GlobalVarSize < Offset + Count) { @@ -463,10 +478,11 @@ ur_result_t urEnqueueDeviceGlobalVariableWrite( // Copy engine is preferred only for host to device transfer. // Device to device transfers run faster on compute engines. - bool PreferCopyEngine = !IsDevicePointer(Queue->Context, Src); + auto Context = Queue->Context; + bool PreferCopyEngine = !IsDevicePointer(Context, Src); // For better performance, Copy Engines are not preferred given Shared // pointers on DG2. - if (Queue->Device->isDG2() && IsSharedPointer(Queue->Context, Src)) { + if (Queue->Device->isDG2() && IsSharedPointer(Context, Src)) { PreferCopyEngine = false; } @@ -476,14 +492,15 @@ ur_result_t urEnqueueDeviceGlobalVariableWrite( return enqueueMemCopyHelper(UR_COMMAND_DEVICE_GLOBAL_VARIABLE_WRITE, Queue, ur_cast(GlobalVarPtr) + Offset, BlockingWrite, Count, Src, NumEventsInWaitList, - EventWaitList, Event, PreferCopyEngine); + v1_cast(EventWaitListOpque), v1_cast(EventOpque), + PreferCopyEngine); } ur_result_t urEnqueueDeviceGlobalVariableRead( /// [in] handle of the queue to submit to. - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the program containing the device global variable. - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, const char /// [in] the unique identifier for the device global variable. *Name, @@ -501,13 +518,14 @@ ur_result_t urEnqueueDeviceGlobalVariableRead( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitListOpque, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t *Event) { + ::ur_event_handle_t *EventOpque) { + auto Queue = v1_cast(QueueOpque); std::scoped_lock lock(Queue->Mutex); ze_module_handle_t ZeModule = - Program->getZeModuleHandle(Queue->Device->ZeDevice); + common_cast(ProgramOpque)->getZeModuleHandle(Queue->Device->ZeDevice); // Find global variable pointer size_t GlobalVarSize = 0; void *GlobalVarPtr = nullptr; @@ -522,10 +540,11 @@ ur_result_t urEnqueueDeviceGlobalVariableRead( // Copy engine is preferred only for host to device transfer. // Device to device transfers run faster on compute engines. - bool PreferCopyEngine = !IsDevicePointer(Queue->Context, Dst); + auto Context = Queue->Context; + bool PreferCopyEngine = !IsDevicePointer(Context, Dst); // For better performance, Copy Engines are not preferred given Shared // pointers on DG2. - if (Queue->Device->isDG2() && IsSharedPointer(Queue->Context, Dst)) { + if (Queue->Device->isDG2() && IsSharedPointer(Context, Dst)) { PreferCopyEngine = false; } @@ -535,20 +554,22 @@ ur_result_t urEnqueueDeviceGlobalVariableRead( return enqueueMemCopyHelper( UR_COMMAND_DEVICE_GLOBAL_VARIABLE_READ, Queue, Dst, BlockingRead, Count, ur_cast(GlobalVarPtr) + Offset, NumEventsInWaitList, - EventWaitList, Event, PreferCopyEngine); + v1_cast(EventWaitListOpque), v1_cast(EventOpque), PreferCopyEngine); } ur_result_t urKernelCreate( /// [in] handle of the program instance - ur_program_handle_t Program, + ::ur_program_handle_t ProgramOpque, /// [in] pointer to null-terminated string. const char *KernelName, /// [out] pointer to handle of kernel object created. - ur_kernel_handle_t *RetKernel) { + ::ur_kernel_handle_t *RetKernelOpque) { + auto Program = common_cast(ProgramOpque); + auto RetKernel = v1_cast(RetKernelOpque); std::shared_lock Guard(Program->Mutex); try { ur_kernel_handle_t_ *UrKernel = new ur_kernel_handle_t_(true, Program); - *RetKernel = reinterpret_cast(UrKernel); + *RetKernel = UrKernel; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -607,7 +628,7 @@ ur_result_t urKernelCreate( ur_result_t urKernelSetArgValue( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] argument index in range [0, num args - 1] uint32_t ArgIndex, /// [in] size of argument type @@ -616,8 +637,9 @@ ur_result_t urKernelSetArgValue( const ur_kernel_arg_value_properties_t * /*Properties*/, /// [in] argument value represented as matching arg type. const void *PArgValue) { + auto Kernel = v1_cast(KernelOpque); - UR_ASSERT(Kernel, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(KernelOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); if (ArgIndex > Kernel->ZeKernelProperties->numKernelArgs - 1) { return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX; @@ -638,7 +660,7 @@ ur_result_t urKernelSetArgValue( ur_result_t urKernelSetArgLocal( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] argument index in range [0, num args - 1] uint32_t ArgIndex, /// [in] size of the local buffer to be allocated by the runtime @@ -646,15 +668,15 @@ ur_result_t urKernelSetArgLocal( /// [in][optional] argument properties const ur_kernel_arg_local_properties_t * /*Properties*/) { - UR_CALL(ur::level_zero::urKernelSetArgValue(Kernel, ArgIndex, ArgSize, - nullptr, nullptr)); + UR_CALL(ur::level_zero::v1::urKernelSetArgValue(KernelOpque, ArgIndex, + ArgSize, nullptr, nullptr)); return UR_RESULT_SUCCESS; } ur_result_t urKernelGetInfo( /// [in] handle of the Kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] name of the Kernel property to query ur_kernel_info_t ParamName, /// [in] the size of the Kernel property value. @@ -667,15 +689,16 @@ ur_result_t urKernelGetInfo( /// [out][optional] pointer to the actual size in bytes of data being /// queried by propName. size_t *PropSizeRet) { + auto Kernel = v1_cast(KernelOpque); UrReturnHelper ReturnValue(PropSize, KernelInfo, PropSizeRet); std::shared_lock Guard(Kernel->Mutex); switch (ParamName) { case UR_KERNEL_INFO_CONTEXT: - return ReturnValue(ur_context_handle_t{Kernel->Program->Context}); + return ReturnValue(::ur_context_handle_t{Kernel->Program->Context}); case UR_KERNEL_INFO_PROGRAM: - return ReturnValue(ur_program_handle_t{Kernel->Program}); + return ReturnValue(::ur_program_handle_t{common_cast(Kernel->Program)}); case UR_KERNEL_INFO_FUNCTION_NAME: try { std::string &KernelName = Kernel->ZeKernelName.get(); @@ -737,9 +760,9 @@ ur_result_t urKernelGetInfo( ur_result_t urKernelGetGroupInfo( /// [in] handle of the Kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] handle of the Device object - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in] name of the work Group property to query ur_kernel_group_info_t ParamName, /// [in] size of the Kernel Work Group property value @@ -750,6 +773,8 @@ ur_result_t urKernelGetGroupInfo( /// [out][optional] pointer to the actual size in bytes of data being /// queried by propName. size_t *ParamValueSizeRet) { + auto Kernel = v1_cast(KernelOpque); + auto Device = common_cast(DeviceOpque); UrReturnHelper ReturnValue(ParamValueSize, ParamValue, ParamValueSizeRet); std::shared_lock Guard(Kernel->Mutex); @@ -825,9 +850,9 @@ ur_result_t urKernelGetGroupInfo( ur_result_t urKernelGetSubGroupInfo( /// [in] handle of the Kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] handle of the Device object - ur_device_handle_t /*Device*/, + ::ur_device_handle_t /*Device*/, /// [in] name of the SubGroup property to query ur_kernel_sub_group_info_t PropName, /// [in] size of the Kernel SubGroup property value @@ -838,6 +863,7 @@ ur_result_t urKernelGetSubGroupInfo( /// [out][optional] pointer to the actual size in bytes of data being /// queried by propName. size_t *PropSizeRet) { + auto Kernel = v1_cast(KernelOpque); UrReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet); @@ -859,7 +885,8 @@ ur_result_t urKernelGetSubGroupInfo( ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain - ur_kernel_handle_t Kernel) { + ::ur_kernel_handle_t KernelOpque) { + auto Kernel = v1_cast(KernelOpque); Kernel->RefCount.retain(); return UR_RESULT_SUCCESS; @@ -867,7 +894,8 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release - ur_kernel_handle_t Kernel) { + ::ur_kernel_handle_t KernelOpque) { + auto Kernel = v1_cast(KernelOpque); if (!Kernel->RefCount.release()) return UR_RESULT_SUCCESS; @@ -888,7 +916,7 @@ ur_result_t urKernelRelease( } Kernel->ZeKernelMap.clear(); if (IndirectAccessTrackingEnabled) { - UR_CALL(ur::level_zero::urContextRelease(KernelProgram->Context)); + UR_CALL(ur::level_zero::v1::urContextRelease(KernelProgram->Context)); } // do a release on the program this kernel was part of without delete of the // program handle @@ -901,7 +929,7 @@ ur_result_t urKernelRelease( ur_result_t urKernelSetArgPointer( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] argument index in range [0, num args - 1] uint32_t ArgIndex, /// [in][optional] argument properties @@ -909,14 +937,15 @@ ur_result_t urKernelSetArgPointer( /// [in][optional] SVM pointer to memory location holding the argument /// value. If null then argument value is considered null. const void *ArgValue) { - UR_ASSERT(Kernel, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + auto Kernel = v1_cast(KernelOpque); + UR_ASSERT(KernelOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); { std::scoped_lock Guard(Kernel->Mutex); // In multi-device context instead of setting pointer arguments immediately // across all device kernels, store them as pending so they can be resolved // per-device at enqueue time. This ensures the correct handle is used for // the device of the queue. - if (Kernel->Program->Context->getDevices().size() > 1) { + if (v1_cast(Kernel->Program->Context)->getDevices().size() > 1) { if (ArgIndex > Kernel->ZeKernelProperties->numKernelArgs - 1) { return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX; } @@ -928,14 +957,14 @@ ur_result_t urKernelSetArgPointer( } // KernelSetArgValue is expecting a pointer to the argument - UR_CALL(ur::level_zero::urKernelSetArgValue( - Kernel, ArgIndex, sizeof(const void *), nullptr, &ArgValue)); + UR_CALL(ur::level_zero::v1::urKernelSetArgValue( + KernelOpque, ArgIndex, sizeof(const void *), nullptr, &ArgValue)); return UR_RESULT_SUCCESS; } ur_result_t urKernelSetExecInfo( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] name of the execution attribute ur_kernel_exec_info_t PropName, /// [in] size in byte the attribute value @@ -945,6 +974,7 @@ ur_result_t urKernelSetExecInfo( /// [in][range(0, propSize)] pointer to memory location holding the property /// value. const void *PropValue) { + auto Kernel = v1_cast(KernelOpque); std::scoped_lock Guard(Kernel->Mutex); for (auto &ZeKernel : Kernel->ZeKernels) { @@ -983,32 +1013,35 @@ ur_result_t urKernelSetExecInfo( ur_result_t urKernelSetArgSampler( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] argument index in range [0, num args - 1] uint32_t ArgIndex, /// [in][optional] argument properties const ur_kernel_arg_sampler_properties_t * /*Properties*/, /// [in] handle of Sampler object. - ur_sampler_handle_t ArgValue) { + ::ur_sampler_handle_t ArgValueOpque) { + auto Kernel = v1_cast(KernelOpque); std::scoped_lock Guard(Kernel->Mutex); if (ArgIndex > Kernel->ZeKernelProperties->numKernelArgs - 1) { return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX; } - ZE2UR_CALL(zeKernelSetArgumentValue, (Kernel->ZeKernel, ArgIndex, - sizeof(void *), &ArgValue->ZeSampler)); + ZE2UR_CALL(zeKernelSetArgumentValue, + (Kernel->ZeKernel, ArgIndex, sizeof(void *), + &ur::level_zero::common_cast(ArgValueOpque)->ZeSampler)); return UR_RESULT_SUCCESS; } ur_result_t urKernelSetArgMemObj( /// [in] handle of the kernel object - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [in] argument index in range [0, num args - 1] uint32_t ArgIndex, /// [in][optional] pointer to Memory object properties. const ur_kernel_arg_mem_obj_properties_t *Properties, /// [in][optional] handle of Memory object. - ur_mem_handle_t ArgValue) { + ::ur_mem_handle_t ArgValueOpque) { + auto Kernel = v1_cast(KernelOpque); std::scoped_lock Guard(Kernel->Mutex); // The ArgValue may be a NULL pointer in which case a NULL value is used for @@ -1018,7 +1051,7 @@ ur_result_t urKernelSetArgMemObj( return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX; } - ur_mem_handle_t_ *UrMem = ur_cast(ArgValue); + auto UrMem = v1_cast(ArgValueOpque); ur_mem_handle_t_::access_mode_t UrAccessMode = ur_mem_handle_t_::read_write; if (Properties) { @@ -1045,9 +1078,10 @@ ur_result_t urKernelSetArgMemObj( ur_result_t urKernelGetNativeHandle( /// [in] handle of the kernel. - ur_kernel_handle_t Kernel, + ::ur_kernel_handle_t KernelOpque, /// [out] a pointer to the native handle of the kernel. - ur_native_handle_t *NativeKernel) { + ::ur_native_handle_t *NativeKernel) { + auto Kernel = v1_cast(KernelOpque); std::shared_lock Guard(Kernel->Mutex); *NativeKernel = reinterpret_cast(Kernel->ZeKernel); @@ -1055,9 +1089,11 @@ ur_result_t urKernelGetNativeHandle( } ur_result_t urKernelSuggestMaxCooperativeGroupCount( - ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, uint32_t workDim, - const size_t *pLocalWorkSize, size_t dynamicSharedMemorySize, - uint32_t *pGroupCountRet) { + ::ur_kernel_handle_t hKernelOpque, ::ur_device_handle_t hDeviceOpque, + uint32_t workDim, const size_t *pLocalWorkSize, + size_t dynamicSharedMemorySize, uint32_t *pGroupCountRet) { + auto hKernel = v1_cast(hKernelOpque); + auto hDevice = common_cast(hDeviceOpque); (void)dynamicSharedMemorySize; std::shared_lock Guard(hKernel->Mutex); @@ -1079,12 +1115,13 @@ ur_result_t urKernelSuggestMaxCooperativeGroupCount( ur_result_t urKernelCreateWithNativeHandle( /// [in] the native handle of the kernel. - ur_native_handle_t NativeKernel, + ::ur_native_handle_t NativeKernel, /// [in] handle of the context object - ur_context_handle_t Context, ur_program_handle_t Program, + ::ur_context_handle_t ContextOpque, ::ur_program_handle_t ProgramOpque, const ur_kernel_native_properties_t *Properties, /// [out] pointer to the handle of the kernel object created. - ur_kernel_handle_t *RetKernel) { + ::ur_kernel_handle_t *RetKernelOpque) { + auto Program = common_cast(ProgramOpque); if (!Program) { return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } @@ -1092,14 +1129,15 @@ ur_result_t urKernelCreateWithNativeHandle( ur_kernel_handle_t_ *Kernel = nullptr; try { auto OwnNativeHandle = Properties ? Properties->isNativeHandleOwned : false; - Kernel = new ur_kernel_handle_t_(ZeKernel, OwnNativeHandle, Context); + Kernel = new ur_kernel_handle_t_(ZeKernel, OwnNativeHandle, + v1_cast(ContextOpque)); if (OwnNativeHandle) { // If ownership is passed to the adapter we need to pass the kernel // to this vector which is then used during ZeKernelRelease. Kernel->ZeKernels.push_back(ZeKernel); } - *RetKernel = reinterpret_cast(Kernel); + *v1_cast(RetKernelOpque) = Kernel; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -1115,7 +1153,7 @@ ur_result_t urKernelCreateWithNativeHandle( ur_result_t urKernelSetSpecializationConstants( /// [in] handle of the kernel object - ur_kernel_handle_t /*Kernel*/, + ::ur_kernel_handle_t /*Kernel*/, /// [in] the number of elements in the pSpecConstants array uint32_t /*Count*/, const ur_specialization_constant_info_t @@ -1127,15 +1165,13 @@ ur_result_t urKernelSetSpecializationConstants( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero - ur_result_t ur_kernel_handle_t_::initialize() { // Retain the program and context to show it's used by this kernel. - UR_CALL(ur::level_zero::urProgramRetain(Program)); + UR_CALL(ur::level_zero::urProgramRetain(common_cast(Program))); if (IndirectAccessTrackingEnabled) // TODO: do piContextRetain without the guard - UR_CALL(ur::level_zero::urContextRetain(Program->Context)); + UR_CALL(ur::level_zero::v1::urContextRetain(Program->Context)); // Set up how to obtain kernel properties when needed. ZeKernelProperties.Compute = [this](ze_kernel_properties_t &Properties) { @@ -1154,3 +1190,5 @@ ur_result_t ur_kernel_handle_t_::initialize() { return UR_RESULT_SUCCESS; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/kernel.hpp b/unified-runtime/source/adapters/level_zero/kernel.hpp index a06f44cf74644..ac6aa30cc6dca 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/kernel.hpp @@ -15,7 +15,9 @@ #include "common/ur_ref_count.hpp" #include "memory.hpp" -struct ur_kernel_handle_t_ : ur_object { +namespace ur::level_zero::v1 { + +struct ur_kernel_handle_t_ : ur_object_t { ur_kernel_handle_t_(bool OwnZeHandle, ur_program_handle_t Program) : Context{nullptr}, Program{Program}, ZeKernel{nullptr}, SubmissionsCount{0}, MemAllocs{} { @@ -117,8 +119,6 @@ struct ur_kernel_handle_t_ : ur_object { ur_result_t getZeKernel(ze_device_handle_t hDevice, ur_kernel_handle_t hKernel, ze_kernel_handle_t *phZeKernel); -namespace ur::level_zero { - ur_result_t urKernelSetArgValueHelper(ur_kernel_handle_t Kernel, uint32_t ArgIndex, size_t ArgSize, const void *PArgValue); @@ -128,4 +128,4 @@ urKernelSetArgMemObjHelper(ur_kernel_handle_t Kernel, uint32_t ArgIndex, const ur_kernel_arg_mem_obj_properties_t *Properties, ur_mem_handle_t ArgValue); -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index 97239a8416564..dd12327bf3f61 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -12,15 +12,17 @@ #include #include +#include "common/helpers/memory_helpers.hpp" +#include "common/image_common.hpp" #include "context.hpp" #include "event.hpp" -#include "helpers/memory_helpers.hpp" -#include "image_common.hpp" #include "logger/ur_logger.hpp" #include "queue.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" +namespace ur::level_zero::v1 { + // Default to using compute engine for fill operation, but allow to // override this with an environment variable. static bool PreferCopyEngine = [] { @@ -286,7 +288,7 @@ static ur_result_t ZeHostMemAllocHelper(void **ResultPtr, // indirect access, that is why explicitly retain context to be sure // that it is released after all memory allocations in this context are // released. - UR_CALL(ur::level_zero::urContextRetain(UrContext)); + UR_CALL(ur::level_zero::v1::urContextRetain(v1_cast(UrContext))); } ZeStruct ZeDesc; @@ -296,10 +298,9 @@ static ur_result_t ZeHostMemAllocHelper(void **ResultPtr, if (IndirectAccessTrackingEnabled) { // Keep track of all memory allocations in the context - UrContext->MemAllocs.emplace( - std::piecewise_construct, std::forward_as_tuple(*ResultPtr), - std::forward_as_tuple( - reinterpret_cast(UrContext))); + UrContext->MemAllocs.emplace(std::piecewise_construct, + std::forward_as_tuple(*ResultPtr), + std::forward_as_tuple(UrContext)); } return UR_RESULT_SUCCESS; } @@ -374,9 +375,9 @@ static ur_result_t enqueueMemImageCommandHelper( UR_RESULT_ERROR_INVALID_IMAGE_SIZE); char *ZeHandleSrc = nullptr; - UR_CALL(SrcMem->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(SrcMem->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, EventWaitList, NumEventsInWaitList)); ZE2UR_CALL(zeCommandListAppendImageCopyToMemory, (ZeCommandList, Dst, ur_cast(ZeHandleSrc), &ZeSrcRegion, ZeEvent, WaitList.Length, WaitList.ZeEventList)); @@ -408,9 +409,9 @@ static ur_result_t enqueueMemImageCommandHelper( UR_RESULT_ERROR_INVALID_IMAGE_SIZE); char *ZeHandleDst = nullptr; - UR_CALL(DstMem->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(DstMem->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitList, NumEventsInWaitList)); ZE2UR_CALL(zeCommandListAppendImageCopyFromMemory, (ZeCommandList, ur_cast(ZeHandleDst), Src, &ZeDstRegion, ZeEvent, WaitList.Length, WaitList.ZeEventList)); @@ -427,12 +428,12 @@ static ur_result_t enqueueMemImageCommandHelper( char *ZeHandleSrc = nullptr; char *ZeHandleDst = nullptr; - UR_CALL(SrcImage->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); - UR_CALL(DstImage->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(SrcImage->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, EventWaitList, NumEventsInWaitList)); + UR_CALL(DstImage->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitList, NumEventsInWaitList)); ZE2UR_CALL(zeCommandListAppendImageCopyRegion, (ZeCommandList, ur_cast(ZeHandleDst), ur_cast(ZeHandleSrc), &ZeDstRegion, @@ -447,13 +448,11 @@ static ur_result_t enqueueMemImageCommandHelper( return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - ur_result_t urEnqueueMemBufferRead( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t hBuffer, + ::ur_mem_handle_t hBufferOpque, /// [in] indicates blocking (true), non-blocking (false) bool blockingRead, /// [in] offset in bytes in the buffer object @@ -468,31 +467,34 @@ ur_result_t urEnqueueMemBufferRead( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *phEventWaitList, + const ::ur_event_handle_t *phEventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *phEvent) { - ur_mem_handle_t_ *Src = ur_cast(hBuffer); + ::ur_event_handle_t *phEvent) { + auto Queue = v1_cast(QueueOpque); + auto Src = v1_cast(hBufferOpque); + auto phEventWaitListInternal = v1_cast(phEventWaitList); + auto phEventInternal = v1_cast(phEvent); std::shared_lock SrcLock(Src->Mutex, std::defer_lock); std::scoped_lock, ur_shared_mutex> LockAll( SrcLock, Queue->Mutex); char *ZeHandleSrc = nullptr; - UR_CALL(Src->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, phEventWaitList, - numEventsInWaitList)); + UR_CALL(Src->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, phEventWaitListInternal, numEventsInWaitList)); return enqueueMemCopyHelper(UR_COMMAND_MEM_BUFFER_READ, Queue, pDst, blockingRead, size, ZeHandleSrc + offset, - numEventsInWaitList, phEventWaitList, phEvent, - true /* PreferCopyEngine */); + numEventsInWaitList, phEventWaitListInternal, + phEventInternal, true /* PreferCopyEngine */); } ur_result_t urEnqueueMemBufferWrite( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t hBuffer, + ::ur_mem_handle_t hBufferOpque, /// [in] indicates blocking (true), non-blocking (false) bool blockingWrite, /// [in] offset in bytes in the buffer object @@ -507,32 +509,35 @@ ur_result_t urEnqueueMemBufferWrite( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *phEventWaitList, + const ::ur_event_handle_t *phEventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *phEvent) { - ur_mem_handle_t_ *Buffer = ur_cast(hBuffer); + ::ur_event_handle_t *phEvent) { + auto Queue = v1_cast(QueueOpque); + auto Buffer = v1_cast(hBufferOpque); + auto phEventWaitListInternal = v1_cast(phEventWaitList); + auto phEventInternal = v1_cast(phEvent); std::scoped_lock Lock(Queue->Mutex, Buffer->Mutex); char *ZeHandleDst = nullptr; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, phEventWaitList, - numEventsInWaitList)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, phEventWaitListInternal, numEventsInWaitList)); return enqueueMemCopyHelper(UR_COMMAND_MEM_BUFFER_WRITE, Queue, ZeHandleDst + offset, // dst blockingWrite, size, pSrc, // src - numEventsInWaitList, phEventWaitList, phEvent, - true /* PreferCopyEngine */); + numEventsInWaitList, phEventWaitListInternal, + phEventInternal, true /* PreferCopyEngine */); } ur_result_t urEnqueueMemBufferReadRect( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t hBuffer, + ::ur_mem_handle_t hBufferOpque, /// [in] indicates blocking (true), non-blocking (false) bool blockingRead, /// [in] 3D offset in the buffer @@ -559,32 +564,35 @@ ur_result_t urEnqueueMemBufferReadRect( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *phEventWaitList, + const ::ur_event_handle_t *phEventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *phEvent) { - ur_mem_handle_t_ *Buffer = ur_cast(hBuffer); + ::ur_event_handle_t *phEvent) { + auto Queue = v1_cast(QueueOpque); + auto Buffer = v1_cast(hBufferOpque); + auto phEventWaitListInternal = v1_cast(phEventWaitList); + auto phEventInternal = v1_cast(phEvent); std::shared_lock SrcLock(Buffer->Mutex, std::defer_lock); std::scoped_lock, ur_shared_mutex> LockAll( SrcLock, Queue->Mutex); char *ZeHandleSrc; - UR_CALL(Buffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, phEventWaitList, - numEventsInWaitList)); + UR_CALL(Buffer->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, phEventWaitListInternal, numEventsInWaitList)); return enqueueMemCopyRectHelper( UR_COMMAND_MEM_BUFFER_READ_RECT, Queue, ZeHandleSrc, pDst, bufferOffset, hostOffset, region, bufferRowPitch, hostRowPitch, bufferSlicePitch, - hostSlicePitch, blockingRead, numEventsInWaitList, phEventWaitList, - phEvent); + hostSlicePitch, blockingRead, numEventsInWaitList, + phEventWaitListInternal, phEventInternal, false /*PreferCopyEngine*/); } ur_result_t urEnqueueMemBufferWriteRect( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t hBuffer, + ::ur_mem_handle_t hBufferOpque, /// [in] indicates blocking (true), non-blocking (false) bool blockingWrite, /// [in] 3D offset in the buffer @@ -611,34 +619,37 @@ ur_result_t urEnqueueMemBufferWriteRect( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *phEventWaitList, + const ::ur_event_handle_t *phEventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *phEvent) { - ur_mem_handle_t_ *Buffer = ur_cast(hBuffer); + ::ur_event_handle_t *phEvent) { + auto Queue = v1_cast(QueueOpque); + auto Buffer = v1_cast(hBufferOpque); + auto phEventWaitListInternal = v1_cast(phEventWaitList); + auto phEventInternal = v1_cast(phEvent); std::scoped_lock Lock(Queue->Mutex, Buffer->Mutex); char *ZeHandleDst = nullptr; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, phEventWaitList, - numEventsInWaitList)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, phEventWaitListInternal, numEventsInWaitList)); return enqueueMemCopyRectHelper( UR_COMMAND_MEM_BUFFER_WRITE_RECT, Queue, const_cast(static_cast(pSrc)), ZeHandleDst, hostOffset, bufferOffset, region, hostRowPitch, bufferRowPitch, hostSlicePitch, bufferSlicePitch, blockingWrite, numEventsInWaitList, - phEventWaitList, phEvent); + phEventWaitListInternal, phEventInternal, false /*PreferCopyEngine*/); } ur_result_t urEnqueueMemBufferCopy( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the src buffer object - ur_mem_handle_t BufferSrc, + ::ur_mem_handle_t BufferSrcOpque, /// [in] handle of the dest buffer object - ur_mem_handle_t BufferDst, + ::ur_mem_handle_t BufferDstOpque, /// [in] offset into hBufferSrc to begin copying from size_t SrcOffset, /// [in] offset info hBufferDst to begin copying into @@ -651,12 +662,15 @@ ur_result_t urEnqueueMemBufferCopy( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { - ur_buffer *SrcBuffer = ur_cast(BufferSrc); - ur_buffer *DstBuffer = ur_cast(BufferDst); + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); + ur_buffer *SrcBuffer = ur_cast(BufferSrcOpque); + ur_buffer *DstBuffer = ur_cast(BufferDstOpque); UR_ASSERT(!SrcBuffer->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); UR_ASSERT(!DstBuffer->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); @@ -674,28 +688,28 @@ ur_result_t urEnqueueMemBufferCopy( PreferCopyEngine |= UseCopyEngineForD2DCopy; char *ZeHandleSrc = nullptr; - UR_CALL(SrcBuffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(SrcBuffer->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); char *ZeHandleDst = nullptr; - UR_CALL(DstBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(DstBuffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); return enqueueMemCopyHelper( UR_COMMAND_MEM_BUFFER_COPY, Queue, ZeHandleDst + DstOffset, false, // blocking - Size, ZeHandleSrc + SrcOffset, NumEventsInWaitList, EventWaitList, - OutEvent, PreferCopyEngine); + Size, ZeHandleSrc + SrcOffset, NumEventsInWaitList, EventWaitListInternal, + OutEventInternal, PreferCopyEngine); } ur_result_t urEnqueueMemBufferCopyRect( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the source buffer object - ur_mem_handle_t BufferSrc, + ::ur_mem_handle_t BufferSrcOpque, /// [in] handle of the dest buffer object - ur_mem_handle_t BufferDst, + ::ur_mem_handle_t BufferDstOpque, /// [in] 3D offset in the source buffer ur_rect_offset_t SrcOrigin, /// [in] 3D offset in the destination buffer @@ -716,12 +730,15 @@ ur_result_t urEnqueueMemBufferCopyRect( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { - ur_buffer *SrcBuffer = ur_cast(BufferSrc); - ur_buffer *DstBuffer = ur_cast(BufferDst); + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); + ur_buffer *SrcBuffer = ur_cast(BufferSrcOpque); + ur_buffer *DstBuffer = ur_cast(BufferDstOpque); UR_ASSERT(!SrcBuffer->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); UR_ASSERT(!DstBuffer->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); @@ -736,27 +753,28 @@ ur_result_t urEnqueueMemBufferCopyRect( bool PreferCopyEngine = (SrcBuffer->OnHost || DstBuffer->OnHost); char *ZeHandleSrc = nullptr; - UR_CALL(SrcBuffer->getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(SrcBuffer->getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); char *ZeHandleDst = nullptr; - UR_CALL(DstBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); - - return enqueueMemCopyRectHelper( - UR_COMMAND_MEM_BUFFER_COPY_RECT, Queue, ZeHandleSrc, ZeHandleDst, - SrcOrigin, DstOrigin, SrcRegion, SrcRowPitch, DstRowPitch, SrcSlicePitch, - DstSlicePitch, - false, // blocking - NumEventsInWaitList, EventWaitList, OutEvent, PreferCopyEngine); + UR_CALL(DstBuffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); + + return enqueueMemCopyRectHelper(UR_COMMAND_MEM_BUFFER_COPY_RECT, Queue, + ZeHandleSrc, ZeHandleDst, SrcOrigin, + DstOrigin, SrcRegion, SrcRowPitch, + DstRowPitch, SrcSlicePitch, DstSlicePitch, + false, // blocking + NumEventsInWaitList, EventWaitListInternal, + OutEventInternal, PreferCopyEngine); } ur_result_t urEnqueueMemBufferFill( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t Buffer, + ::ur_mem_handle_t BufferOpque, /// [in] pointer to the fill pattern const void *Pattern, /// [in] size in bytes of the pattern @@ -771,30 +789,34 @@ ur_result_t urEnqueueMemBufferFill( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto Buffer = v1_cast(BufferOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); std::scoped_lock Lock(Queue->Mutex, Buffer->Mutex); char *ZeHandleDst = nullptr; - ur_buffer *UrBuffer = reinterpret_cast(Buffer); + ur_buffer *UrBuffer = reinterpret_cast(BufferOpque); UR_CALL(UrBuffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); return enqueueMemFillHelper( UR_COMMAND_MEM_BUFFER_FILL, Queue, ZeHandleDst + Offset, Pattern, // It will be interpreted as an 8-bit value, PatternSize, // which is indicated with this pattern_size==1 - Size, NumEventsInWaitList, EventWaitList, OutEvent); + Size, NumEventsInWaitList, EventWaitListInternal, OutEventInternal); } ur_result_t urEnqueueMemImageRead( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the image object - ur_mem_handle_t Image, + ::ur_mem_handle_t ImageOpque, /// [in] indicates blocking (true), non-blocking (false) bool BlockingRead, /// [in] defines the (x,y,z) offset in pixels in the 1D, 2D, or 3D image @@ -814,23 +836,27 @@ ur_result_t urEnqueueMemImageRead( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto Image = v1_cast(ImageOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); std::scoped_lock Lock(Queue->Mutex, Image->Mutex); return enqueueMemImageCommandHelper( - UR_COMMAND_MEM_IMAGE_READ, Queue, Image, Dst, BlockingRead, &Origin, + UR_COMMAND_MEM_IMAGE_READ, Queue, ImageOpque, Dst, BlockingRead, &Origin, nullptr, &Region, RowPitch, SlicePitch, NumEventsInWaitList, - EventWaitList, OutEvent); + EventWaitListInternal, OutEventInternal); } ur_result_t urEnqueueMemImageWrite( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the image object - ur_mem_handle_t Image, + ::ur_mem_handle_t ImageOpque, /// [in] indicates blocking (true), non-blocking (false) bool BlockingWrite, /// [in] defines the (x,y,z) offset in pixels in the 1D, 2D, or 3D image @@ -850,25 +876,29 @@ ur_result_t urEnqueueMemImageWrite( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto Image = v1_cast(ImageOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); std::scoped_lock Lock(Queue->Mutex, Image->Mutex); return enqueueMemImageCommandHelper( - UR_COMMAND_MEM_IMAGE_WRITE, Queue, Src, Image, BlockingWrite, nullptr, - &Origin, &Region, RowPitch, SlicePitch, NumEventsInWaitList, - EventWaitList, OutEvent); + UR_COMMAND_MEM_IMAGE_WRITE, Queue, Src, ImageOpque, BlockingWrite, + nullptr, &Origin, &Region, RowPitch, SlicePitch, NumEventsInWaitList, + EventWaitListInternal, OutEventInternal); } ur_result_t urEnqueueMemImageCopy( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the src image object - ur_mem_handle_t ImageSrc, + ::ur_mem_handle_t ImageSrcOpque, /// [in] handle of the dest image object - ur_mem_handle_t ImageDst, + ::ur_mem_handle_t ImageDstOpque, /// [in] defines the (x,y,z) offset in pixels in the source 1D, 2D, or 3D /// image ur_rect_offset_t SrcOrigin, @@ -884,10 +914,15 @@ ur_result_t urEnqueueMemImageCopy( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto ImageSrc = v1_cast(ImageSrcOpque); + auto ImageDst = v1_cast(ImageDstOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); std::shared_lock SrcLock(ImageSrc->Mutex, std::defer_lock); std::scoped_lock, ur_shared_mutex, ur_shared_mutex> @@ -897,19 +932,20 @@ ur_result_t urEnqueueMemImageCopy( // Images are always allocated on device. bool PreferCopyEngine = false; return enqueueMemImageCommandHelper( - UR_COMMAND_MEM_IMAGE_COPY, Queue, ImageSrc, ImageDst, + UR_COMMAND_MEM_IMAGE_COPY, Queue, ImageSrcOpque, ImageDstOpque, false, // is_blocking &SrcOrigin, &DstOrigin, &Region, 0, // row pitch 0, // slice pitch - NumEventsInWaitList, EventWaitList, OutEvent, PreferCopyEngine); + NumEventsInWaitList, EventWaitListInternal, OutEventInternal, + PreferCopyEngine); } ur_result_t urEnqueueMemBufferMap( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the buffer object - ur_mem_handle_t Buf, + ::ur_mem_handle_t BufOpque, /// [in] indicates blocking (true), non-blocking (false) bool BlockingMap, /// [in] flags for read, write, readwrite mapping @@ -924,20 +960,24 @@ ur_result_t urEnqueueMemBufferMap( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent, + ::ur_event_handle_t *OutEvent, /// [in,out] return mapped pointer. TODO: move it before /// numEventsInWaitList? void **RetMap) { - auto Buffer = ur_cast(Buf); + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); + auto Buffer = ur_cast(BufOpque); UR_ASSERT(!Buffer->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); ur_event_handle_t InternalEvent; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; ze_event_handle_t ZeEvent = nullptr; bool UseCopyEngine = false; @@ -947,7 +987,7 @@ ur_result_t urEnqueueMemBufferMap( ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); UR_CALL(createEventAndAssociateQueue( Queue, Event, UR_COMMAND_MEM_BUFFER_MAP, Queue->CommandListMap.end(), @@ -958,19 +998,20 @@ ur_result_t urEnqueueMemBufferMap( } // Translate the host access mode info. - ur_mem_handle_t_::access_mode_t AccessMode = ur_mem_handle_t_::unknown; + ur::level_zero::v1::ur_mem_handle_t_::access_mode_t AccessMode = + ur::level_zero::v1::ur_mem_handle_t_::unknown; if (MapFlags & UR_MAP_FLAG_WRITE_INVALIDATE_REGION) - AccessMode = ur_mem_handle_t_::write_only; + AccessMode = ur::level_zero::v1::ur_mem_handle_t_::write_only; else { if (MapFlags & UR_MAP_FLAG_READ) { - AccessMode = ur_mem_handle_t_::read_only; + AccessMode = ur::level_zero::v1::ur_mem_handle_t_::read_only; if (MapFlags & UR_MAP_FLAG_WRITE) - AccessMode = ur_mem_handle_t_::read_write; + AccessMode = ur::level_zero::v1::ur_mem_handle_t_::read_write; } else if (MapFlags & UR_MAP_FLAG_WRITE) - AccessMode = ur_mem_handle_t_::write_only; + AccessMode = ur::level_zero::v1::ur_mem_handle_t_::write_only; } - UR_ASSERT(AccessMode != ur_mem_handle_t_::unknown, + UR_ASSERT(AccessMode != ur::level_zero::v1::ur_mem_handle_t_::unknown, UR_RESULT_ERROR_INVALID_VALUE); // TODO: Level Zero is missing the memory "mapping" capabilities, so we are @@ -989,22 +1030,23 @@ ur_result_t urEnqueueMemBufferMap( if (Buffer->OnHost) { // Wait on incoming events before doing the copy if (NumEventsInWaitList > 0) - UR_CALL(ur::level_zero::urEventWait(NumEventsInWaitList, EventWaitList)); + UR_CALL( + ur::level_zero::v1::urEventWait(NumEventsInWaitList, EventWaitList)); if (Queue->isInOrderQueue()) - UR_CALL(ur::level_zero::urQueueFinish(Queue)); + UR_CALL(ur::level_zero::v1::urQueueFinish(QueueOpque)); // Lock automatically releases when this goes out of scope. std::scoped_lock Guard(Buffer->Mutex); char *ZeHandleSrc; UR_CALL(Buffer->getZeHandle(ZeHandleSrc, AccessMode, Queue->Device, - EventWaitList, NumEventsInWaitList)); + EventWaitListInternal, NumEventsInWaitList)); if (Buffer->MapHostPtr) { *RetMap = Buffer->MapHostPtr + Offset; if (ZeHandleSrc != Buffer->MapHostPtr && - AccessMode != ur_mem_handle_t_::write_only) { + AccessMode != ur::level_zero::v1::ur_mem_handle_t_::write_only) { memcpy(*RetMap, ZeHandleSrc + Offset, Size); } } else { @@ -1040,14 +1082,15 @@ ur_result_t urEnqueueMemBufferMap( } // Take a shortcut if the host is not going to read buffer's data. - if (AccessMode == ur_mem_handle_t_::write_only) { + if (AccessMode == ur::level_zero::v1::ur_mem_handle_t_::write_only) { (*Event)->Completed = true; } else { // For discrete devices we need a command list ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, - false /*AllowBatching*/, nullptr /*ForcedCmdQueue*/)); + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, false /*AllowBatching*/, + nullptr /*ForcedCmdQueue*/)); // Add the event to the command list. CommandList->second.append(reinterpret_cast(*Event)); @@ -1058,10 +1101,10 @@ ur_result_t urEnqueueMemBufferMap( char *ZeHandleSrc; UR_CALL(Buffer->getZeHandle(ZeHandleSrc, AccessMode, Queue->Device, - EventWaitList, NumEventsInWaitList)); + EventWaitListInternal, NumEventsInWaitList)); UR_CALL(setSignalEvent(Queue, UseCopyEngine, &ZeEvent, Event, - NumEventsInWaitList, EventWaitList, + NumEventsInWaitList, EventWaitListInternal, CommandList->second.ZeQueue)); ZE2UR_CALL(zeCommandListAppendMemoryCopy, @@ -1084,9 +1127,9 @@ ur_result_t urEnqueueMemBufferMap( ur_result_t urEnqueueMemUnmap( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] handle of the memory (buffer or image) object - ur_mem_handle_t Mem, + ::ur_mem_handle_t MemOpque, /// [in] mapped host address void *MappedPtr, /// [in] size of the event wait list @@ -1095,27 +1138,32 @@ ur_result_t urEnqueueMemUnmap( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitListOpque, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEventOpque) { + auto Queue = v1_cast(QueueOpque); + auto Mem = v1_cast(MemOpque); + auto EventWaitListInternal = v1_cast(EventWaitListOpque); + auto OutEventInternal = v1_cast(OutEventOpque); UR_ASSERT(!Mem->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); - auto Buffer = ur_cast(Mem); + auto Buffer = static_cast(Mem); bool UseCopyEngine = false; ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; { // Lock automatically releases when this goes out of scope. std::scoped_lock lock(Queue->Mutex); ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_MEM_UNMAP, Queue->CommandListMap.end(), @@ -1150,15 +1198,16 @@ ur_result_t urEnqueueMemUnmap( if (Buffer->OnHost) { // Wait on incoming events before doing the copy if (NumEventsInWaitList > 0) - UR_CALL(ur::level_zero::urEventWait(NumEventsInWaitList, EventWaitList)); + UR_CALL(ur::level_zero::v1::urEventWait(NumEventsInWaitList, + EventWaitListOpque)); if (Queue->isInOrderQueue()) - UR_CALL(ur::level_zero::urQueueFinish(Queue)); + UR_CALL(ur::level_zero::v1::urQueueFinish(QueueOpque)); char *ZeHandleDst; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); std::scoped_lock Guard(Buffer->Mutex); if (Buffer->MapHostPtr) @@ -1177,8 +1226,8 @@ ur_result_t urEnqueueMemUnmap( ur_command_list_ptr_t CommandList{}; UR_CALL(Queue->Context->getAvailableCommandList( - reinterpret_cast(Queue), CommandList, UseCopyEngine, - NumEventsInWaitList, EventWaitList, false /*AllowBatching*/, + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, false /*AllowBatching*/, nullptr /*ForcedCmdQueue*/)); CommandList->second.append(reinterpret_cast(*Event)); @@ -1193,12 +1242,12 @@ ur_result_t urEnqueueMemUnmap( // piEnqueueMemBufferMap. char *ZeHandleDst; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Queue->Device, EventWaitList, - NumEventsInWaitList)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Queue->Device, EventWaitListInternal, NumEventsInWaitList)); UR_CALL(setSignalEvent(Queue, UseCopyEngine, &ZeEvent, Event, - NumEventsInWaitList, EventWaitList, + NumEventsInWaitList, EventWaitListInternal, CommandList->second.ZeQueue)); ZE2UR_CALL(zeCommandListAppendMemoryCopy, @@ -1216,7 +1265,7 @@ ur_result_t urEnqueueMemUnmap( ur_result_t urEnqueueUSMMemcpy( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] blocking or non-blocking copy bool Blocking, /// [in] pointer to the destination USM memory object @@ -1231,10 +1280,13 @@ ur_result_t urEnqueueUSMMemcpy( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); std::scoped_lock lock(Queue->Mutex); // Device to Device copies are found to execute slower on copy engine @@ -1253,12 +1305,13 @@ ur_result_t urEnqueueUSMMemcpy( return enqueueMemCopyHelper( // TODO: do we need a new command type for this? UR_COMMAND_MEM_BUFFER_COPY, Queue, Dst, Blocking, Size, Src, - NumEventsInWaitList, EventWaitList, OutEvent, PreferCopyEngine); + NumEventsInWaitList, EventWaitListInternal, OutEventInternal, + PreferCopyEngine); } ur_result_t urEnqueueUSMPrefetch( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] pointer to the USM memory object const void *Mem, /// [in] size in bytes to be fetched @@ -1271,10 +1324,13 @@ ur_result_t urEnqueueUSMPrefetch( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto OutEventInternal = v1_cast(OutEvent); switch (Flags) { case UR_USM_MIGRATION_FLAG_HOST_TO_DEVICE: break; @@ -1300,21 +1356,23 @@ ur_result_t urEnqueueUSMPrefetch( // ur_ze_event_list_t TmpWaitList; UR_CALL(TmpWaitList.createAndRetainUrZeEventList( - NumEventsInWaitList, EventWaitList, Queue, UseCopyEngine)); + NumEventsInWaitList, EventWaitListInternal, Queue, UseCopyEngine)); // Get a new command list to be used on this call ur_command_list_ptr_t CommandList{}; // TODO: Change UseCopyEngine argument to 'true' once L0 backend // support is added UR_CALL(Queue->Context->getAvailableCommandList( - Queue, CommandList, UseCopyEngine, NumEventsInWaitList, EventWaitList, - false /*AllowBatching*/, nullptr /*ForcedCmdQueue*/)); + Queue, CommandList, UseCopyEngine, NumEventsInWaitList, + EventWaitListInternal, false /*AllowBatching*/, + nullptr /*ForcedCmdQueue*/)); // TODO: do we need to create a unique command type for this? ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_USM_PREFETCH, CommandList, IsInternal, false)); ZeEvent = (*Event)->ZeEvent; @@ -1343,7 +1401,7 @@ ur_result_t urEnqueueUSMPrefetch( ur_result_t urEnqueueUSMAdvise( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] pointer to the USM memory object const void *Mem, /// [in] size in bytes to be advised @@ -1352,7 +1410,9 @@ ur_result_t urEnqueueUSMAdvise( ur_usm_advice_flags_t Advice, /// [in,out][optional] return an event object that identifies this /// particular command instance. - ur_event_handle_t *OutEvent) { + ::ur_event_handle_t *OutEvent) { + auto Queue = v1_cast(QueueOpque); + auto OutEventInternal = v1_cast(OutEvent); // Lock automatically releases when this goes out of scope. std::scoped_lock lock(Queue->Mutex); @@ -1376,8 +1436,9 @@ ur_result_t urEnqueueUSMAdvise( // TODO: do we need to create a unique command type for this? ze_event_handle_t ZeEvent = nullptr; ur_event_handle_t InternalEvent{}; - bool IsInternal = OutEvent == nullptr; - ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + bool IsInternal = OutEventInternal == nullptr; + ur_event_handle_t *Event = + OutEventInternal ? OutEventInternal : &InternalEvent; UR_CALL(createEventAndAssociateQueue(Queue, Event, UR_COMMAND_USM_ADVISE, CommandList, IsInternal, false)); ZeEvent = (*Event)->ZeEvent; @@ -1406,7 +1467,7 @@ ur_result_t urEnqueueUSMAdvise( ur_result_t urEnqueueUSMFill2D( /// [in] handle of the queue to submit to. - ur_queue_handle_t /*Queue*/, + ::ur_queue_handle_t /*Queue*/, /// [in] pointer to memory to be filled. void * /*Mem*/, /// [in] the total width of the destination memory including padding. @@ -1425,10 +1486,10 @@ ur_result_t urEnqueueUSMFill2D( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t * /*EventWaitList*/, + const ::ur_event_handle_t * /*EventWaitList*/, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t * /*OutEvent*/) { + ::ur_event_handle_t * /*OutEvent*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -1437,7 +1498,7 @@ ur_result_t urEnqueueUSMFill2D( ur_result_t urEnqueueUSMMemcpy2D( /// [in] handle of the queue to submit to. - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] indicates if this operation should block the host. bool Blocking, /// [in] pointer to memory where data will be copied. @@ -1458,13 +1519,14 @@ ur_result_t urEnqueueUSMMemcpy2D( /// events that must be complete before the kernel execution. If /// nullptr, the numEventsInWaitList must be 0, indicating that no wait /// event. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [in,out][optional] return an event object that identifies this /// particular kernel execution instance. - ur_event_handle_t *Event) { + ::ur_event_handle_t *Event) { ur_rect_offset_t ZeroOffset{0, 0, 0}; ur_rect_region_t Region{Width, Height, 0}; + auto Queue = v1_cast(QueueOpque); std::scoped_lock lock(Queue->Mutex); // Device to Device copies are found to execute slower on copy engine @@ -1486,12 +1548,13 @@ ur_result_t urEnqueueUSMMemcpy2D( UR_COMMAND_MEM_BUFFER_COPY_RECT, Queue, Src, Dst, ZeroOffset, ZeroOffset, Region, SrcPitch, DstPitch, 0, /*SrcSlicePitch=*/ 0, /*DstSlicePitch=*/ - Blocking, NumEventsInWaitList, EventWaitList, Event, PreferCopyEngine); + Blocking, NumEventsInWaitList, v1_cast(EventWaitList), v1_cast(Event), + PreferCopyEngine); } ur_result_t urMemImageCreate( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] allocation and usage information flags ur_mem_flags_t Flags, /// [in] pointer to image format specification @@ -1501,7 +1564,8 @@ ur_result_t urMemImageCreate( /// [in] pointer to the buffer data void *Host, /// [out] pointer to handle of image object created - ur_mem_handle_t *Mem) { + ::ur_mem_handle_t *Mem) { + auto Context = v1_cast(ContextOpque); // TODO: implement read-only, write-only if ((Flags & UR_MEM_FLAG_READ_WRITE) == 0) { die("urMemImageCreate: Level-Zero implements only read-write buffer," @@ -1517,7 +1581,7 @@ ur_result_t urMemImageCreate( // own the image. // TODO: Implement explicit copying for acessing the image from other devices // in the context. - ur_device_handle_t Device = Context->Devices[0]; + ur_device_handle_t Device = Context->getDevices()[0]; ze_image_handle_t ZeImage; ZE2UR_CALL(zeImageCreate, (Context->ZeContext, Device->ZeDevice, &ZeImageDesc, &ZeImage)); @@ -1542,15 +1606,16 @@ ur_result_t urMemImageCreate( ur_result_t urMemImageCreateWithNativeHandle( /// [in] the native handle to the memory. - ur_native_handle_t NativeMem, + ::ur_native_handle_t NativeMem, /// [in] handle of the context object. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] pointer to image format specification. [[maybe_unused]] const ur_image_format_t *ImageFormat, /// [in] pointer to image description. [[maybe_unused]] const ur_image_desc_t *ImageDesc, /// [in][optional] pointer to native memory creation properties. - const ur_mem_native_properties_t *Properties, ur_mem_handle_t *Mem) { + const ur_mem_native_properties_t *Properties, ::ur_mem_handle_t *Mem) { + auto Context = v1_cast(ContextOpque); std::shared_lock Lock(Context->Mutex); ze_image_handle_t ZeHImage = ur_cast(NativeMem); @@ -1571,13 +1636,14 @@ ur_result_t urMemImageCreateWithNativeHandle( ur_result_t urMemBufferCreate( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] allocation and usage information flags ur_mem_flags_t Flags, /// [in] size in bytes of the memory object to be allocated size_t Size, const ur_buffer_properties_t *Properties, /// [out] pointer to handle of the memory buffer created - ur_mem_handle_t *RetBuffer) { + ::ur_mem_handle_t *RetBuffer) { + auto Context = v1_cast(ContextOpque); if (Flags & UR_MEM_FLAG_ALLOC_HOST_POINTER) { // Having PI_MEM_FLAGS_HOST_PTR_ALLOC for buffer requires allocation of // pinned host memory, see: @@ -1619,8 +1685,9 @@ ur_result_t urMemBufferCreate( // device in the context be the master, and hold the initial valid // allocation. char *ZeHandleDst; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Context->Devices[0], nullptr, 0u)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, + Context->getDevices()[0], nullptr, 0u)); if (Buffer->OnHost) { // Do a host to host copy. // For an imported HostPtr the copy is unneeded. @@ -1641,29 +1708,31 @@ ur_result_t urMemBufferCreate( die("urMemBufferCreate: not implemented"); } - *RetBuffer = reinterpret_cast(Buffer); + *RetBuffer = v1_cast(static_cast(Buffer)); return UR_RESULT_SUCCESS; } ur_result_t urMemRetain( /// [in] handle of the memory object to get access - ur_mem_handle_t Mem) { - Mem->RefCount.retain(); + ::ur_mem_handle_t MemOpque) { + v1_cast(MemOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urMemRelease( /// [in] handle of the memory object to release - ur_mem_handle_t Mem) { - if (!Mem->RefCount.release()) + ::ur_mem_handle_t MemOpque) { + auto UrMem = v1_cast(MemOpque); + if (!UrMem->RefCount.release()) return UR_RESULT_SUCCESS; - if (Mem->isImage()) { + if (UrMem->isImage()) { char *ZeHandleImage; - auto Image = static_cast(Mem); + auto Image = static_cast(UrMem); if (Image->OwnNativeHandle) { - UR_CALL(Mem->getZeHandle(ZeHandleImage, ur_mem_handle_t_::write_only, - nullptr, nullptr, 0u)); + UR_CALL(UrMem->getZeHandle( + ZeHandleImage, ur::level_zero::v1::ur_mem_handle_t_::write_only, + nullptr, nullptr, 0u)); if (checkL0LoaderTeardown()) { auto ZeResult = ZE_CALL_NOCHECK( zeImageDestroy, (ur_cast(ZeHandleImage))); @@ -1678,7 +1747,7 @@ ur_result_t urMemRelease( } delete Image; } else { - auto Buffer = reinterpret_cast(Mem); + auto Buffer = static_cast(UrMem); Buffer->free(); delete Buffer; } @@ -1689,7 +1758,7 @@ ur_result_t urMemRelease( ur_result_t urMemBufferPartition( /// [in] handle of the buffer object to allocate from - ur_mem_handle_t Buffer, + ::ur_mem_handle_t BufferOpque, /// [in] allocation and usage information flags ur_mem_flags_t Flags, /// [in] buffer creation type @@ -1697,8 +1766,9 @@ ur_result_t urMemBufferPartition( /// [in] pointer to buffer create region information const ur_buffer_region_t *BufferCreateInfo, /// [out] pointer to the handle of sub buffer created - ur_mem_handle_t *RetMem) { - UR_ASSERT(Buffer && !Buffer->isImage() && + ::ur_mem_handle_t *RetMem) { + auto Buffer = v1_cast(BufferOpque); + UR_ASSERT(BufferOpque && !Buffer->isImage() && !(static_cast(Buffer))->isSubBuffer(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); @@ -1713,7 +1783,7 @@ ur_result_t urMemBufferPartition( auto partitionedBuffer = new ur_buffer(static_cast(Buffer), BufferCreateInfo->origin, BufferCreateInfo->size); - *RetMem = reinterpret_cast(partitionedBuffer); + *RetMem = v1_cast(static_cast(partitionedBuffer)); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -1725,15 +1795,17 @@ ur_result_t urMemBufferPartition( ur_result_t urMemGetNativeHandle( /// [in] handle of the mem. - ur_mem_handle_t Mem, + ::ur_mem_handle_t MemOpque, /// [in] handle of the device. - ur_device_handle_t, + ::ur_device_handle_t, /// [out] a pointer to the native handle of the mem. - ur_native_handle_t *NativeMem) { + ::ur_native_handle_t *NativeMem) { + auto Mem = v1_cast(MemOpque); std::shared_lock Guard(Mem->Mutex); char *ZeHandle = nullptr; - UR_CALL(Mem->getZeHandle(ZeHandle, ur_mem_handle_t_::read_write, nullptr, - nullptr, 0u)); + UR_CALL(Mem->getZeHandle(ZeHandle, + ur::level_zero::v1::ur_mem_handle_t_::read_write, + nullptr, nullptr, 0u)); *NativeMem = ur_cast(ZeHandle); return UR_RESULT_SUCCESS; @@ -1741,13 +1813,14 @@ ur_result_t urMemGetNativeHandle( ur_result_t urMemBufferCreateWithNativeHandle( /// [in] the native handle to the memory. - ur_native_handle_t NativeMem, + ::ur_native_handle_t NativeMem, /// [in] handle of the context object. - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in][optional] pointer to native memory creation properties. const ur_mem_native_properties_t *Properties, /// [out] pointer to handle of buffer memory object created. - ur_mem_handle_t *Mem) { + ::ur_mem_handle_t *Mem) { + auto Context = v1_cast(ContextOpque); bool OwnNativeHandle = Properties ? Properties->isNativeHandleOwned : false; std::shared_lock Lock(Context->Mutex); @@ -1778,7 +1851,7 @@ ur_result_t urMemBufferCreateWithNativeHandle( try { Buffer = new ur_buffer(Context, Size, Device, ur_cast(NativeMem), OwnNativeHandle); - *Mem = reinterpret_cast(Buffer); + *Mem = v1_cast(static_cast(Buffer)); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -1797,7 +1870,7 @@ ur_result_t urMemBufferCreateWithNativeHandle( ContextsLock.lock(); // Retain context to be sure that it is released after all memory // allocations in this context are released. - UR_CALL(ur::level_zero::urContextRetain(Context)); + UR_CALL(ur::level_zero::v1::urContextRetain(ContextOpque)); Context->MemAllocs.emplace(std::piecewise_construct, std::forward_as_tuple(Ptr), @@ -1815,8 +1888,9 @@ ur_result_t urMemBufferCreateWithNativeHandle( // represent the buffer in this context) copy the data to a newly // created device allocation. char *ZeHandleDst; - UR_CALL(Buffer->getZeHandle(ZeHandleDst, ur_mem_handle_t_::write_only, - Device, nullptr, 0u)); + UR_CALL(Buffer->getZeHandle( + ZeHandleDst, ur::level_zero::v1::ur_mem_handle_t_::write_only, Device, + nullptr, 0u)); // Indicate that this buffer has the device buffer mapped to a native buffer // and track the native pointer such that the memory is synced later at @@ -1835,7 +1909,7 @@ ur_result_t urMemBufferCreateWithNativeHandle( ur_result_t urMemGetInfo( /// [in] handle to the memory object being queried. - ur_mem_handle_t Memory, + ::ur_mem_handle_t MemoryOpque, /// [in] type of the info to retrieve. ur_mem_info_t MemInfoType, /// [in] the number of bytes of memory pointed to by pMemInfo. @@ -1848,7 +1922,7 @@ ur_result_t urMemGetInfo( /// [out][optional] pointer to the actual size in bytes of data queried by /// pMemInfo. size_t *PropSizeRet) { - auto Buffer = reinterpret_cast(Memory); + auto Buffer = reinterpret_cast(MemoryOpque); std::shared_lock Lock(Buffer->Mutex); UrReturnHelper ReturnValue(PropSize, MemInfo, PropSizeRet); @@ -1873,7 +1947,7 @@ ur_result_t urMemGetInfo( ur_result_t urMemImageGetInfo( /// [in] handle to the image object being queried. - ur_mem_handle_t /*Memory*/, + ::ur_mem_handle_t /*Memory*/, /// [in] type of image info to retrieve. ur_image_info_t /*ImgInfoType*/, /// [in] the number of bytes of memory pointer to by pImgInfo. @@ -1894,7 +1968,7 @@ ur_result_t urMemImageGetInfo( ur_result_t urEnqueueUSMFill( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] pointer to USM memory object void *Ptr, /// [in] the size in bytes of the pattern. Must be a power of 2 and less @@ -1910,10 +1984,13 @@ ur_result_t urEnqueueUSMFill( /// events that must be complete before this command can be executed. If /// nullptr, the numEventsInWaitList must be 0, indicating that this /// command does not wait on any event to complete. - const ur_event_handle_t *EventWaitList, + const ::ur_event_handle_t *EventWaitList, /// [out][optional] return an event object that identifies this particular /// command instance. - ur_event_handle_t *Event) { + ::ur_event_handle_t *Event) { + auto Queue = v1_cast(QueueOpque); + auto EventWaitListInternal = v1_cast(EventWaitList); + auto EventInternal = v1_cast(Event); std::scoped_lock Lock(Queue->Mutex); return enqueueMemFillHelper( @@ -1921,18 +1998,16 @@ ur_result_t urEnqueueUSMFill( UR_COMMAND_MEM_BUFFER_FILL, Queue, Ptr, Pattern, // It will be interpreted as an 8-bit value, PatternSize, // which is indicated with this pattern_size==1 - Size, NumEventsInWaitList, EventWaitList, Event); + Size, NumEventsInWaitList, EventWaitListInternal, EventInternal); } /// Host Pipes -ur_result_t urEnqueueReadHostPipe(ur_queue_handle_t /*hQueue*/, - ur_program_handle_t /*hProgram*/, - const char * /*pipe_symbol*/, - bool /*blocking*/, void * /*pDst*/, - size_t /*size*/, - uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, - ur_event_handle_t * /*phEvent*/) { +ur_result_t urEnqueueReadHostPipe( + ::ur_queue_handle_t /*hQueue*/, ::ur_program_handle_t /*hProgram*/, + const char * /*pipe_symbol*/, bool /*blocking*/, void * /*pDst*/, + size_t /*size*/, uint32_t /*numEventsInWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, + ::ur_event_handle_t * /*phEvent*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -1940,18 +2015,18 @@ ur_result_t urEnqueueReadHostPipe(ur_queue_handle_t /*hQueue*/, } ur_result_t urEnqueueWriteHostPipe( - ur_queue_handle_t /*hQueue*/, ur_program_handle_t /*hProgram*/, + ::ur_queue_handle_t /*hQueue*/, ::ur_program_handle_t /*hProgram*/, const char * /*pipe_symbol*/, bool /*blocking*/, void * /*pSrc*/, size_t /*size*/, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, - ur_event_handle_t * /*phEvent*/) { + const ::ur_event_handle_t * /*phEventWaitList*/, + ::ur_event_handle_t * /*phEvent*/) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCGetMemHandleExp(ur_context_handle_t, void *pMem, +ur_result_t urIPCGetMemHandleExp(::ur_context_handle_t, void *pMem, void **ppIPCMemHandleData, size_t *pIPCMemHandleDataSizeRet) { umf_memory_pool_handle_t umfPool; @@ -1973,17 +2048,20 @@ ur_result_t urIPCGetMemHandleExp(ur_context_handle_t, void *pMem, umfHandleSize)); } -ur_result_t urIPCPutMemHandleExp(ur_context_handle_t, void *pIPCMemHandleData) { +ur_result_t urIPCPutMemHandleExp(::ur_context_handle_t, + void *pIPCMemHandleData) { return umf::umf2urResult( umfPutIPCHandle(reinterpret_cast(pIPCMemHandleData))); } -ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urIPCOpenMemHandleExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDevice, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem) { - auto *pool = hContext->DefaultPool.getPool(usm::pool_descriptor{ - &hContext->DefaultPool, hContext, hDevice, UR_USM_TYPE_DEVICE, false}); + auto hContext = v1_cast(hContextOpque); + auto *pool = hContext->DefaultPool.getPool( + usm::pool_descriptor{v1_cast(&hContext->DefaultPool), hContextOpque, + hDevice, UR_USM_TYPE_DEVICE, false}); if (!pool) return UR_RESULT_ERROR_INVALID_CONTEXT; umf_memory_pool_handle_t umfPool = pool->UmfPool.get(); @@ -2007,12 +2085,10 @@ ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, ppMem)); } -ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t, void *pMem) { +ur_result_t urIPCCloseMemHandleExp(::ur_context_handle_t, void *pMem) { return umf::umf2urResult(umfCloseIPCHandle(pMem)); } -} // namespace ur::level_zero - // If indirect access tracking is enabled then performs reference counting, // otherwise just calls zeMemAllocDevice. static ur_result_t ZeDeviceMemAllocHelper(void **ResultPtr, @@ -2032,7 +2108,7 @@ static ur_result_t ZeDeviceMemAllocHelper(void **ResultPtr, // indirect access, that is why explicitly retain context to be sure // that it is released after all memory allocations in this context are // released. - UR_CALL(ur::level_zero::urContextRetain(Context)); + UR_CALL(ur::level_zero::v1::urContextRetain(v1_cast(Context))); } ze_device_mem_alloc_desc_t ZeDesc = {}; @@ -2065,7 +2141,7 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, // If the device is still not selected then use the first one in // the context of the buffer. if (!Device) - Device = UrContext->Devices[0]; + Device = UrContext->getDevices()[0]; auto &Allocation = Allocations[Device]; @@ -2107,8 +2183,8 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, ur_usm_desc_t USMDesc{}; USMDesc.align = getAlignment(); ur_usm_pool_handle_t Pool{}; - UR_CALL(ur::level_zero::urUSMHostAlloc( - UrContext, &USMDesc, Pool, Size, + UR_CALL(ur::level_zero::v1::urUSMHostAlloc( + v1_cast(UrContext), &USMDesc, v1_cast(Pool), Size, reinterpret_cast(&ZeHandle))); } else { HostAllocation.ReleaseAction = allocation_t::free_native; @@ -2133,9 +2209,9 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, ur_usm_desc_t USMDesc{}; USMDesc.align = getAlignment(); ur_usm_pool_handle_t Pool{}; - UR_CALL(ur::level_zero::urUSMDeviceAlloc( - UrContext, Device, &USMDesc, Pool, Size, - reinterpret_cast(&ZeHandle))); + UR_CALL(ur::level_zero::v1::urUSMDeviceAlloc( + v1_cast(UrContext), common_cast(Device), &USMDesc, v1_cast(Pool), + Size, reinterpret_cast(&ZeHandle))); } else { Allocation.ReleaseAction = allocation_t::free_native; UR_CALL(ZeDeviceMemAllocHelper(reinterpret_cast(&ZeHandle), @@ -2154,7 +2230,8 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, // For write-only access the allocation contents is not going to be used. // So don't do anything to make it "valid". - bool NeedCopy = AccessMode != ur_mem_handle_t_::write_only; + bool NeedCopy = + AccessMode != ur::level_zero::v1::ur_mem_handle_t_::write_only; // It's also possible that the buffer doesn't have a valid allocation // yet presumably when it is passed to a kernel that will perform // it's intialization. @@ -2163,9 +2240,9 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, } char *ZeHandleSrc = nullptr; if (NeedCopy) { - UR_CALL(getZeHandle(ZeHandleSrc, ur_mem_handle_t_::read_only, - LastDeviceWithValidAllocation, phWaitEvents, - numWaitEvents)); + UR_CALL(getZeHandle( + ZeHandleSrc, ur::level_zero::v1::ur_mem_handle_t_::read_only, + LastDeviceWithValidAllocation, phWaitEvents, numWaitEvents)); // It's possible with the single root-device contexts that // the buffer is represented by the single root-device // allocation and then skip the copy to itself. @@ -2225,8 +2302,9 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, ur_usm_desc_t USMDesc{}; USMDesc.align = getAlignment(); ur_usm_pool_handle_t Pool{}; - UR_CALL(ur::level_zero::urUSMHostAlloc(UrContext, &USMDesc, Pool, - Size, &ZeHandleHost)); + UR_CALL(ur::level_zero::v1::urUSMHostAlloc(v1_cast(UrContext), + &USMDesc, v1_cast(Pool), + Size, &ZeHandleHost)); } else { HostAllocation.ReleaseAction = allocation_t::free_native; UR_CALL(ZeHostMemAllocHelper(&ZeHandleHost, UrContext, Size)); @@ -2264,7 +2342,7 @@ ur_result_t ur_buffer::getBufferZeHandle(char *&ZeHandle, // Invalidate other allocations that would become not valid if // this access is not read-only. - if (AccessMode != ur_mem_handle_t_::read_only) { + if (AccessMode != ur::level_zero::v1::ur_mem_handle_t_::read_only) { for (auto &Alloc : Allocations) { if (Alloc.first != LastDeviceWithValidAllocation) Alloc.second.Valid = false; @@ -2304,8 +2382,7 @@ ur_result_t ur_buffer::free() { (UrContext->ZeCommandListInit, DeviceMappedHostNativePtr, ZeHandle, Size, nullptr, 0, nullptr)); } - UR_CALL(USMFreeHelper(reinterpret_cast(UrContext), - ZeHandle)); + UR_CALL(USMFreeHelper(UrContext, ZeHandle)); break; } case allocation_t::free_native: @@ -2334,8 +2411,8 @@ ur_buffer::ur_buffer(ur_context_handle_t Context, size_t Size, char *HostPtr, // For integrated devices, allocating the buffer in the host memory // enables automatic access from the device, and makes copying // unnecessary in the map/unmap operations. This improves performance. - OnHost = Context->Devices.size() == 1 && - Context->Devices[0]->ZeDeviceProperties->flags & + OnHost = Context->getDevices().size() == 1 && + Context->getDevices()[0]->ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED; // Fill the host allocation data. @@ -2373,8 +2450,8 @@ ur_buffer::ur_buffer(ur_context_handle_t Context, size_t Size, // Check if this buffer can always stay on host OnHost = false; if (!Device) { // Host allocation - if (Context->Devices.size() == 1 && - Context->Devices[0]->ZeDeviceProperties->flags & + if (Context->getDevices().size() == 1 && + Context->getDevices()[0]->ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) { OnHost = true; MapHostPtr = ZeMemHandle; // map to this allocation @@ -2385,38 +2462,40 @@ ur_buffer::ur_buffer(ur_context_handle_t Context, size_t Size, ur_buffer::~ur_buffer() { if (isSubBuffer()) - ur::level_zero::urMemRelease(SubBuffer->Parent); + ur::level_zero::v1::urMemRelease( + reinterpret_cast<::ur_mem_handle_t>(SubBuffer->Parent)); } -ur_result_t ur_mem_handle_t_::getZeHandle(char *&ZeHandle, access_mode_t mode, - ur_device_handle_t Device, - const ur_event_handle_t *phWaitEvents, - uint32_t numWaitEvents) { +ur_result_t ur::level_zero::v1::ur_mem_handle_t_::getZeHandle( + char *&ZeHandle, access_mode_t mode, ur_device_handle_t Device, + const ur_event_handle_t *phWaitEvents, uint32_t numWaitEvents) { switch (mem_type) { - case ur_mem_handle_t_::image: + case ur::level_zero::v1::ur_mem_handle_t_::image: return reinterpret_cast(this)->getImageZeHandle( ZeHandle, mode, Device, phWaitEvents, numWaitEvents); - case ur_mem_handle_t_::buffer: + case ur::level_zero::v1::ur_mem_handle_t_::buffer: return reinterpret_cast(this)->getBufferZeHandle( ZeHandle, mode, Device, phWaitEvents, numWaitEvents); default: - die("ur_mem_handle_t_::getZeHandle: Unhandled memory type"); + die("ur::level_zero::v1::ur_mem_handle_t_::getZeHandle: Unhandled memory " + "type"); } ur::unreachable(); } -ur_result_t ur_mem_handle_t_::getZeHandlePtr( +ur_result_t ur::level_zero::v1::ur_mem_handle_t_::getZeHandlePtr( char **&ZeHandlePtr, access_mode_t mode, ur_device_handle_t Device, const ur_event_handle_t *phWaitEvents, uint32_t numWaitEvents) { switch (mem_type) { - case ur_mem_handle_t_::image: + case ur::level_zero::v1::ur_mem_handle_t_::image: return reinterpret_cast(this)->getImageZeHandlePtr( ZeHandlePtr, mode, Device, phWaitEvents, numWaitEvents); - case ur_mem_handle_t_::buffer: + case ur::level_zero::v1::ur_mem_handle_t_::buffer: return reinterpret_cast(this)->getBufferZeHandlePtr( ZeHandlePtr, mode, Device, phWaitEvents, numWaitEvents); default: - die("ur_mem_handle_t_::getZeHandle: Unhandled memory type"); + die("ur::level_zero::v1::ur_mem_handle_t_::getZeHandle: Unhandled memory " + "type"); } ur::unreachable(); } @@ -2451,3 +2530,5 @@ size_t ur_buffer::getAlignment() const { Alignment = 1UL; return Alignment; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 363ced281ea97..efae4f47b1011 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -18,16 +18,18 @@ #include #include "common.hpp" +#include "common/sampler.hpp" #include "common/ur_ref_count.hpp" #include "context.hpp" #include "event.hpp" #include "program.hpp" #include "queue.hpp" -#include "sampler.hpp" #include struct ur_device_handle_t_; +namespace ur::level_zero::v1 { + bool IsDevicePointer(ur_context_handle_t Context, const void *Ptr); bool IsSharedPointer(ur_context_handle_t Context, const void *Ptr); @@ -60,7 +62,7 @@ ur_result_t enqueueMemCopyRectHelper( uint32_t NumEventsInWaitList, const ur_event_handle_t *EventWaitList, ur_event_handle_t *OutEvent, bool PreferCopyEngine = false); -struct ur_mem_handle_t_ : ur_object { +struct ur_mem_handle_t_ : ur_object_t { // Keeps the PI context of this memory handle. ur_context_handle_t UrContext; @@ -259,3 +261,5 @@ createUrMemFromZeImage(ur_context_handle_t Context, ze_image_handle_t ZeImage, } return UR_RESULT_SUCCESS; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 96f7c9d0c2f99..4e926283e708d 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -1,4 +1,4 @@ -//===---------------- physical_mem.cpp - Level Zero Adapter ---------------===// +//===--------- physical_mem.cpp - Level Zero Adapter ----------------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM @@ -7,17 +7,20 @@ // //===----------------------------------------------------------------------===// -#include "physical_mem.hpp" #include "common.hpp" -#include "context.hpp" -#include "device.hpp" +#include "ur_interface_loader.hpp" +#include "ur_level_zero.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v1 { ur_result_t urPhysicalMemCreate( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + ::ur_context_handle_t hContextOpque, ::ur_device_handle_t hDeviceOpque, + size_t size, [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, - ur_physical_mem_handle_t *phPhysicalMem) { + ::ur_physical_mem_handle_t *phPhysicalMem) { + auto hContext = common_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + ZeStruct PhysicalMemDesc; PhysicalMemDesc.flags = 0; PhysicalMemDesc.size = size; @@ -26,9 +29,8 @@ ur_result_t urPhysicalMemCreate( ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, &PhysicalMemDesc, &ZePhysicalMem)); try { - *phPhysicalMem = - new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, hDevice, size, - /*EnableIpc=*/false); + *phPhysicalMem = common_cast(new ur_physical_mem_handle_t_( + ZePhysicalMem, hContextOpque, hDevice, size, /*EnableIpc=*/false)); } catch (const std::bad_alloc &) { zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; @@ -39,31 +41,34 @@ ur_result_t urPhysicalMemCreate( return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->RefCount.retain(); +ur_result_t urPhysicalMemRetain(::ur_physical_mem_handle_t hPhysicalMemOpque) { + common_cast(hPhysicalMemOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { +ur_result_t urPhysicalMemRelease(::ur_physical_mem_handle_t hPhysicalMemOpque) { + auto hPhysicalMem = common_cast(hPhysicalMemOpque); if (!hPhysicalMem->RefCount.release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { - ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(), - hPhysicalMem->ZePhysicalMem)); + ZE2UR_CALL(zePhysicalMemDestroy, + (common_cast(hPhysicalMem->Context)->getZeHandle(), + hPhysicalMem->ZePhysicalMem)); } delete hPhysicalMem; return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, +ur_result_t urPhysicalMemGetInfo(::ur_physical_mem_handle_t hPhysicalMemOpque, ur_physical_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + auto hPhysicalMem = common_cast(hPhysicalMemOpque); + switch (propName) { case UR_PHYSICAL_MEM_INFO_CONTEXT: return ReturnValue(hPhysicalMem->Context); @@ -85,25 +90,44 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, return UR_RESULT_SUCCESS; } -ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, - ur_physical_mem_handle_t, void **, - size_t *) { +ur_result_t +urIPCGetPhysMemHandleExp(::ur_context_handle_t /* hContext */, + ::ur_physical_mem_handle_t /* hPhysMem */, + void ** /* ppIPCPhysMemHandleData */, + size_t * /* pIPCPhysMemHandleDataSizeRet */) { + UR_LOG_LEGACY(ERR, + logger::LegacyMessage("[UR][L0] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { +ur_result_t urIPCPutPhysMemHandleExp(::ur_context_handle_t /* hContext */, + const void * /* pIPCPhysMemHandleData */) { + UR_LOG_LEGACY(ERR, + logger::LegacyMessage("[UR][L0] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, - const void *, size_t, - ur_physical_mem_handle_t *) { +ur_result_t +urIPCOpenPhysMemHandleExp(::ur_context_handle_t /* hContext */, + ::ur_device_handle_t /* hDevice */, + const void * /* pIPCPhysMemHandleData */, + size_t /* ipcPhysMemHandleDataSize */, + ::ur_physical_mem_handle_t * /* phPhysMem */) { + UR_LOG_LEGACY(ERR, + logger::LegacyMessage("[UR][L0] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, - ur_physical_mem_handle_t) { +ur_result_t +urIPCClosePhysMemHandleExp(::ur_context_handle_t /* hContext */, + ::ur_physical_mem_handle_t /* hPhysMem */) { + UR_LOG_LEGACY(ERR, + logger::LegacyMessage("[UR][L0] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/program.hpp b/unified-runtime/source/adapters/level_zero/program.hpp index d7a593da01bf9..6f469d70b4050 100644 --- a/unified-runtime/source/adapters/level_zero/program.hpp +++ b/unified-runtime/source/adapters/level_zero/program.hpp @@ -8,11 +8,13 @@ //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +#include "common/device.hpp" +#include "common/interfaces.hpp" #include "common/ur_ref_count.hpp" -#include "device.hpp" -struct ur_program_handle_t_ : ur_object { +namespace ur::level_zero { + +struct ur_program_handle_t_ : ur_object_t { // ur_program_handle_t_() {} typedef enum { @@ -271,3 +273,5 @@ struct ur_program_handle_t_ : ur_object { // TODO: Currently interoparability UR API does not support multiple devices. ze_module_handle_t InteropZeModule = nullptr; }; + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index 2afab55a99c6d..1844140ef9c1c 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -14,8 +14,10 @@ #include #include -#include "adapter.hpp" #include "common.hpp" +#include "common/adapter.hpp" +#include "common/device.hpp" +#include "common/platform.hpp" #include "event.hpp" #include "queue.hpp" #include "ur_interface_loader.hpp" @@ -23,6 +25,8 @@ #include "ur_util.hpp" #include "ze_api.h" +namespace ur::level_zero::v1 { + // Hard limit for the event completion batches. static const uint64_t CompletionBatchesMax = [] { // Default value chosen empirically to maximize the number of asynchronous @@ -238,7 +242,7 @@ ur_result_t CleanupEventsInImmCmdLists(ur_queue_handle_t UrQueue, if (!UrQueue || !UrQueue->UsingImmCmdLists) return UR_RESULT_SUCCESS; - ur_event_handle_t_ *UrCompletedEvent = + ur::level_zero::v1::ur_event_handle_t_ *UrCompletedEvent = reinterpret_cast(CompletedEvent); std::vector EventListToCleanup; @@ -274,8 +278,9 @@ ur_result_t CleanupEventsInImmCmdLists(ur_queue_handle_t UrQueue, auto &CmdListEvents = UrCompletedEvent->CommandList.value()->second.EventList; - auto CompletedEventIt = std::find(CmdListEvents.begin(), - CmdListEvents.end(), UrCompletedEvent); + auto CompletedEventIt = + std::find(CmdListEvents.begin(), CmdListEvents.end(), + reinterpret_cast(UrCompletedEvent)); if (CompletedEventIt != CmdListEvents.end()) { // We can cleanup all events prior to the completed event in this // command list and completed event itself. @@ -346,11 +351,9 @@ ur_result_t resetCommandLists(ur_queue_handle_t Queue) { return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - ur_result_t urQueueGetInfo( /// [in] handle of the queue object - ur_queue_handle_t Queue, + ::ur_queue_handle_t QueueOpque, /// [in] name of the queue property to query ur_queue_info_t ParamName, /// [in] size in bytes of the queue property value provided @@ -359,6 +362,7 @@ ur_result_t urQueueGetInfo( void *ParamValue, /// [out] size in bytes returned in queue property value size_t *ParamValueSizeRet) { + auto Queue = v1_cast(QueueOpque); std::shared_lock Lock(Queue->Mutex); UrReturnHelper ReturnValue(ParamValueSize, ParamValue, ParamValueSizeRet); // TODO: consider support for queue properties and size @@ -477,9 +481,9 @@ static bool doEagerInit = [] { ur_result_t urQueueCreate( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] handle of the device object - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in] specifies a list of queue properties and their corresponding /// values. Each property name is immediately followed by the /// corresponding desired value. The list is terminated with a 0. If a @@ -487,7 +491,10 @@ ur_result_t urQueueCreate( /// used. const ur_queue_properties_t *Props, /// [out] pointer to handle of queue object created - ur_queue_handle_t *Queue) { + ::ur_queue_handle_t *QueueOpque) { + auto Context = v1_cast(ContextOpque); + auto Device = common_cast(DeviceOpque); + auto Queue = v1_cast(QueueOpque); ur_queue_flags_t Flags{}; if (Props) { Flags = Props->flags; @@ -511,7 +518,9 @@ ur_result_t urQueueCreate( // Create placeholder queues in the compute queue group. // Actual L0 queues will be created at first use. std::vector ZeComputeCommandQueues( - Device->QueueGroup[ur_queue_handle_t_::queue_type::Compute] + Device + ->QueueGroup + [ur::level_zero::v1::ur_queue_handle_t_::queue_type::Compute] .ZeProperties.numQueues, nullptr); @@ -521,21 +530,25 @@ ur_result_t urQueueCreate( size_t NumCopyGroups = 0; if (Device->hasMainCopyEngine()) { NumCopyGroups += - Device->QueueGroup[ur_queue_handle_t_::queue_type::MainCopy] + Device + ->QueueGroup + [ur::level_zero::v1::ur_queue_handle_t_::queue_type::MainCopy] .ZeProperties.numQueues; } if (Device->hasLinkCopyEngine()) { NumCopyGroups += - Device->QueueGroup[ur_queue_handle_t_::queue_type::LinkCopy] + Device + ->QueueGroup + [ur::level_zero::v1::ur_queue_handle_t_::queue_type::LinkCopy] .ZeProperties.numQueues; } std::vector ZeCopyCommandQueues(NumCopyGroups, nullptr); try { - *Queue = - new ur_queue_handle_t_(ZeComputeCommandQueues, ZeCopyCommandQueues, - Context, Device, true, Flags, ForceComputeIndex); + *Queue = new ur::level_zero::v1::ur_queue_handle_t_( + ZeComputeCommandQueues, ZeCopyCommandQueues, Context, Device, true, + Flags, ForceComputeIndex); } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { @@ -587,7 +600,8 @@ ur_result_t urQueueCreate( ur_result_t urQueueRetain( /// [in] handle of the queue object to get access - ur_queue_handle_t Queue) { + ::ur_queue_handle_t QueueOpque) { + auto Queue = v1_cast(QueueOpque); { std::scoped_lock Lock(Queue->Mutex); Queue->RefCountExternal++; @@ -598,7 +612,8 @@ ur_result_t urQueueRetain( ur_result_t urQueueRelease( /// [in] handle of the queue object to release - ur_queue_handle_t Queue) { + ::ur_queue_handle_t QueueOpque) { + auto Queue = v1_cast(QueueOpque); if (!checkL0LoaderTeardown()) return UR_RESULT_SUCCESS; @@ -629,9 +644,10 @@ ur_result_t urQueueRelease( UR_CALL(Queue->synchronize()); // Cleanup the allocations from 'AsyncPool' made by this queue. - Queue->Context->AsyncPool.cleanupPoolsForQueue(Queue); - std::shared_lock ContextLock(Queue->Context->Mutex); - for (auto &Pool : Queue->Context->UsmPoolHandles) { + auto Context = Queue->Context; + Context->AsyncPool.cleanupPoolsForQueue(Queue); + std::shared_lock ContextLock(Context->Mutex); + for (auto &Pool : Context->UsmPoolHandles) { Pool->cleanupPoolsForQueue(Queue); } ContextLock.unlock(); @@ -669,18 +685,15 @@ ur_result_t urQueueRelease( } } if (Queue->UsingImmCmdLists && Queue->OwnZeCommandQueue) { - std::scoped_lock Lock( - Queue->Context->ZeCommandListCacheMutex); + std::scoped_lock Lock(Context->ZeCommandListCacheMutex); const ur_command_list_info_t &MapEntry = it->second; if (MapEntry.CanReuse) { // Add commandlist to the cache for future use. // It will be deleted when the context is destroyed. auto &ZeCommandListCache = MapEntry.isCopy(Queue) - ? Queue->Context - ->ZeCopyCommandListCache[Queue->Device->ZeDevice] - : Queue->Context - ->ZeComputeCommandListCache[Queue->Device->ZeDevice]; + ? Context->ZeCopyCommandListCache[Queue->Device->ZeDevice] + : Context->ZeComputeCommandListCache[Queue->Device->ZeDevice]; struct l0_command_list_cache_info ListInfo; ListInfo.ZeQueueDesc = it->second.ZeQueueDesc; ListInfo.InOrderList = it->second.IsInOrderList; @@ -718,9 +731,10 @@ ur_result_t urQueueRelease( ur_result_t urQueueGetNativeHandle( /// [in] handle of the queue. - ur_queue_handle_t Queue, ur_queue_native_desc_t *Desc, + ::ur_queue_handle_t QueueOpque, ur_queue_native_desc_t *Desc, /// [out] a pointer to the native handle of the queue. - ur_native_handle_t *NativeQueue) { + ::ur_native_handle_t *NativeQueue) { + auto Queue = v1_cast(QueueOpque); // Lock automatically releases when this goes out of scope. std::shared_lock lock(Queue->Mutex); @@ -754,15 +768,17 @@ ur_result_t urQueueGetNativeHandle( ur_result_t urQueueCreateWithNativeHandle( /// [in] the native handle of the queue. - ur_native_handle_t NativeQueue, + ::ur_native_handle_t NativeQueue, /// [in] handle of the context object - ur_context_handle_t Context, - ur_device_handle_t Device, /// + ::ur_context_handle_t ContextOpque, + ::ur_device_handle_t DeviceOpque, /// const ur_queue_native_properties_t *NativeProperties, /// /// [out] pointer to the handle of the queue object /// created. - ur_queue_handle_t *RetQueue) { + ::ur_queue_handle_t *RetQueueOpque) { + auto Context = v1_cast(ContextOpque); + auto RetQueue = v1_cast(RetQueueOpque); bool OwnNativeHandle = false; ur_queue_flags_t Flags{}; int32_t NativeHandleDesc{}; @@ -793,15 +809,18 @@ ur_result_t urQueueCreateWithNativeHandle( // Get the device handle from first device in the platform // Maybe this is not completely correct. uint32_t NumEntries = 1; - ur_platform_handle_t Platform{}; - ur_adapter_handle_t AdapterHandle = GlobalAdapter; - UR_CALL(ur::level_zero::urPlatformGet(AdapterHandle, NumEntries, &Platform, - nullptr)); + ::ur_platform_handle_t PlatformOpque{}; + ::ur_adapter_handle_t AdapterHandleOpque = + ur::level_zero::common_cast(ur::level_zero::GlobalAdapter); + UR_CALL(ur::level_zero::urPlatformGet(AdapterHandleOpque, NumEntries, + &PlatformOpque, nullptr)); - ur_device_handle_t UrDevice = Device; + ur_device_handle_t UrDevice = common_cast(DeviceOpque); if (UrDevice == nullptr) { - UR_CALL(ur::level_zero::urDeviceGet(Platform, UR_DEVICE_TYPE_GPU, - NumEntries, &UrDevice, nullptr)); + ::ur_device_handle_t UrDeviceOpque = nullptr; + UR_CALL(ur::level_zero::urDeviceGet(PlatformOpque, UR_DEVICE_TYPE_GPU, + NumEntries, &UrDeviceOpque, nullptr)); + UrDevice = common_cast(UrDeviceOpque); } // The NativeHandleDesc has value if if the native handle is an immediate @@ -811,9 +830,11 @@ ur_result_t urQueueCreateWithNativeHandle( std::vector CopyQueues; try { - ur_queue_handle_t_ *Queue = new ur_queue_handle_t_( - ComputeQueues, CopyQueues, Context, UrDevice, OwnNativeHandle, Flags); - *RetQueue = reinterpret_cast(Queue); + ur::level_zero::v1::ur_queue_handle_t_ *Queue = + new ur::level_zero::v1::ur_queue_handle_t_(ComputeQueues, CopyQueues, + Context, UrDevice, + OwnNativeHandle, Flags); + *RetQueue = Queue; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -833,9 +854,11 @@ ur_result_t urQueueCreateWithNativeHandle( std::vector ZeroCopyQueues; try { - ur_queue_handle_t_ *Queue = new ur_queue_handle_t_( - ZeQueues, ZeroCopyQueues, Context, UrDevice, OwnNativeHandle, Flags); - *RetQueue = reinterpret_cast(Queue); + ur::level_zero::v1::ur_queue_handle_t_ *Queue = + new ur::level_zero::v1::ur_queue_handle_t_(ZeQueues, ZeroCopyQueues, + Context, UrDevice, + OwnNativeHandle, Flags); + *RetQueue = Queue; } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -849,7 +872,8 @@ ur_result_t urQueueCreateWithNativeHandle( ur_result_t urQueueFinish( /// [in] handle of the queue to be finished. - ur_queue_handle_t Queue) { + ::ur_queue_handle_t QueueOpque) { + auto Queue = v1_cast(QueueOpque); if (Queue->UsingImmCmdLists) { // Lock automatically releases when this goes out of scope. std::scoped_lock Lock(Queue->Mutex); @@ -912,9 +936,10 @@ ur_result_t urQueueFinish( resetCommandLists(Queue); } - Queue->Context->AsyncPool.cleanupPoolsForQueue(Queue); - std::shared_lock ContextLock(Queue->Context->Mutex); - for (auto &Pool : Queue->Context->UsmPoolHandles) { + auto Context = Queue->Context; + Context->AsyncPool.cleanupPoolsForQueue(Queue); + std::shared_lock ContextLock(Context->Mutex); + for (auto &Pool : Context->UsmPoolHandles) { Pool->cleanupPoolsForQueue(Queue); } ContextLock.unlock(); @@ -924,12 +949,13 @@ ur_result_t urQueueFinish( ur_result_t urQueueFlush( /// [in] handle of the queue to be flushed. - ur_queue_handle_t Queue) { + ::ur_queue_handle_t QueueOpque) { + auto Queue = v1_cast(QueueOpque); std::scoped_lock Lock(Queue->Mutex); return Queue->executeAllOpenCommandLists(); } -ur_result_t urQueueBeginGraphCaptureExp(ur_queue_handle_t /* hQueue */) { +ur_result_t urQueueBeginGraphCaptureExp(::ur_queue_handle_t /* hQueue */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -937,34 +963,34 @@ ur_result_t urQueueBeginGraphCaptureExp(ur_queue_handle_t /* hQueue */) { } ur_result_t -urQueueBeginCaptureIntoGraphExp(ur_queue_handle_t /* hQueue */, - ur_exp_graph_handle_t /* hGraph */) { +urQueueBeginCaptureIntoGraphExp(::ur_queue_handle_t /* hQueue */, + ::ur_exp_graph_handle_t /* hGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urQueueEndGraphCaptureExp(ur_queue_handle_t /* hQueue */, - ur_exp_graph_handle_t * /* phGraph */) { +ur_result_t urQueueEndGraphCaptureExp(::ur_queue_handle_t /* hQueue */, + ::ur_exp_graph_handle_t * /* phGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urEnqueueGraphExp(ur_queue_handle_t /* hQueue */, - ur_exp_executable_graph_handle_t /* hGraph */, +ur_result_t urEnqueueGraphExp(::ur_queue_handle_t /* hQueue */, + ::ur_exp_executable_graph_handle_t /* hGraph */, uint32_t /* numEventsInWaitList */, - const ur_event_handle_t * /* phEventWaitList */, - ur_event_handle_t * /* phEvent */) { + const ::ur_event_handle_t * /* phEventWaitList */, + ::ur_event_handle_t * /* phEvent */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urQueueIsGraphCaptureEnabledExp(ur_queue_handle_t /* hQueue */, +ur_result_t urQueueIsGraphCaptureEnabledExp(::ur_queue_handle_t /* hQueue */, bool * /* hResult */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), @@ -972,8 +998,8 @@ ur_result_t urQueueIsGraphCaptureEnabledExp(ur_queue_handle_t /* hQueue */, return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urQueueGetGraphExp(ur_queue_handle_t /* hQueue */, - ur_exp_graph_handle_t * /* phGraph */) { +ur_result_t urQueueGetGraphExp(::ur_queue_handle_t /* hQueue */, + ::ur_exp_graph_handle_t * /* phGraph */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); @@ -981,19 +1007,18 @@ ur_result_t urQueueGetGraphExp(ur_queue_handle_t /* hQueue */, } ur_result_t -urEnqueueHostTaskExp(ur_queue_handle_t /* hQueue */, +urEnqueueHostTaskExp(::ur_queue_handle_t /* hQueue */, ur_exp_host_task_function_t /* pfnHostTask */, void * /* data */, const ur_exp_host_task_properties_t * /* pProperties */, uint32_t /* numEventsInWaitList */, - const ur_event_handle_t * /* phEventWaitList */, - ur_event_handle_t * /* phEvent */) { + const ::ur_event_handle_t * /* phEventWaitList */, + ::ur_event_handle_t * /* phEvent */) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not implemented!"), "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero // Configuration of the command-list batching. struct zeCommandListBatchConfig { @@ -1148,7 +1173,7 @@ static const zeCommandListBatchConfig ZeCommandListBatchCopyConfig = [] { return ZeCommandListBatchConfig(IsCopy{true}); }(); -ur_queue_handle_t_::ur_queue_handle_t_( +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_handle_t_( std::vector &ComputeQueues, std::vector &CopyQueues, ur_context_handle_t Context, ur_device_handle_t Device, @@ -1266,7 +1291,8 @@ ur_queue_handle_t_::ur_queue_handle_t_( true /*Only Allow Driver In Order List if requested*/); } -void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) { +void ur::level_zero::v1::ur_queue_handle_t_::adjustBatchSizeForFullBatch( + bool IsCopy) { auto &CommandBatch = IsCopy ? CopyCommandBatch : ComputeCommandBatch; auto &ZeCommandListBatchConfig = IsCopy ? ZeCommandListBatchCopyConfig : ZeCommandListBatchComputeConfig; @@ -1293,7 +1319,8 @@ void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) { } } -void ur_queue_handle_t_::adjustBatchSizeForPartialBatch(bool IsCopy) { +void ur::level_zero::v1::ur_queue_handle_t_::adjustBatchSizeForPartialBatch( + bool IsCopy) { auto &CommandBatch = IsCopy ? CopyCommandBatch : ComputeCommandBatch; auto &ZeCommandListBatchConfig = IsCopy ? ZeCommandListBatchCopyConfig : ZeCommandListBatchComputeConfig; @@ -1319,9 +1346,8 @@ void ur_queue_handle_t_::adjustBatchSizeForPartialBatch(bool IsCopy) { } } -ur_result_t -ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, - bool IsBlocking, bool OKToBatchCommand) { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::executeCommandList( + ur_command_list_ptr_t CommandList, bool IsBlocking, bool OKToBatchCommand) { // Do nothing if command list is already closed. if (CommandList->second.IsClosed) return UR_RESULT_SUCCESS; @@ -1558,12 +1584,12 @@ ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, return UR_RESULT_SUCCESS; } -bool ur_queue_handle_t_::doReuseDiscardedEvents() { +bool ur::level_zero::v1::ur_queue_handle_t_::doReuseDiscardedEvents() { return ReuseDiscardedEvents && isInOrderQueue() && isDiscardEvents(); } -ur_result_t -ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::resetDiscardedEvent( + ur_command_list_ptr_t CommandList) { if (LastCommandEvent && LastCommandEvent->IsDiscarded) { ZE2UR_CALL(zeCommandListAppendBarrier, (CommandList->first, nullptr, 1, &(LastCommandEvent->ZeEvent))); @@ -1575,9 +1601,9 @@ ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) { // Create new ur_event_handle_t but with the same ze_event_handle_t. We are // going to use this ur_event_handle_t for the next command with discarded // event. - ur_event_handle_t_ *UREvent; + ur::level_zero::v1::ur_event_handle_t_ *UREvent; try { - UREvent = new ur_event_handle_t_( + UREvent = new ur::level_zero::v1::ur_event_handle_t_( LastCommandEvent->ZeEvent, LastCommandEvent->ZeEventPool, reinterpret_cast(Context), UR_EXT_COMMAND_TYPE_USER, true); @@ -1596,7 +1622,8 @@ ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) { return UR_RESULT_SUCCESS; } -ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::addEventToQueueCache( + ur_event_handle_t Event) { if (!Event->IsMultiDevice) { auto EventCachesMap = Event->isHostVisible() ? &EventCachesDeviceMap[0] : &EventCachesDeviceMap[1]; @@ -1612,19 +1639,20 @@ ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { return UR_RESULT_SUCCESS; } -void ur_queue_handle_t_::active_barriers::add(ur_event_handle_t &Event) { +void ur::level_zero::v1::ur_queue_handle_t_::active_barriers::add( + ur_event_handle_t &Event) { Event->RefCount.retain(); Events.push_back(Event); } -ur_result_t ur_queue_handle_t_::active_barriers::clear() { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::active_barriers::clear() { for (const auto &Event : Events) UR_CALL(urEventReleaseInternal(Event)); Events.clear(); return UR_RESULT_SUCCESS; } -void ur_queue_handle_t_::clearEndTimeRecordings() { +void ur::level_zero::v1::ur_queue_handle_t_::clearEndTimeRecordings() { const double ZeTimerResolution = Device->getTimerResolution(); const uint64_t TimestampMaxValue = Device->getTimestampMask(); @@ -1696,39 +1724,40 @@ ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { return UR_RESULT_SUCCESS; } -bool ur_queue_handle_t_::isBatchingAllowed(bool IsCopy) const { +bool ur::level_zero::v1::ur_queue_handle_t_::isBatchingAllowed( + bool IsCopy) const { auto &CommandBatch = IsCopy ? CopyCommandBatch : ComputeCommandBatch; return (CommandBatch.QueueBatchSize > 0 && ((UrL0Serialize & UrL0SerializeBlock) == 0)); } -bool ur_queue_handle_t_::isDiscardEvents() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isDiscardEvents() const { return ((this->Properties & UR_QUEUE_FLAG_DISCARD_EVENTS) != 0); } -bool ur_queue_handle_t_::isPriorityLow() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isPriorityLow() const { return ((this->Properties & UR_QUEUE_FLAG_PRIORITY_LOW) != 0); } -bool ur_queue_handle_t_::isPriorityHigh() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isPriorityHigh() const { return ((this->Properties & UR_QUEUE_FLAG_PRIORITY_HIGH) != 0); } -bool ur_queue_handle_t_::isBatchedSubmission() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isBatchedSubmission() const { return ((this->Properties & UR_QUEUE_FLAG_SUBMISSION_BATCHED) != 0); } -bool ur_queue_handle_t_::isImmediateSubmission() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isImmediateSubmission() const { return ((this->Properties & UR_QUEUE_FLAG_SUBMISSION_IMMEDIATE) != 0); } -bool ur_queue_handle_t_::isInOrderQueue() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isInOrderQueue() const { // If out-of-order queue property is not set, then this is a in-order queue. return ((this->Properties & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) == 0); } -bool ur_queue_handle_t_::isLowPowerEvents() const { +bool ur::level_zero::v1::ur_queue_handle_t_::isLowPowerEvents() const { return ((this->Properties & UR_QUEUE_FLAG_LOW_POWER_EVENTS_SUPPORT_EXP) != 0); } @@ -1757,11 +1786,11 @@ ur_result_t CleanupEventListFromResetCmdList( // TODO: Event release in immediate commandlist mode is driven by the SYCL // runtime. Need to investigate whether relase can be done earlier, at sync // points such as this, to reduce total number of active Events. -ur_result_t ur_queue_handle_t_::synchronize() { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::synchronize() { if (!Healthy) return UR_RESULT_SUCCESS; - auto syncImmCmdList = [](ur_queue_handle_t_ *Queue, + auto syncImmCmdList = [](ur::level_zero::v1::ur_queue_handle_t_ *Queue, ur_command_list_ptr_t ImmCmdList) { if (ImmCmdList == Queue->CommandListMap.end()) return UR_RESULT_SUCCESS; @@ -1852,8 +1881,9 @@ ur_result_t ur_queue_handle_t_::synchronize() { return UR_RESULT_SUCCESS; } -ur_event_handle_t ur_queue_handle_t_::getEventFromQueueCache(bool IsMultiDevice, - bool HostVisible) { +ur_event_handle_t +ur::level_zero::v1::ur_queue_handle_t_::getEventFromQueueCache( + bool IsMultiDevice, bool HostVisible) { std::list *Cache; if (!IsMultiDevice) { @@ -2003,19 +2033,19 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // event will not be waited/released by SYCL RT, so it must be destroyed by // EventRelease in resetCommandList. if (!IsInternal) - UR_CALL(ur::level_zero::urEventRetain(*Event)); + UR_CALL(ur::level_zero::v1::urEventRetain(v1_cast(*Event))); return UR_RESULT_SUCCESS; } -void ur_queue_handle_t_::CaptureIndirectAccesses() { +void ur::level_zero::v1::ur_queue_handle_t_::CaptureIndirectAccesses() { for (auto &Kernel : KernelsToBeSubmitted) { if (!Kernel->hasIndirectAccess()) continue; auto &Contexts = Device->Platform->Contexts; for (auto &Ctx : Contexts) { - for (auto &Elem : Ctx->MemAllocs) { + for (auto &Elem : v1_cast(Ctx)->MemAllocs) { const auto &Pair = Kernel->MemAllocs.insert(&Elem); // Kernel is referencing this memory allocation from now. // If this memory allocation was already captured for this kernel, it @@ -2032,8 +2062,9 @@ void ur_queue_handle_t_::CaptureIndirectAccesses() { KernelsToBeSubmitted.clear(); } -ur_result_t ur_queue_handle_t_::signalEventFromCmdListIfLastEventDiscarded( - ur_command_list_ptr_t CommandList) { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_:: + signalEventFromCmdListIfLastEventDiscarded( + ur_command_list_ptr_t CommandList) { // We signal new event at the end of command list only if we have queue with // discard_events property and the last command event is discarded. if (!(doReuseDiscardedEvents() && LastCommandEvent && @@ -2058,7 +2089,8 @@ ur_result_t ur_queue_handle_t_::signalEventFromCmdListIfLastEventDiscarded( return UR_RESULT_SUCCESS; } -ur_result_t ur_queue_handle_t_::executeOpenCommandList(bool IsCopy) { +ur_result_t +ur::level_zero::v1::ur_queue_handle_t_::executeOpenCommandList(bool IsCopy) { auto &CommandBatch = IsCopy ? CopyCommandBatch : ComputeCommandBatch; // If there are any commands still in the open command list for this // queue, then close and execute that command list now. @@ -2074,10 +2106,11 @@ ur_result_t ur_queue_handle_t_::executeOpenCommandList(bool IsCopy) { return UR_RESULT_SUCCESS; } -ur_result_t ur_queue_handle_t_::resetCommandList( +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::resetCommandList( ur_command_list_ptr_t CommandList, bool MakeAvailable, std::vector &EventListToCleanup, bool CheckStatus) { - bool UseCopyEngine = CommandList->second.isCopy(this); + auto self = this; + bool UseCopyEngine = CommandList->second.isCopy(self); // Immediate commandlists do not have an associated fence. if (CommandList->second.ZeFence != nullptr) { @@ -2136,7 +2169,7 @@ ur_result_t ur_queue_handle_t_::resetCommandList( } if (auto &completions = CommandList->second.completions; completions) { - if (completions->tryCleanup(this, CommandList->first, EventList, + if (completions->tryCleanup(self, CommandList->first, EventList, EventListToCleanup) == UR_RESULT_SUCCESS) { return UR_RESULT_SUCCESS; } @@ -2192,7 +2225,8 @@ void ur_command_list_info_t::append(ur_event_handle_t Event) { } ur_command_list_ptr_t -ur_queue_handle_t_::eventOpenCommandList(ur_event_handle_t Event) { +ur::level_zero::v1::ur_queue_handle_t_::eventOpenCommandList( + ur_event_handle_t Event) { using IsCopy = bool; if (UsingImmCmdLists) { @@ -2217,7 +2251,7 @@ ur_queue_handle_t_::eventOpenCommandList(ur_event_handle_t Event) { return CommandListMap.end(); } -void ur_queue_handle_t_::ur_queue_group_t::setImmCmdList( +void ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t::setImmCmdList( ur_queue_handle_t queue, ze_command_list_handle_t ZeCommandList) { // An immediate command list was given to us but we don't have the queue // descriptor information. Create a dummy and note that it is not recycleable. @@ -2234,15 +2268,16 @@ void ur_queue_handle_t_::ur_queue_group_t::setImmCmdList( .first); } -ur_queue_handle_t_::ur_queue_group_t & -ur_queue_handle_t_::getQueueGroup(bool UseCopyEngine) { +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t & +ur::level_zero::v1::ur_queue_handle_t_::getQueueGroup(bool UseCopyEngine) { auto &Map = (UseCopyEngine ? CopyQueueGroupsByTID : ComputeQueueGroupsByTID); return Map.get(); } // Return the index of the next queue to use based on a // round robin strategy and the queue group ordinal. -uint32_t ur_queue_handle_t_::ur_queue_group_t::getQueueIndex( +uint32_t +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t::getQueueIndex( uint32_t *QueueGroupOrdinal, uint32_t *QueueIndex, bool QueryOnly) { auto CurrentIndex = NextIndex; @@ -2276,7 +2311,8 @@ uint32_t ur_queue_handle_t_::ur_queue_group_t::getQueueIndex( // This function will return one of possibly multiple available native // queues and the value of the queue group ordinal. ze_command_queue_handle_t & -ur_queue_handle_t_::ur_queue_group_t::getZeQueue(uint32_t *QueueGroupOrdinal) { +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t::getZeQueue( + uint32_t *QueueGroupOrdinal) { // QueueIndex is the proper L0 index. // Index is the plugins concept of index, with main and link copy engines in @@ -2322,7 +2358,8 @@ ur_queue_handle_t_::ur_queue_group_t::getZeQueue(uint32_t *QueueGroupOrdinal) { return ZeQueue; } -int32_t ur_queue_handle_t_::ur_queue_group_t::getCmdQueueOrdinal( +int32_t +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t::getCmdQueueOrdinal( ze_command_queue_handle_t CmdQueue) { // Find out the right queue group ordinal (first queue might be "main" or // "link") @@ -2334,7 +2371,7 @@ int32_t ur_queue_handle_t_::ur_queue_group_t::getCmdQueueOrdinal( return Queue->Device->QueueGroup[QueueType].ZeOrdinal; } -bool ur_queue_handle_t_::useCompletionBatching() { +bool ur::level_zero::v1::ur_queue_handle_t_::useCompletionBatching() { static bool enabled = getenv_tobool( "UR_L0_IMMEDIATE_COMMANDLISTS_BATCH_EVENT_COMPLETIONS", false); return enabled && !isInOrderQueue() && UsingImmCmdLists; @@ -2344,7 +2381,7 @@ bool ur_queue_handle_t_::useCompletionBatching() { // fence tracking its completion. This command list & fence are added to the // map of command lists in this queue with ZeFenceInUse = false. // The caller must hold a lock of the queue already. -ur_result_t ur_queue_handle_t_::createCommandList( +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::createCommandList( bool UseCopyEngine, ur_command_list_ptr_t &CommandList, ze_command_queue_handle_t *ForcedCmdQueue) { @@ -2395,9 +2432,8 @@ ur_result_t ur_queue_handle_t_::createCommandList( return UR_RESULT_SUCCESS; } -ur_result_t -ur_queue_handle_t_::insertActiveBarriers(ur_command_list_ptr_t &CmdList, - bool UseCopyEngine) { +ur_result_t ur::level_zero::v1::ur_queue_handle_t_::insertActiveBarriers( + ur_command_list_ptr_t &CmdList, bool UseCopyEngine) { // Early exit if there are no active barriers. if (ActiveBarriers.empty()) return UR_RESULT_SUCCESS; @@ -2439,7 +2475,8 @@ ur_queue_handle_t_::insertActiveBarriers(ur_command_list_ptr_t &CmdList, return UR_RESULT_SUCCESS; } -ur_result_t ur_queue_handle_t_::insertStartBarrierIfDiscardEventsMode( +ur_result_t +ur::level_zero::v1::ur_queue_handle_t_::insertStartBarrierIfDiscardEventsMode( ur_command_list_ptr_t &CmdList) { // If current command list is different from the last command list then insert // a barrier waiting for the last command event. @@ -2465,7 +2502,8 @@ static const bool UseCopyEngineForInOrderQueue = [] { (std::stoi(CopyEngineForInOrderQueue) != 0)); }(); -bool ur_queue_handle_t_::useCopyEngine(bool PreferCopyEngine) const { +bool ur::level_zero::v1::ur_queue_handle_t_::useCopyEngine( + bool PreferCopyEngine) const { auto InitialCopyGroup = CopyQueueGroupsByTID.begin()->second; return PreferCopyEngine && InitialCopyGroup.ZeQueues.size() > 0 && (!isInOrderQueue() || UseCopyEngineForInOrderQueue); @@ -2473,7 +2511,8 @@ bool ur_queue_handle_t_::useCopyEngine(bool PreferCopyEngine) const { // This function will return one of po6ssibly multiple available // immediate commandlists associated with this Queue. -ur_command_list_ptr_t &ur_queue_handle_t_::ur_queue_group_t::getImmCmdList() { +ur_command_list_ptr_t & +ur::level_zero::v1::ur_queue_handle_t_::ur_queue_group_t::getImmCmdList() { uint32_t QueueIndex, QueueOrdinal; auto Index = getQueueIndex(&QueueOrdinal, &QueueIndex); @@ -2590,7 +2629,10 @@ static const size_t ImmCmdListsEventCleanupThreshold = [] { return Threshold; }(); -size_t ur_queue_handle_t_::getImmdCmmdListsEventCleanupThreshold() { +size_t ur::level_zero::v1::ur_queue_handle_t_:: + getImmdCmmdListsEventCleanupThreshold() { return useCompletionBatching() ? CompletionEventsPerBatch : ImmCmdListsEventCleanupThreshold; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/queue.hpp b/unified-runtime/source/adapters/level_zero/queue.hpp index 30bc4cecab9ac..7dafb23af5aef 100644 --- a/unified-runtime/source/adapters/level_zero/queue.hpp +++ b/unified-runtime/source/adapters/level_zero/queue.hpp @@ -24,12 +24,12 @@ #include #include "common.hpp" +#include "common/device.hpp" #include "common/ur_ref_count.hpp" -#include "device.hpp" -extern "C" { +namespace ur::level_zero::v1 { + ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue); -} // extern "C" struct ur_completion_batch; using ur_completion_batch_list = std::list; @@ -224,7 +224,7 @@ using ur_command_list_map_t = // The iterator pointing to a specific command-list in use. using ur_command_list_ptr_t = ur_command_list_map_t::iterator; -struct ur_queue_handle_t_ : ur_object { +struct ur_queue_handle_t_ : ur_object_t { ur_queue_handle_t_(std::vector &ComputeQueues, std::vector &CopyQueues, ur_context_handle_t Context, ur_device_handle_t Device, @@ -738,3 +738,5 @@ ur_result_t setSignalEvent(ur_queue_handle_t Queue, bool UseCopyEngine, ur_result_t CleanupEventListFromResetCmdList( std::vector &EventListToCleanup, bool QueueLocked = false); + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp index 35cbb5404b325..cb5da50c4768c 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp @@ -26,8 +26,9 @@ static ur_result_t validateProcInputs(ur_api_version_t version, return UR_RESULT_SUCCESS; } +#include "ur_interface_loader_common_forwarders.hpp" #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO -namespace ur::level_zero { +namespace ur::level_zero::v1 { #else extern "C" { #endif @@ -39,14 +40,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetAdapterProcAddrTable( return result; } - pDdiTable->pfnGet = ur::level_zero::urAdapterGet; - pDdiTable->pfnRelease = ur::level_zero::urAdapterRelease; - pDdiTable->pfnRetain = ur::level_zero::urAdapterRetain; - pDdiTable->pfnGetLastError = ur::level_zero::urAdapterGetLastError; - pDdiTable->pfnGetInfo = ur::level_zero::urAdapterGetInfo; - pDdiTable->pfnSetLoggerCallback = ur::level_zero::urAdapterSetLoggerCallback; + pDdiTable->pfnGet = ur::level_zero::v1::urAdapterGet; + pDdiTable->pfnRelease = ur::level_zero::v1::urAdapterRelease; + pDdiTable->pfnRetain = ur::level_zero::v1::urAdapterRetain; + pDdiTable->pfnGetLastError = ur::level_zero::v1::urAdapterGetLastError; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urAdapterGetInfo; + pDdiTable->pfnSetLoggerCallback = + ur::level_zero::v1::urAdapterSetLoggerCallback; pDdiTable->pfnSetLoggerCallbackLevel = - ur::level_zero::urAdapterSetLoggerCallbackLevel; + ur::level_zero::v1::urAdapterSetLoggerCallbackLevel; return result; } @@ -59,48 +61,49 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( } pDdiTable->pfnUnsampledImageHandleDestroyExp = - ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp; + ur::level_zero::v1::urBindlessImagesUnsampledImageHandleDestroyExp; pDdiTable->pfnSampledImageHandleDestroyExp = - ur::level_zero::urBindlessImagesSampledImageHandleDestroyExp; + ur::level_zero::v1::urBindlessImagesSampledImageHandleDestroyExp; pDdiTable->pfnImageAllocateExp = - ur::level_zero::urBindlessImagesImageAllocateExp; - pDdiTable->pfnImageFreeExp = ur::level_zero::urBindlessImagesImageFreeExp; + ur::level_zero::v1::urBindlessImagesImageAllocateExp; + pDdiTable->pfnImageFreeExp = ur::level_zero::v1::urBindlessImagesImageFreeExp; pDdiTable->pfnUnsampledImageCreateExp = - ur::level_zero::urBindlessImagesUnsampledImageCreateExp; + ur::level_zero::v1::urBindlessImagesUnsampledImageCreateExp; pDdiTable->pfnSampledImageCreateExp = - ur::level_zero::urBindlessImagesSampledImageCreateExp; - pDdiTable->pfnImageCopyExp = ur::level_zero::urBindlessImagesImageCopyExp; + ur::level_zero::v1::urBindlessImagesSampledImageCreateExp; + pDdiTable->pfnImageCopyExp = ur::level_zero::v1::urBindlessImagesImageCopyExp; pDdiTable->pfnImageGetInfoExp = - ur::level_zero::urBindlessImagesImageGetInfoExp; + ur::level_zero::v1::urBindlessImagesImageGetInfoExp; pDdiTable->pfnGetImageMemoryHandleTypeSupportExp = - ur::level_zero::urBindlessImagesGetImageMemoryHandleTypeSupportExp; + ur::level_zero::v1::urBindlessImagesGetImageMemoryHandleTypeSupportExp; pDdiTable->pfnGetImageUnsampledHandleSupportExp = - ur::level_zero::urBindlessImagesGetImageUnsampledHandleSupportExp; + ur::level_zero::v1::urBindlessImagesGetImageUnsampledHandleSupportExp; pDdiTable->pfnGetImageSampledHandleSupportExp = - ur::level_zero::urBindlessImagesGetImageSampledHandleSupportExp; + ur::level_zero::v1::urBindlessImagesGetImageSampledHandleSupportExp; pDdiTable->pfnMipmapGetLevelExp = - ur::level_zero::urBindlessImagesMipmapGetLevelExp; - pDdiTable->pfnMipmapFreeExp = ur::level_zero::urBindlessImagesMipmapFreeExp; + ur::level_zero::v1::urBindlessImagesMipmapGetLevelExp; + pDdiTable->pfnMipmapFreeExp = + ur::level_zero::v1::urBindlessImagesMipmapFreeExp; pDdiTable->pfnImportExternalMemoryExp = - ur::level_zero::urBindlessImagesImportExternalMemoryExp; + ur::level_zero::v1::urBindlessImagesImportExternalMemoryExp; pDdiTable->pfnMapExternalArrayExp = - ur::level_zero::urBindlessImagesMapExternalArrayExp; + ur::level_zero::v1::urBindlessImagesMapExternalArrayExp; pDdiTable->pfnMapExternalLinearMemoryExp = - ur::level_zero::urBindlessImagesMapExternalLinearMemoryExp; + ur::level_zero::v1::urBindlessImagesMapExternalLinearMemoryExp; pDdiTable->pfnReleaseExternalMemoryExp = - ur::level_zero::urBindlessImagesReleaseExternalMemoryExp; + ur::level_zero::v1::urBindlessImagesReleaseExternalMemoryExp; pDdiTable->pfnFreeMappedLinearMemoryExp = - ur::level_zero::urBindlessImagesFreeMappedLinearMemoryExp; + ur::level_zero::v1::urBindlessImagesFreeMappedLinearMemoryExp; pDdiTable->pfnSupportsImportingHandleTypeExp = - ur::level_zero::urBindlessImagesSupportsImportingHandleTypeExp; + ur::level_zero::v1::urBindlessImagesSupportsImportingHandleTypeExp; pDdiTable->pfnImportExternalSemaphoreExp = - ur::level_zero::urBindlessImagesImportExternalSemaphoreExp; + ur::level_zero::v1::urBindlessImagesImportExternalSemaphoreExp; pDdiTable->pfnReleaseExternalSemaphoreExp = - ur::level_zero::urBindlessImagesReleaseExternalSemaphoreExp; + ur::level_zero::v1::urBindlessImagesReleaseExternalSemaphoreExp; pDdiTable->pfnWaitExternalSemaphoreExp = - ur::level_zero::urBindlessImagesWaitExternalSemaphoreExp; + ur::level_zero::v1::urBindlessImagesWaitExternalSemaphoreExp; pDdiTable->pfnSignalExternalSemaphoreExp = - ur::level_zero::urBindlessImagesSignalExternalSemaphoreExp; + ur::level_zero::v1::urBindlessImagesSignalExternalSemaphoreExp; return result; } @@ -112,47 +115,47 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetCommandBufferExpProcAddrTable( return result; } - pDdiTable->pfnCreateExp = ur::level_zero::urCommandBufferCreateExp; - pDdiTable->pfnRetainExp = ur::level_zero::urCommandBufferRetainExp; - pDdiTable->pfnReleaseExp = ur::level_zero::urCommandBufferReleaseExp; - pDdiTable->pfnFinalizeExp = ur::level_zero::urCommandBufferFinalizeExp; + pDdiTable->pfnCreateExp = ur::level_zero::v1::urCommandBufferCreateExp; + pDdiTable->pfnRetainExp = ur::level_zero::v1::urCommandBufferRetainExp; + pDdiTable->pfnReleaseExp = ur::level_zero::v1::urCommandBufferReleaseExp; + pDdiTable->pfnFinalizeExp = ur::level_zero::v1::urCommandBufferFinalizeExp; pDdiTable->pfnAppendKernelLaunchExp = - ur::level_zero::urCommandBufferAppendKernelLaunchExp; + ur::level_zero::v1::urCommandBufferAppendKernelLaunchExp; pDdiTable->pfnAppendKernelLaunchWithArgsExp = - ur::level_zero::urCommandBufferAppendKernelLaunchWithArgsExp; + ur::level_zero::v1::urCommandBufferAppendKernelLaunchWithArgsExp; pDdiTable->pfnAppendUSMMemcpyExp = - ur::level_zero::urCommandBufferAppendUSMMemcpyExp; + ur::level_zero::v1::urCommandBufferAppendUSMMemcpyExp; pDdiTable->pfnAppendUSMFillExp = - ur::level_zero::urCommandBufferAppendUSMFillExp; + ur::level_zero::v1::urCommandBufferAppendUSMFillExp; pDdiTable->pfnAppendMemBufferCopyExp = - ur::level_zero::urCommandBufferAppendMemBufferCopyExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferCopyExp; pDdiTable->pfnAppendMemBufferWriteExp = - ur::level_zero::urCommandBufferAppendMemBufferWriteExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferWriteExp; pDdiTable->pfnAppendMemBufferReadExp = - ur::level_zero::urCommandBufferAppendMemBufferReadExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferReadExp; pDdiTable->pfnAppendMemBufferCopyRectExp = - ur::level_zero::urCommandBufferAppendMemBufferCopyRectExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferCopyRectExp; pDdiTable->pfnAppendMemBufferWriteRectExp = - ur::level_zero::urCommandBufferAppendMemBufferWriteRectExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferWriteRectExp; pDdiTable->pfnAppendMemBufferReadRectExp = - ur::level_zero::urCommandBufferAppendMemBufferReadRectExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferReadRectExp; pDdiTable->pfnAppendMemBufferFillExp = - ur::level_zero::urCommandBufferAppendMemBufferFillExp; + ur::level_zero::v1::urCommandBufferAppendMemBufferFillExp; pDdiTable->pfnAppendUSMPrefetchExp = - ur::level_zero::urCommandBufferAppendUSMPrefetchExp; + ur::level_zero::v1::urCommandBufferAppendUSMPrefetchExp; pDdiTable->pfnAppendUSMAdviseExp = - ur::level_zero::urCommandBufferAppendUSMAdviseExp; + ur::level_zero::v1::urCommandBufferAppendUSMAdviseExp; pDdiTable->pfnAppendNativeCommandExp = - ur::level_zero::urCommandBufferAppendNativeCommandExp; + ur::level_zero::v1::urCommandBufferAppendNativeCommandExp; pDdiTable->pfnUpdateKernelLaunchExp = - ur::level_zero::urCommandBufferUpdateKernelLaunchExp; + ur::level_zero::v1::urCommandBufferUpdateKernelLaunchExp; pDdiTable->pfnUpdateSignalEventExp = - ur::level_zero::urCommandBufferUpdateSignalEventExp; + ur::level_zero::v1::urCommandBufferUpdateSignalEventExp; pDdiTable->pfnUpdateWaitEventsExp = - ur::level_zero::urCommandBufferUpdateWaitEventsExp; - pDdiTable->pfnGetInfoExp = ur::level_zero::urCommandBufferGetInfoExp; + ur::level_zero::v1::urCommandBufferUpdateWaitEventsExp; + pDdiTable->pfnGetInfoExp = ur::level_zero::v1::urCommandBufferGetInfoExp; pDdiTable->pfnGetNativeHandleExp = - ur::level_zero::urCommandBufferGetNativeHandleExp; + ur::level_zero::v1::urCommandBufferGetNativeHandleExp; return result; } @@ -164,15 +167,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetContextProcAddrTable( return result; } - pDdiTable->pfnCreate = ur::level_zero::urContextCreate; - pDdiTable->pfnRetain = ur::level_zero::urContextRetain; - pDdiTable->pfnRelease = ur::level_zero::urContextRelease; - pDdiTable->pfnGetInfo = ur::level_zero::urContextGetInfo; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urContextGetNativeHandle; + pDdiTable->pfnCreate = ur::level_zero::v1::urContextCreate; + pDdiTable->pfnRetain = ur::level_zero::v1::urContextRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urContextRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urContextGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urContextGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urContextCreateWithNativeHandle; + ur::level_zero::v1::urContextCreateWithNativeHandle; pDdiTable->pfnSetExtendedDeleter = - ur::level_zero::urContextSetExtendedDeleter; + ur::level_zero::v1::urContextSetExtendedDeleter; return result; } @@ -184,36 +187,38 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueProcAddrTable( return result; } - pDdiTable->pfnEventsWait = ur::level_zero::urEnqueueEventsWait; + pDdiTable->pfnEventsWait = ur::level_zero::v1::urEnqueueEventsWait; pDdiTable->pfnEventsWaitWithBarrier = - ur::level_zero::urEnqueueEventsWaitWithBarrier; - pDdiTable->pfnMemBufferRead = ur::level_zero::urEnqueueMemBufferRead; - pDdiTable->pfnMemBufferWrite = ur::level_zero::urEnqueueMemBufferWrite; - pDdiTable->pfnMemBufferReadRect = ur::level_zero::urEnqueueMemBufferReadRect; + ur::level_zero::v1::urEnqueueEventsWaitWithBarrier; + pDdiTable->pfnMemBufferRead = ur::level_zero::v1::urEnqueueMemBufferRead; + pDdiTable->pfnMemBufferWrite = ur::level_zero::v1::urEnqueueMemBufferWrite; + pDdiTable->pfnMemBufferReadRect = + ur::level_zero::v1::urEnqueueMemBufferReadRect; pDdiTable->pfnMemBufferWriteRect = - ur::level_zero::urEnqueueMemBufferWriteRect; - pDdiTable->pfnMemBufferCopy = ur::level_zero::urEnqueueMemBufferCopy; - pDdiTable->pfnMemBufferCopyRect = ur::level_zero::urEnqueueMemBufferCopyRect; - pDdiTable->pfnMemBufferFill = ur::level_zero::urEnqueueMemBufferFill; - pDdiTable->pfnMemImageRead = ur::level_zero::urEnqueueMemImageRead; - pDdiTable->pfnMemImageWrite = ur::level_zero::urEnqueueMemImageWrite; - pDdiTable->pfnMemImageCopy = ur::level_zero::urEnqueueMemImageCopy; - pDdiTable->pfnMemBufferMap = ur::level_zero::urEnqueueMemBufferMap; - pDdiTable->pfnMemUnmap = ur::level_zero::urEnqueueMemUnmap; - pDdiTable->pfnUSMFill = ur::level_zero::urEnqueueUSMFill; - pDdiTable->pfnUSMMemcpy = ur::level_zero::urEnqueueUSMMemcpy; - pDdiTable->pfnUSMPrefetch = ur::level_zero::urEnqueueUSMPrefetch; - pDdiTable->pfnUSMAdvise = ur::level_zero::urEnqueueUSMAdvise; - pDdiTable->pfnUSMFill2D = ur::level_zero::urEnqueueUSMFill2D; - pDdiTable->pfnUSMMemcpy2D = ur::level_zero::urEnqueueUSMMemcpy2D; + ur::level_zero::v1::urEnqueueMemBufferWriteRect; + pDdiTable->pfnMemBufferCopy = ur::level_zero::v1::urEnqueueMemBufferCopy; + pDdiTable->pfnMemBufferCopyRect = + ur::level_zero::v1::urEnqueueMemBufferCopyRect; + pDdiTable->pfnMemBufferFill = ur::level_zero::v1::urEnqueueMemBufferFill; + pDdiTable->pfnMemImageRead = ur::level_zero::v1::urEnqueueMemImageRead; + pDdiTable->pfnMemImageWrite = ur::level_zero::v1::urEnqueueMemImageWrite; + pDdiTable->pfnMemImageCopy = ur::level_zero::v1::urEnqueueMemImageCopy; + pDdiTable->pfnMemBufferMap = ur::level_zero::v1::urEnqueueMemBufferMap; + pDdiTable->pfnMemUnmap = ur::level_zero::v1::urEnqueueMemUnmap; + pDdiTable->pfnUSMFill = ur::level_zero::v1::urEnqueueUSMFill; + pDdiTable->pfnUSMMemcpy = ur::level_zero::v1::urEnqueueUSMMemcpy; + pDdiTable->pfnUSMPrefetch = ur::level_zero::v1::urEnqueueUSMPrefetch; + pDdiTable->pfnUSMAdvise = ur::level_zero::v1::urEnqueueUSMAdvise; + pDdiTable->pfnUSMFill2D = ur::level_zero::v1::urEnqueueUSMFill2D; + pDdiTable->pfnUSMMemcpy2D = ur::level_zero::v1::urEnqueueUSMMemcpy2D; pDdiTable->pfnDeviceGlobalVariableWrite = - ur::level_zero::urEnqueueDeviceGlobalVariableWrite; + ur::level_zero::v1::urEnqueueDeviceGlobalVariableWrite; pDdiTable->pfnDeviceGlobalVariableRead = - ur::level_zero::urEnqueueDeviceGlobalVariableRead; - pDdiTable->pfnReadHostPipe = ur::level_zero::urEnqueueReadHostPipe; - pDdiTable->pfnWriteHostPipe = ur::level_zero::urEnqueueWriteHostPipe; + ur::level_zero::v1::urEnqueueDeviceGlobalVariableRead; + pDdiTable->pfnReadHostPipe = ur::level_zero::v1::urEnqueueReadHostPipe; + pDdiTable->pfnWriteHostPipe = ur::level_zero::v1::urEnqueueWriteHostPipe; pDdiTable->pfnEventsWaitWithBarrierExt = - ur::level_zero::urEnqueueEventsWaitWithBarrierExt; + ur::level_zero::v1::urEnqueueEventsWaitWithBarrierExt; return result; } @@ -226,17 +231,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( } pDdiTable->pfnKernelLaunchWithArgsExp = - ur::level_zero::urEnqueueKernelLaunchWithArgsExp; - pDdiTable->pfnUSMDeviceAllocExp = ur::level_zero::urEnqueueUSMDeviceAllocExp; - pDdiTable->pfnUSMSharedAllocExp = ur::level_zero::urEnqueueUSMSharedAllocExp; - pDdiTable->pfnUSMHostAllocExp = ur::level_zero::urEnqueueUSMHostAllocExp; - pDdiTable->pfnUSMFreeExp = ur::level_zero::urEnqueueUSMFreeExp; + ur::level_zero::v1::urEnqueueKernelLaunchWithArgsExp; + pDdiTable->pfnUSMDeviceAllocExp = + ur::level_zero::v1::urEnqueueUSMDeviceAllocExp; + pDdiTable->pfnUSMSharedAllocExp = + ur::level_zero::v1::urEnqueueUSMSharedAllocExp; + pDdiTable->pfnUSMHostAllocExp = ur::level_zero::v1::urEnqueueUSMHostAllocExp; + pDdiTable->pfnUSMFreeExp = ur::level_zero::v1::urEnqueueUSMFreeExp; pDdiTable->pfnTimestampRecordingExp = - ur::level_zero::urEnqueueTimestampRecordingExp; - pDdiTable->pfnCommandBufferExp = ur::level_zero::urEnqueueCommandBufferExp; - pDdiTable->pfnHostTaskExp = ur::level_zero::urEnqueueHostTaskExp; - pDdiTable->pfnNativeCommandExp = ur::level_zero::urEnqueueNativeCommandExp; - pDdiTable->pfnGraphExp = ur::level_zero::urEnqueueGraphExp; + ur::level_zero::v1::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = + ur::level_zero::v1::urEnqueueCommandBufferExp; + pDdiTable->pfnHostTaskExp = ur::level_zero::v1::urEnqueueHostTaskExp; + pDdiTable->pfnNativeCommandExp = + ur::level_zero::v1::urEnqueueNativeCommandExp; + pDdiTable->pfnGraphExp = ur::level_zero::v1::urEnqueueGraphExp; return result; } @@ -248,15 +257,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetEventProcAddrTable( return result; } - pDdiTable->pfnGetInfo = ur::level_zero::urEventGetInfo; - pDdiTable->pfnGetProfilingInfo = ur::level_zero::urEventGetProfilingInfo; - pDdiTable->pfnWait = ur::level_zero::urEventWait; - pDdiTable->pfnRetain = ur::level_zero::urEventRetain; - pDdiTable->pfnRelease = ur::level_zero::urEventRelease; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urEventGetNativeHandle; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urEventGetInfo; + pDdiTable->pfnGetProfilingInfo = ur::level_zero::v1::urEventGetProfilingInfo; + pDdiTable->pfnWait = ur::level_zero::v1::urEventWait; + pDdiTable->pfnRetain = ur::level_zero::v1::urEventRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urEventRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urEventGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urEventCreateWithNativeHandle; - pDdiTable->pfnSetCallback = ur::level_zero::urEventSetCallback; + ur::level_zero::v1::urEventCreateWithNativeHandle; + pDdiTable->pfnSetCallback = ur::level_zero::v1::urEventSetCallback; return result; } @@ -268,7 +277,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetEventExpProcAddrTable( return result; } - pDdiTable->pfnCreateExp = ur::level_zero::urEventCreateExp; + pDdiTable->pfnCreateExp = ur::level_zero::v1::urEventCreateExp; return result; } @@ -280,20 +289,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetGraphExpProcAddrTable( return result; } - pDdiTable->pfnCreateExp = ur::level_zero::urGraphCreateExp; + pDdiTable->pfnCreateExp = ur::level_zero::v1::urGraphCreateExp; pDdiTable->pfnInstantiateGraphExp = - ur::level_zero::urGraphInstantiateGraphExp; - pDdiTable->pfnDestroyExp = ur::level_zero::urGraphDestroyExp; + ur::level_zero::v1::urGraphInstantiateGraphExp; + pDdiTable->pfnDestroyExp = ur::level_zero::v1::urGraphDestroyExp; pDdiTable->pfnExecutableGraphDestroyExp = - ur::level_zero::urGraphExecutableGraphDestroyExp; - pDdiTable->pfnIsEmptyExp = ur::level_zero::urGraphIsEmptyExp; - pDdiTable->pfnGetIdExp = ur::level_zero::urGraphGetIdExp; + ur::level_zero::v1::urGraphExecutableGraphDestroyExp; + pDdiTable->pfnIsEmptyExp = ur::level_zero::v1::urGraphIsEmptyExp; + pDdiTable->pfnGetIdExp = ur::level_zero::v1::urGraphGetIdExp; pDdiTable->pfnSetDestructionCallbackExp = - ur::level_zero::urGraphSetDestructionCallbackExp; - pDdiTable->pfnDumpContentsExp = ur::level_zero::urGraphDumpContentsExp; - pDdiTable->pfnGetNativeHandleExp = ur::level_zero::urGraphGetNativeHandleExp; + ur::level_zero::v1::urGraphSetDestructionCallbackExp; + pDdiTable->pfnDumpContentsExp = ur::level_zero::v1::urGraphDumpContentsExp; + pDdiTable->pfnGetNativeHandleExp = + ur::level_zero::v1::urGraphGetNativeHandleExp; pDdiTable->pfnExecutableGraphGetNativeHandleExp = - ur::level_zero::urGraphExecutableGraphGetNativeHandleExp; + ur::level_zero::v1::urGraphExecutableGraphGetNativeHandleExp; return result; } @@ -305,19 +315,22 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( return result; } - pDdiTable->pfnGetMemHandleExp = ur::level_zero::urIPCGetMemHandleExp; - pDdiTable->pfnPutMemHandleExp = ur::level_zero::urIPCPutMemHandleExp; - pDdiTable->pfnOpenMemHandleExp = ur::level_zero::urIPCOpenMemHandleExp; - pDdiTable->pfnCloseMemHandleExp = ur::level_zero::urIPCCloseMemHandleExp; - pDdiTable->pfnGetPhysMemHandleExp = ur::level_zero::urIPCGetPhysMemHandleExp; - pDdiTable->pfnPutPhysMemHandleExp = ur::level_zero::urIPCPutPhysMemHandleExp; + pDdiTable->pfnGetMemHandleExp = ur::level_zero::v1::urIPCGetMemHandleExp; + pDdiTable->pfnPutMemHandleExp = ur::level_zero::v1::urIPCPutMemHandleExp; + pDdiTable->pfnOpenMemHandleExp = ur::level_zero::v1::urIPCOpenMemHandleExp; + pDdiTable->pfnCloseMemHandleExp = ur::level_zero::v1::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur::level_zero::v1::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur::level_zero::v1::urIPCPutPhysMemHandleExp; pDdiTable->pfnOpenPhysMemHandleExp = - ur::level_zero::urIPCOpenPhysMemHandleExp; + ur::level_zero::v1::urIPCOpenPhysMemHandleExp; pDdiTable->pfnClosePhysMemHandleExp = - ur::level_zero::urIPCClosePhysMemHandleExp; - pDdiTable->pfnGetEventHandleExp = ur::level_zero::urIPCGetEventHandleExp; - pDdiTable->pfnPutEventHandleExp = ur::level_zero::urIPCPutEventHandleExp; - pDdiTable->pfnOpenEventHandleExp = ur::level_zero::urIPCOpenEventHandleExp; + ur::level_zero::v1::urIPCClosePhysMemHandleExp; + pDdiTable->pfnGetEventHandleExp = ur::level_zero::v1::urIPCGetEventHandleExp; + pDdiTable->pfnPutEventHandleExp = ur::level_zero::v1::urIPCPutEventHandleExp; + pDdiTable->pfnOpenEventHandleExp = + ur::level_zero::v1::urIPCOpenEventHandleExp; return result; } @@ -329,24 +342,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetKernelProcAddrTable( return result; } - pDdiTable->pfnCreate = ur::level_zero::urKernelCreate; - pDdiTable->pfnGetInfo = ur::level_zero::urKernelGetInfo; - pDdiTable->pfnGetGroupInfo = ur::level_zero::urKernelGetGroupInfo; - pDdiTable->pfnGetSubGroupInfo = ur::level_zero::urKernelGetSubGroupInfo; - pDdiTable->pfnRetain = ur::level_zero::urKernelRetain; - pDdiTable->pfnRelease = ur::level_zero::urKernelRelease; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urKernelGetNativeHandle; + pDdiTable->pfnCreate = ur::level_zero::v1::urKernelCreate; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urKernelGetInfo; + pDdiTable->pfnGetGroupInfo = ur::level_zero::v1::urKernelGetGroupInfo; + pDdiTable->pfnGetSubGroupInfo = ur::level_zero::v1::urKernelGetSubGroupInfo; + pDdiTable->pfnRetain = ur::level_zero::v1::urKernelRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urKernelRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urKernelGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urKernelCreateWithNativeHandle; + ur::level_zero::v1::urKernelCreateWithNativeHandle; pDdiTable->pfnGetSuggestedLocalWorkSize = - ur::level_zero::urKernelGetSuggestedLocalWorkSize; + ur::level_zero::v1::urKernelGetSuggestedLocalWorkSize; pDdiTable->pfnGetSuggestedLocalWorkSizeWithArgs = - ur::level_zero::urKernelGetSuggestedLocalWorkSizeWithArgs; - pDdiTable->pfnSetExecInfo = ur::level_zero::urKernelSetExecInfo; + ur::level_zero::v1::urKernelGetSuggestedLocalWorkSizeWithArgs; + pDdiTable->pfnSetExecInfo = ur::level_zero::v1::urKernelSetExecInfo; pDdiTable->pfnSetSpecializationConstants = - ur::level_zero::urKernelSetSpecializationConstants; + ur::level_zero::v1::urKernelSetSpecializationConstants; pDdiTable->pfnSuggestMaxCooperativeGroupCount = - ur::level_zero::urKernelSuggestMaxCooperativeGroupCount; + ur::level_zero::v1::urKernelSuggestMaxCooperativeGroupCount; return result; } @@ -358,18 +371,18 @@ urGetMemProcAddrTable(ur_api_version_t version, ur_mem_dditable_t *pDdiTable) { return result; } - pDdiTable->pfnImageCreate = ur::level_zero::urMemImageCreate; - pDdiTable->pfnBufferCreate = ur::level_zero::urMemBufferCreate; - pDdiTable->pfnRetain = ur::level_zero::urMemRetain; - pDdiTable->pfnRelease = ur::level_zero::urMemRelease; - pDdiTable->pfnBufferPartition = ur::level_zero::urMemBufferPartition; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urMemGetNativeHandle; + pDdiTable->pfnImageCreate = ur::level_zero::v1::urMemImageCreate; + pDdiTable->pfnBufferCreate = ur::level_zero::v1::urMemBufferCreate; + pDdiTable->pfnRetain = ur::level_zero::v1::urMemRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urMemRelease; + pDdiTable->pfnBufferPartition = ur::level_zero::v1::urMemBufferPartition; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urMemGetNativeHandle; pDdiTable->pfnBufferCreateWithNativeHandle = - ur::level_zero::urMemBufferCreateWithNativeHandle; + ur::level_zero::v1::urMemBufferCreateWithNativeHandle; pDdiTable->pfnImageCreateWithNativeHandle = - ur::level_zero::urMemImageCreateWithNativeHandle; - pDdiTable->pfnGetInfo = ur::level_zero::urMemGetInfo; - pDdiTable->pfnImageGetInfo = ur::level_zero::urMemImageGetInfo; + ur::level_zero::v1::urMemImageCreateWithNativeHandle; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urMemGetInfo; + pDdiTable->pfnImageGetInfo = ur::level_zero::v1::urMemImageGetInfo; return result; } @@ -382,11 +395,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( } pDdiTable->pfnAllocExportableMemoryExp = - ur::level_zero::urMemoryExportAllocExportableMemoryExp; + ur::level_zero::v1::urMemoryExportAllocExportableMemoryExp; pDdiTable->pfnFreeExportableMemoryExp = - ur::level_zero::urMemoryExportFreeExportableMemoryExp; + ur::level_zero::v1::urMemoryExportFreeExportableMemoryExp; pDdiTable->pfnExportMemoryHandleExp = - ur::level_zero::urMemoryExportExportMemoryHandleExp; + ur::level_zero::v1::urMemoryExportExportMemoryHandleExp; return result; } @@ -398,10 +411,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( return result; } - pDdiTable->pfnCreate = ur::level_zero::urPhysicalMemCreate; - pDdiTable->pfnRetain = ur::level_zero::urPhysicalMemRetain; - pDdiTable->pfnRelease = ur::level_zero::urPhysicalMemRelease; - pDdiTable->pfnGetInfo = ur::level_zero::urPhysicalMemGetInfo; + pDdiTable->pfnCreate = ur::level_zero::v1::urPhysicalMemCreate; + pDdiTable->pfnRetain = ur::level_zero::v1::urPhysicalMemRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urPhysicalMemRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urPhysicalMemGetInfo; return result; } @@ -413,13 +426,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetPlatformProcAddrTable( return result; } - pDdiTable->pfnGet = ur::level_zero::urPlatformGet; - pDdiTable->pfnGetInfo = ur::level_zero::urPlatformGetInfo; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urPlatformGetNativeHandle; + pDdiTable->pfnGet = ur::level_zero::v1::urPlatformGet; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urPlatformGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urPlatformGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urPlatformCreateWithNativeHandle; - pDdiTable->pfnGetApiVersion = ur::level_zero::urPlatformGetApiVersion; - pDdiTable->pfnGetBackendOption = ur::level_zero::urPlatformGetBackendOption; + ur::level_zero::v1::urPlatformCreateWithNativeHandle; + pDdiTable->pfnGetApiVersion = ur::level_zero::v1::urPlatformGetApiVersion; + pDdiTable->pfnGetBackendOption = + ur::level_zero::v1::urPlatformGetBackendOption; return result; } @@ -431,24 +445,25 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetProgramProcAddrTable( return result; } - pDdiTable->pfnCreateWithIL = ur::level_zero::urProgramCreateWithIL; - pDdiTable->pfnCreateWithBinary = ur::level_zero::urProgramCreateWithBinary; - pDdiTable->pfnBuild = ur::level_zero::urProgramBuild; - pDdiTable->pfnCompile = ur::level_zero::urProgramCompile; - pDdiTable->pfnLink = ur::level_zero::urProgramLink; - pDdiTable->pfnRetain = ur::level_zero::urProgramRetain; - pDdiTable->pfnRelease = ur::level_zero::urProgramRelease; + pDdiTable->pfnCreateWithIL = ur::level_zero::v1::urProgramCreateWithIL; + pDdiTable->pfnCreateWithBinary = + ur::level_zero::v1::urProgramCreateWithBinary; + pDdiTable->pfnBuild = ur::level_zero::v1::urProgramBuild; + pDdiTable->pfnCompile = ur::level_zero::v1::urProgramCompile; + pDdiTable->pfnLink = ur::level_zero::v1::urProgramLink; + pDdiTable->pfnRetain = ur::level_zero::v1::urProgramRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urProgramRelease; pDdiTable->pfnGetFunctionPointer = - ur::level_zero::urProgramGetFunctionPointer; + ur::level_zero::v1::urProgramGetFunctionPointer; pDdiTable->pfnGetGlobalVariablePointer = - ur::level_zero::urProgramGetGlobalVariablePointer; - pDdiTable->pfnGetInfo = ur::level_zero::urProgramGetInfo; - pDdiTable->pfnGetBuildInfo = ur::level_zero::urProgramGetBuildInfo; + ur::level_zero::v1::urProgramGetGlobalVariablePointer; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urProgramGetInfo; + pDdiTable->pfnGetBuildInfo = ur::level_zero::v1::urProgramGetBuildInfo; pDdiTable->pfnSetSpecializationConstants = - ur::level_zero::urProgramSetSpecializationConstants; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urProgramGetNativeHandle; + ur::level_zero::v1::urProgramSetSpecializationConstants; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urProgramGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urProgramCreateWithNativeHandle; + ur::level_zero::v1::urProgramCreateWithNativeHandle; return result; } @@ -460,10 +475,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetProgramExpProcAddrTable( return result; } - pDdiTable->pfnDynamicLinkExp = ur::level_zero::urProgramDynamicLinkExp; - pDdiTable->pfnBuildExp = ur::level_zero::urProgramBuildExp; - pDdiTable->pfnCompileExp = ur::level_zero::urProgramCompileExp; - pDdiTable->pfnLinkExp = ur::level_zero::urProgramLinkExp; + pDdiTable->pfnDynamicLinkExp = ur::level_zero::v1::urProgramDynamicLinkExp; + pDdiTable->pfnBuildExp = ur::level_zero::v1::urProgramBuildExp; + pDdiTable->pfnCompileExp = ur::level_zero::v1::urProgramCompileExp; + pDdiTable->pfnLinkExp = ur::level_zero::v1::urProgramLinkExp; return result; } @@ -475,15 +490,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetQueueProcAddrTable( return result; } - pDdiTable->pfnGetInfo = ur::level_zero::urQueueGetInfo; - pDdiTable->pfnCreate = ur::level_zero::urQueueCreate; - pDdiTable->pfnRetain = ur::level_zero::urQueueRetain; - pDdiTable->pfnRelease = ur::level_zero::urQueueRelease; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urQueueGetNativeHandle; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urQueueGetInfo; + pDdiTable->pfnCreate = ur::level_zero::v1::urQueueCreate; + pDdiTable->pfnRetain = ur::level_zero::v1::urQueueRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urQueueRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urQueueGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urQueueCreateWithNativeHandle; - pDdiTable->pfnFinish = ur::level_zero::urQueueFinish; - pDdiTable->pfnFlush = ur::level_zero::urQueueFlush; + ur::level_zero::v1::urQueueCreateWithNativeHandle; + pDdiTable->pfnFinish = ur::level_zero::v1::urQueueFinish; + pDdiTable->pfnFlush = ur::level_zero::v1::urQueueFlush; return result; } @@ -496,13 +511,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetQueueExpProcAddrTable( } pDdiTable->pfnBeginGraphCaptureExp = - ur::level_zero::urQueueBeginGraphCaptureExp; + ur::level_zero::v1::urQueueBeginGraphCaptureExp; pDdiTable->pfnBeginCaptureIntoGraphExp = - ur::level_zero::urQueueBeginCaptureIntoGraphExp; - pDdiTable->pfnEndGraphCaptureExp = ur::level_zero::urQueueEndGraphCaptureExp; + ur::level_zero::v1::urQueueBeginCaptureIntoGraphExp; + pDdiTable->pfnEndGraphCaptureExp = + ur::level_zero::v1::urQueueEndGraphCaptureExp; pDdiTable->pfnIsGraphCaptureEnabledExp = - ur::level_zero::urQueueIsGraphCaptureEnabledExp; - pDdiTable->pfnGetGraphExp = ur::level_zero::urQueueGetGraphExp; + ur::level_zero::v1::urQueueIsGraphCaptureEnabledExp; + pDdiTable->pfnGetGraphExp = ur::level_zero::v1::urQueueGetGraphExp; return result; } @@ -514,13 +530,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetSamplerProcAddrTable( return result; } - pDdiTable->pfnCreate = ur::level_zero::urSamplerCreate; - pDdiTable->pfnRetain = ur::level_zero::urSamplerRetain; - pDdiTable->pfnRelease = ur::level_zero::urSamplerRelease; - pDdiTable->pfnGetInfo = ur::level_zero::urSamplerGetInfo; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urSamplerGetNativeHandle; + pDdiTable->pfnCreate = ur::level_zero::v1::urSamplerCreate; + pDdiTable->pfnRetain = ur::level_zero::v1::urSamplerRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urSamplerRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urSamplerGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urSamplerGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urSamplerCreateWithNativeHandle; + ur::level_zero::v1::urSamplerCreateWithNativeHandle; return result; } @@ -532,15 +548,15 @@ urGetUSMProcAddrTable(ur_api_version_t version, ur_usm_dditable_t *pDdiTable) { return result; } - pDdiTable->pfnHostAlloc = ur::level_zero::urUSMHostAlloc; - pDdiTable->pfnDeviceAlloc = ur::level_zero::urUSMDeviceAlloc; - pDdiTable->pfnSharedAlloc = ur::level_zero::urUSMSharedAlloc; - pDdiTable->pfnFree = ur::level_zero::urUSMFree; - pDdiTable->pfnGetMemAllocInfo = ur::level_zero::urUSMGetMemAllocInfo; - pDdiTable->pfnPoolCreate = ur::level_zero::urUSMPoolCreate; - pDdiTable->pfnPoolRetain = ur::level_zero::urUSMPoolRetain; - pDdiTable->pfnPoolRelease = ur::level_zero::urUSMPoolRelease; - pDdiTable->pfnPoolGetInfo = ur::level_zero::urUSMPoolGetInfo; + pDdiTable->pfnHostAlloc = ur::level_zero::v1::urUSMHostAlloc; + pDdiTable->pfnDeviceAlloc = ur::level_zero::v1::urUSMDeviceAlloc; + pDdiTable->pfnSharedAlloc = ur::level_zero::v1::urUSMSharedAlloc; + pDdiTable->pfnFree = ur::level_zero::v1::urUSMFree; + pDdiTable->pfnGetMemAllocInfo = ur::level_zero::v1::urUSMGetMemAllocInfo; + pDdiTable->pfnPoolCreate = ur::level_zero::v1::urUSMPoolCreate; + pDdiTable->pfnPoolRetain = ur::level_zero::v1::urUSMPoolRetain; + pDdiTable->pfnPoolRelease = ur::level_zero::v1::urUSMPoolRelease; + pDdiTable->pfnPoolGetInfo = ur::level_zero::v1::urUSMPoolGetInfo; return result; } @@ -552,25 +568,25 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( return result; } - pDdiTable->pfnPoolCreateExp = ur::level_zero::urUSMPoolCreateExp; - pDdiTable->pfnPoolDestroyExp = ur::level_zero::urUSMPoolDestroyExp; + pDdiTable->pfnPoolCreateExp = ur::level_zero::v1::urUSMPoolCreateExp; + pDdiTable->pfnPoolDestroyExp = ur::level_zero::v1::urUSMPoolDestroyExp; pDdiTable->pfnPoolGetDefaultDevicePoolExp = - ur::level_zero::urUSMPoolGetDefaultDevicePoolExp; - pDdiTable->pfnPoolGetInfoExp = ur::level_zero::urUSMPoolGetInfoExp; - pDdiTable->pfnPoolSetInfoExp = ur::level_zero::urUSMPoolSetInfoExp; + ur::level_zero::v1::urUSMPoolGetDefaultDevicePoolExp; + pDdiTable->pfnPoolGetInfoExp = ur::level_zero::v1::urUSMPoolGetInfoExp; + pDdiTable->pfnPoolSetInfoExp = ur::level_zero::v1::urUSMPoolSetInfoExp; pDdiTable->pfnPoolSetDevicePoolExp = - ur::level_zero::urUSMPoolSetDevicePoolExp; + ur::level_zero::v1::urUSMPoolSetDevicePoolExp; pDdiTable->pfnPoolGetDevicePoolExp = - ur::level_zero::urUSMPoolGetDevicePoolExp; - pDdiTable->pfnPoolTrimToExp = ur::level_zero::urUSMPoolTrimToExp; - pDdiTable->pfnPitchedAllocExp = ur::level_zero::urUSMPitchedAllocExp; - pDdiTable->pfnContextMemcpyExp = ur::level_zero::urUSMContextMemcpyExp; + ur::level_zero::v1::urUSMPoolGetDevicePoolExp; + pDdiTable->pfnPoolTrimToExp = ur::level_zero::v1::urUSMPoolTrimToExp; + pDdiTable->pfnPitchedAllocExp = ur::level_zero::v1::urUSMPitchedAllocExp; + pDdiTable->pfnContextMemcpyExp = ur::level_zero::v1::urUSMContextMemcpyExp; pDdiTable->pfnHostAllocUnregisterExp = - ur::level_zero::urUSMHostAllocUnregisterExp; + ur::level_zero::v1::urUSMHostAllocUnregisterExp; pDdiTable->pfnHostAllocRegisterExp = - ur::level_zero::urUSMHostAllocRegisterExp; - pDdiTable->pfnImportExp = ur::level_zero::urUSMImportExp; - pDdiTable->pfnReleaseExp = ur::level_zero::urUSMReleaseExp; + ur::level_zero::v1::urUSMHostAllocRegisterExp; + pDdiTable->pfnImportExp = ur::level_zero::v1::urUSMImportExp; + pDdiTable->pfnReleaseExp = ur::level_zero::v1::urUSMReleaseExp; return result; } @@ -583,11 +599,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetUsmP2PExpProcAddrTable( } pDdiTable->pfnEnablePeerAccessExp = - ur::level_zero::urUsmP2PEnablePeerAccessExp; + ur::level_zero::v1::urUsmP2PEnablePeerAccessExp; pDdiTable->pfnDisablePeerAccessExp = - ur::level_zero::urUsmP2PDisablePeerAccessExp; + ur::level_zero::v1::urUsmP2PDisablePeerAccessExp; pDdiTable->pfnPeerAccessGetInfoExp = - ur::level_zero::urUsmP2PPeerAccessGetInfoExp; + ur::level_zero::v1::urUsmP2PPeerAccessGetInfoExp; return result; } @@ -600,13 +616,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( } pDdiTable->pfnGranularityGetInfo = - ur::level_zero::urVirtualMemGranularityGetInfo; - pDdiTable->pfnReserve = ur::level_zero::urVirtualMemReserve; - pDdiTable->pfnFree = ur::level_zero::urVirtualMemFree; - pDdiTable->pfnMap = ur::level_zero::urVirtualMemMap; - pDdiTable->pfnUnmap = ur::level_zero::urVirtualMemUnmap; - pDdiTable->pfnSetAccess = ur::level_zero::urVirtualMemSetAccess; - pDdiTable->pfnGetInfo = ur::level_zero::urVirtualMemGetInfo; + ur::level_zero::v1::urVirtualMemGranularityGetInfo; + pDdiTable->pfnReserve = ur::level_zero::v1::urVirtualMemReserve; + pDdiTable->pfnFree = ur::level_zero::v1::urVirtualMemFree; + pDdiTable->pfnMap = ur::level_zero::v1::urVirtualMemMap; + pDdiTable->pfnUnmap = ur::level_zero::v1::urVirtualMemUnmap; + pDdiTable->pfnSetAccess = ur::level_zero::v1::urVirtualMemSetAccess; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urVirtualMemGetInfo; return result; } @@ -618,17 +634,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceProcAddrTable( return result; } - pDdiTable->pfnGet = ur::level_zero::urDeviceGet; - pDdiTable->pfnGetInfo = ur::level_zero::urDeviceGetInfo; - pDdiTable->pfnRetain = ur::level_zero::urDeviceRetain; - pDdiTable->pfnRelease = ur::level_zero::urDeviceRelease; - pDdiTable->pfnPartition = ur::level_zero::urDevicePartition; - pDdiTable->pfnSelectBinary = ur::level_zero::urDeviceSelectBinary; - pDdiTable->pfnGetNativeHandle = ur::level_zero::urDeviceGetNativeHandle; + pDdiTable->pfnGet = ur::level_zero::v1::urDeviceGet; + pDdiTable->pfnGetInfo = ur::level_zero::v1::urDeviceGetInfo; + pDdiTable->pfnRetain = ur::level_zero::v1::urDeviceRetain; + pDdiTable->pfnRelease = ur::level_zero::v1::urDeviceRelease; + pDdiTable->pfnPartition = ur::level_zero::v1::urDevicePartition; + pDdiTable->pfnSelectBinary = ur::level_zero::v1::urDeviceSelectBinary; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v1::urDeviceGetNativeHandle; pDdiTable->pfnCreateWithNativeHandle = - ur::level_zero::urDeviceCreateWithNativeHandle; + ur::level_zero::v1::urDeviceCreateWithNativeHandle; pDdiTable->pfnGetGlobalTimestamps = - ur::level_zero::urDeviceGetGlobalTimestamps; + ur::level_zero::v1::urDeviceGetGlobalTimestamps; return result; } @@ -640,13 +656,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceExpProcAddrTable( return result; } - pDdiTable->pfnWaitExp = ur::level_zero::urDeviceWaitExp; + pDdiTable->pfnWaitExp = ur::level_zero::v1::urDeviceWaitExp; return result; } #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 #else } // extern "C" #endif @@ -660,7 +676,7 @@ ur_result_t populateDdiTable(ur_dditable_t *ddi) { ur_result_t result; #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO -#define NAMESPACE_ ::ur::level_zero +#define NAMESPACE_ ::ur::level_zero::v1 #else #define NAMESPACE_ #endif @@ -774,7 +790,7 @@ ur_result_t populateDdiTable(ur_dditable_t *ddi) { } } // namespace -namespace ur::level_zero { +namespace ur::level_zero::v1 { const ur_dditable_t *ddi_getter::value() { static std::once_flag flag; static ur_dditable_t table; @@ -788,4 +804,4 @@ ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) { return populateDdiTable(ddi); } #endif -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index 72d0d0b2c40e4..234847534e25e 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -12,896 +12,929 @@ #include #include -namespace ur::level_zero { -ur_result_t urAdapterGet(uint32_t NumEntries, ur_adapter_handle_t *phAdapters, +namespace ur::level_zero::v1 { +ur_result_t urAdapterGet(uint32_t NumEntries, ::ur_adapter_handle_t *phAdapters, uint32_t *pNumAdapters); -ur_result_t urAdapterRelease(ur_adapter_handle_t hAdapter); -ur_result_t urAdapterRetain(ur_adapter_handle_t hAdapter); -ur_result_t urAdapterGetLastError(ur_adapter_handle_t hAdapter, +ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapter); +ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapter); +ur_result_t urAdapterGetLastError(::ur_adapter_handle_t hAdapter, const char **ppMessage, int32_t *pError); -ur_result_t urAdapterGetInfo(ur_adapter_handle_t hAdapter, +ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapter, ur_adapter_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urAdapterSetLoggerCallback(ur_adapter_handle_t hAdapter, +ur_result_t urAdapterSetLoggerCallback(::ur_adapter_handle_t hAdapter, ur_logger_callback_t pfnLoggerCallback, void *pUserData, ur_logger_level_t level); -ur_result_t urAdapterSetLoggerCallbackLevel(ur_adapter_handle_t hAdapter, +ur_result_t urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapter, ur_logger_level_t level); -ur_result_t urPlatformGet(ur_adapter_handle_t hAdapter, uint32_t NumEntries, - ur_platform_handle_t *phPlatforms, +ur_result_t urPlatformGet(::ur_adapter_handle_t hAdapter, uint32_t NumEntries, + ::ur_platform_handle_t *phPlatforms, uint32_t *pNumPlatforms); -ur_result_t urPlatformGetInfo(ur_platform_handle_t hPlatform, +ur_result_t urPlatformGetInfo(::ur_platform_handle_t hPlatform, ur_platform_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urPlatformGetApiVersion(ur_platform_handle_t hPlatform, +ur_result_t urPlatformGetApiVersion(::ur_platform_handle_t hPlatform, ur_api_version_t *pVersion); -ur_result_t urPlatformGetNativeHandle(ur_platform_handle_t hPlatform, - ur_native_handle_t *phNativePlatform); +ur_result_t urPlatformGetNativeHandle(::ur_platform_handle_t hPlatform, + ::ur_native_handle_t *phNativePlatform); ur_result_t urPlatformCreateWithNativeHandle( - ur_native_handle_t hNativePlatform, ur_adapter_handle_t hAdapter, + ::ur_native_handle_t hNativePlatform, ::ur_adapter_handle_t hAdapter, const ur_platform_native_properties_t *pProperties, - ur_platform_handle_t *phPlatform); -ur_result_t urPlatformGetBackendOption(ur_platform_handle_t hPlatform, + ::ur_platform_handle_t *phPlatform); +ur_result_t urPlatformGetBackendOption(::ur_platform_handle_t hPlatform, const char *pFrontendOption, const char **ppPlatformOption); -ur_result_t urDeviceGet(ur_platform_handle_t hPlatform, +ur_result_t urDeviceGet(::ur_platform_handle_t hPlatform, ur_device_type_t DeviceType, uint32_t NumEntries, - ur_device_handle_t *phDevices, uint32_t *pNumDevices); -ur_result_t urDeviceGetInfo(ur_device_handle_t hDevice, + ::ur_device_handle_t *phDevices, uint32_t *pNumDevices); +ur_result_t urDeviceGetInfo(::ur_device_handle_t hDevice, ur_device_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urDeviceRetain(ur_device_handle_t hDevice); -ur_result_t urDeviceRelease(ur_device_handle_t hDevice); +ur_result_t urDeviceRetain(::ur_device_handle_t hDevice); +ur_result_t urDeviceRelease(::ur_device_handle_t hDevice); ur_result_t -urDevicePartition(ur_device_handle_t hDevice, +urDevicePartition(::ur_device_handle_t hDevice, const ur_device_partition_properties_t *pProperties, - uint32_t NumDevices, ur_device_handle_t *phSubDevices, + uint32_t NumDevices, ::ur_device_handle_t *phSubDevices, uint32_t *pNumDevicesRet); -ur_result_t urDeviceSelectBinary(ur_device_handle_t hDevice, +ur_result_t urDeviceSelectBinary(::ur_device_handle_t hDevice, const ur_device_binary_t *pBinaries, uint32_t NumBinaries, uint32_t *pSelectedBinary); -ur_result_t urDeviceGetNativeHandle(ur_device_handle_t hDevice, - ur_native_handle_t *phNativeDevice); +ur_result_t urDeviceGetNativeHandle(::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeDevice); ur_result_t -urDeviceCreateWithNativeHandle(ur_native_handle_t hNativeDevice, - ur_adapter_handle_t hAdapter, +urDeviceCreateWithNativeHandle(::ur_native_handle_t hNativeDevice, + ::ur_adapter_handle_t hAdapter, const ur_device_native_properties_t *pProperties, - ur_device_handle_t *phDevice); -ur_result_t urDeviceGetGlobalTimestamps(ur_device_handle_t hDevice, + ::ur_device_handle_t *phDevice); +ur_result_t urDeviceGetGlobalTimestamps(::ur_device_handle_t hDevice, uint64_t *pDeviceTimestamp, uint64_t *pHostTimestamp); ur_result_t urContextCreate(uint32_t DeviceCount, - const ur_device_handle_t *phDevices, + const ::ur_device_handle_t *phDevices, const ur_context_properties_t *pProperties, - ur_context_handle_t *phContext); -ur_result_t urContextRetain(ur_context_handle_t hContext); -ur_result_t urContextRelease(ur_context_handle_t hContext); -ur_result_t urContextGetInfo(ur_context_handle_t hContext, + ::ur_context_handle_t *phContext); +ur_result_t urContextRetain(::ur_context_handle_t hContext); +ur_result_t urContextRelease(::ur_context_handle_t hContext); +ur_result_t urContextGetInfo(::ur_context_handle_t hContext, ur_context_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urContextGetNativeHandle(ur_context_handle_t hContext, - ur_native_handle_t *phNativeContext); +ur_result_t urContextGetNativeHandle(::ur_context_handle_t hContext, + ::ur_native_handle_t *phNativeContext); ur_result_t urContextCreateWithNativeHandle( - ur_native_handle_t hNativeContext, ur_adapter_handle_t hAdapter, - uint32_t numDevices, const ur_device_handle_t *phDevices, + ::ur_native_handle_t hNativeContext, ::ur_adapter_handle_t hAdapter, + uint32_t numDevices, const ::ur_device_handle_t *phDevices, const ur_context_native_properties_t *pProperties, - ur_context_handle_t *phContext); + ::ur_context_handle_t *phContext); ur_result_t -urContextSetExtendedDeleter(ur_context_handle_t hContext, +urContextSetExtendedDeleter(::ur_context_handle_t hContext, ur_context_extended_deleter_t pfnDeleter, void *pUserData); -ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, +ur_result_t urMemImageCreate(::ur_context_handle_t hContext, + ur_mem_flags_t flags, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, void *pHost, - ur_mem_handle_t *phMem); -ur_result_t urMemBufferCreate(ur_context_handle_t hContext, + ::ur_mem_handle_t *phMem); +ur_result_t urMemBufferCreate(::ur_context_handle_t hContext, ur_mem_flags_t flags, size_t size, const ur_buffer_properties_t *pProperties, - ur_mem_handle_t *phBuffer); -ur_result_t urMemRetain(ur_mem_handle_t hMem); -ur_result_t urMemRelease(ur_mem_handle_t hMem); -ur_result_t urMemBufferPartition(ur_mem_handle_t hBuffer, ur_mem_flags_t flags, + ::ur_mem_handle_t *phBuffer); +ur_result_t urMemRetain(::ur_mem_handle_t hMem); +ur_result_t urMemRelease(::ur_mem_handle_t hMem); +ur_result_t urMemBufferPartition(::ur_mem_handle_t hBuffer, + ur_mem_flags_t flags, ur_buffer_create_type_t bufferCreateType, const ur_buffer_region_t *pRegion, - ur_mem_handle_t *phMem); -ur_result_t urMemGetNativeHandle(ur_mem_handle_t hMem, - ur_device_handle_t hDevice, - ur_native_handle_t *phNativeMem); + ::ur_mem_handle_t *phMem); +ur_result_t urMemGetNativeHandle(::ur_mem_handle_t hMem, + ::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeMem); ur_result_t urMemBufferCreateWithNativeHandle( - ur_native_handle_t hNativeMem, ur_context_handle_t hContext, - const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem); + ::ur_native_handle_t hNativeMem, ::ur_context_handle_t hContext, + const ur_mem_native_properties_t *pProperties, ::ur_mem_handle_t *phMem); ur_result_t urMemImageCreateWithNativeHandle( - ur_native_handle_t hNativeMem, ur_context_handle_t hContext, + ::ur_native_handle_t hNativeMem, ::ur_context_handle_t hContext, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem); -ur_result_t urMemGetInfo(ur_mem_handle_t hMemory, ur_mem_info_t propName, + const ur_mem_native_properties_t *pProperties, ::ur_mem_handle_t *phMem); +ur_result_t urMemGetInfo(::ur_mem_handle_t hMemory, ur_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urMemImageGetInfo(ur_mem_handle_t hMemory, ur_image_info_t propName, - size_t propSize, void *pPropValue, - size_t *pPropSizeRet); -ur_result_t urSamplerCreate(ur_context_handle_t hContext, +ur_result_t urMemImageGetInfo(::ur_mem_handle_t hMemory, + ur_image_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urSamplerCreate(::ur_context_handle_t hContext, const ur_sampler_desc_t *pDesc, - ur_sampler_handle_t *phSampler); -ur_result_t urSamplerRetain(ur_sampler_handle_t hSampler); -ur_result_t urSamplerRelease(ur_sampler_handle_t hSampler); -ur_result_t urSamplerGetInfo(ur_sampler_handle_t hSampler, + ::ur_sampler_handle_t *phSampler); +ur_result_t urSamplerRetain(::ur_sampler_handle_t hSampler); +ur_result_t urSamplerRelease(::ur_sampler_handle_t hSampler); +ur_result_t urSamplerGetInfo(::ur_sampler_handle_t hSampler, ur_sampler_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urSamplerGetNativeHandle(ur_sampler_handle_t hSampler, - ur_native_handle_t *phNativeSampler); +ur_result_t urSamplerGetNativeHandle(::ur_sampler_handle_t hSampler, + ::ur_native_handle_t *phNativeSampler); ur_result_t urSamplerCreateWithNativeHandle( - ur_native_handle_t hNativeSampler, ur_context_handle_t hContext, + ::ur_native_handle_t hNativeSampler, ::ur_context_handle_t hContext, const ur_sampler_native_properties_t *pProperties, - ur_sampler_handle_t *phSampler); -ur_result_t urUSMHostAlloc(ur_context_handle_t hContext, + ::ur_sampler_handle_t *phSampler); +ur_result_t urUSMHostAlloc(::ur_context_handle_t hContext, const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t size, + ::ur_usm_pool_handle_t pool, size_t size, void **ppMem); -ur_result_t urUSMDeviceAlloc(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urUSMDeviceAlloc(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t size, + ::ur_usm_pool_handle_t pool, size_t size, void **ppMem); -ur_result_t urUSMSharedAlloc(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urUSMSharedAlloc(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t size, + ::ur_usm_pool_handle_t pool, size_t size, void **ppMem); -ur_result_t urUSMFree(ur_context_handle_t hContext, void *pMem); -ur_result_t urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem, - ur_usm_alloc_info_t propName, size_t propSize, - void *pPropValue, size_t *pPropSizeRet); -ur_result_t urUSMPoolCreate(ur_context_handle_t hContext, +ur_result_t urUSMFree(::ur_context_handle_t hContext, void *pMem); +ur_result_t urUSMGetMemAllocInfo(::ur_context_handle_t hContext, + const void *pMem, ur_usm_alloc_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urUSMPoolCreate(::ur_context_handle_t hContext, ur_usm_pool_desc_t *pPoolDesc, - ur_usm_pool_handle_t *ppPool); -ur_result_t urUSMPoolRetain(ur_usm_pool_handle_t pPool); -ur_result_t urUSMPoolRelease(ur_usm_pool_handle_t pPool); -ur_result_t urUSMPoolGetInfo(ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t *ppPool); +ur_result_t urUSMPoolRetain(::ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolRelease(::ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolGetInfo(::ur_usm_pool_handle_t hPool, ur_usm_pool_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); ur_result_t urVirtualMemGranularityGetInfo( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, size_t allocationSize, ur_virtual_mem_granularity_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urVirtualMemReserve(ur_context_handle_t hContext, +ur_result_t urVirtualMemReserve(::ur_context_handle_t hContext, const void *pStart, size_t size, void **ppStart); -ur_result_t urVirtualMemFree(ur_context_handle_t hContext, const void *pStart, +ur_result_t urVirtualMemFree(::ur_context_handle_t hContext, const void *pStart, size_t size); -ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, - size_t size, ur_physical_mem_handle_t hPhysicalMem, +ur_result_t urVirtualMemMap(::ur_context_handle_t hContext, const void *pStart, + size_t size, + ::ur_physical_mem_handle_t hPhysicalMem, size_t offset, ur_virtual_mem_access_flags_t flags); -ur_result_t urVirtualMemUnmap(ur_context_handle_t hContext, const void *pStart, - size_t size); -ur_result_t urVirtualMemSetAccess(ur_context_handle_t hContext, +ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContext, + const void *pStart, size_t size); +ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContext, const void *pStart, size_t size, ur_virtual_mem_access_flags_t flags); -ur_result_t urVirtualMemGetInfo(ur_context_handle_t hContext, +ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContext, const void *pStart, size_t size, ur_virtual_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, - ur_device_handle_t hDevice, size_t size, +ur_result_t urPhysicalMemCreate(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, size_t size, const ur_physical_mem_properties_t *pProperties, - ur_physical_mem_handle_t *phPhysicalMem); -ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem); -ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem); -ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, + ::ur_physical_mem_handle_t *phPhysicalMem); +ur_result_t urPhysicalMemRetain(::ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemRelease(::ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemGetInfo(::ur_physical_mem_handle_t hPhysicalMem, ur_physical_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urProgramCreateWithIL(ur_context_handle_t hContext, const void *pIL, - size_t length, +ur_result_t urProgramCreateWithIL(::ur_context_handle_t hContext, + const void *pIL, size_t length, const ur_program_properties_t *pProperties, - ur_program_handle_t *phProgram); -ur_result_t urProgramCreateWithBinary( - ur_context_handle_t hContext, uint32_t numDevices, - ur_device_handle_t *phDevices, size_t *pLengths, const uint8_t **ppBinaries, - const ur_program_properties_t *pProperties, ur_program_handle_t *phProgram); -ur_result_t urProgramBuild(ur_context_handle_t hContext, - ur_program_handle_t hProgram, const char *pOptions); -ur_result_t urProgramCompile(ur_context_handle_t hContext, - ur_program_handle_t hProgram, + ::ur_program_handle_t *phProgram); +ur_result_t +urProgramCreateWithBinary(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, size_t *pLengths, + const uint8_t **ppBinaries, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram); +ur_result_t urProgramBuild(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions); +ur_result_t urProgramCompile(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, const char *pOptions); -ur_result_t urProgramLink(ur_context_handle_t hContext, uint32_t count, - const ur_program_handle_t *phPrograms, - const char *pOptions, ur_program_handle_t *phProgram); -ur_result_t urProgramRetain(ur_program_handle_t hProgram); -ur_result_t urProgramRelease(ur_program_handle_t hProgram); -ur_result_t urProgramGetFunctionPointer(ur_device_handle_t hDevice, - ur_program_handle_t hProgram, +ur_result_t urProgramLink(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms, + const char *pOptions, + ::ur_program_handle_t *phProgram); +ur_result_t urProgramRetain(::ur_program_handle_t hProgram); +ur_result_t urProgramRelease(::ur_program_handle_t hProgram); +ur_result_t urProgramGetFunctionPointer(::ur_device_handle_t hDevice, + ::ur_program_handle_t hProgram, const char *pFunctionName, void **ppFunctionPointer); ur_result_t urProgramGetGlobalVariablePointer( - ur_device_handle_t hDevice, ur_program_handle_t hProgram, + ::ur_device_handle_t hDevice, ::ur_program_handle_t hProgram, const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, void **ppGlobalVariablePointerRet); -ur_result_t urProgramGetInfo(ur_program_handle_t hProgram, +ur_result_t urProgramGetInfo(::ur_program_handle_t hProgram, ur_program_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urProgramGetBuildInfo(ur_program_handle_t hProgram, - ur_device_handle_t hDevice, +ur_result_t urProgramGetBuildInfo(::ur_program_handle_t hProgram, + ::ur_device_handle_t hDevice, ur_program_build_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); ur_result_t urProgramSetSpecializationConstants( - ur_program_handle_t hProgram, uint32_t count, + ::ur_program_handle_t hProgram, uint32_t count, const ur_specialization_constant_info_t *pSpecConstants); -ur_result_t urProgramGetNativeHandle(ur_program_handle_t hProgram, - ur_native_handle_t *phNativeProgram); +ur_result_t urProgramGetNativeHandle(::ur_program_handle_t hProgram, + ::ur_native_handle_t *phNativeProgram); ur_result_t urProgramCreateWithNativeHandle( - ur_native_handle_t hNativeProgram, ur_context_handle_t hContext, + ::ur_native_handle_t hNativeProgram, ::ur_context_handle_t hContext, const ur_program_native_properties_t *pProperties, - ur_program_handle_t *phProgram); -ur_result_t urQueueGetInfo(ur_queue_handle_t hQueue, ur_queue_info_t propName, + ::ur_program_handle_t *phProgram); +ur_result_t urQueueGetInfo(::ur_queue_handle_t hQueue, ur_queue_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urQueueCreate(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urQueueCreate(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_queue_properties_t *pProperties, - ur_queue_handle_t *phQueue); -ur_result_t urQueueRetain(ur_queue_handle_t hQueue); -ur_result_t urQueueRelease(ur_queue_handle_t hQueue); -ur_result_t urQueueGetNativeHandle(ur_queue_handle_t hQueue, + ::ur_queue_handle_t *phQueue); +ur_result_t urQueueRetain(::ur_queue_handle_t hQueue); +ur_result_t urQueueRelease(::ur_queue_handle_t hQueue); +ur_result_t urQueueGetNativeHandle(::ur_queue_handle_t hQueue, ur_queue_native_desc_t *pDesc, - ur_native_handle_t *phNativeQueue); -ur_result_t urQueueCreateWithNativeHandle( - ur_native_handle_t hNativeQueue, ur_context_handle_t hContext, - ur_device_handle_t hDevice, const ur_queue_native_properties_t *pProperties, - ur_queue_handle_t *phQueue); -ur_result_t urQueueFinish(ur_queue_handle_t hQueue); -ur_result_t urQueueFlush(ur_queue_handle_t hQueue); -ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, + ::ur_native_handle_t *phNativeQueue); +ur_result_t +urQueueCreateWithNativeHandle(::ur_native_handle_t hNativeQueue, + ::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_queue_native_properties_t *pProperties, + ::ur_queue_handle_t *phQueue); +ur_result_t urQueueFinish(::ur_queue_handle_t hQueue); +ur_result_t urQueueFlush(::ur_queue_handle_t hQueue); +ur_result_t urEventGetInfo(::ur_event_handle_t hEvent, ur_event_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urEventGetProfilingInfo(ur_event_handle_t hEvent, +ur_result_t urEventGetProfilingInfo(::ur_event_handle_t hEvent, ur_profiling_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); ur_result_t urEventWait(uint32_t numEvents, - const ur_event_handle_t *phEventWaitList); -ur_result_t urEventRetain(ur_event_handle_t hEvent); -ur_result_t urEventRelease(ur_event_handle_t hEvent); -ur_result_t urEventGetNativeHandle(ur_event_handle_t hEvent, - ur_native_handle_t *phNativeEvent); + const ::ur_event_handle_t *phEventWaitList); +ur_result_t urEventRetain(::ur_event_handle_t hEvent); +ur_result_t urEventRelease(::ur_event_handle_t hEvent); +ur_result_t urEventGetNativeHandle(::ur_event_handle_t hEvent, + ::ur_native_handle_t *phNativeEvent); ur_result_t -urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent, - ur_context_handle_t hContext, +urEventCreateWithNativeHandle(::ur_native_handle_t hNativeEvent, + ::ur_context_handle_t hContext, const ur_event_native_properties_t *pProperties, - ur_event_handle_t *phEvent); -ur_result_t urEventSetCallback(ur_event_handle_t hEvent, + ::ur_event_handle_t *phEvent); +ur_result_t urEventSetCallback(::ur_event_handle_t hEvent, ur_execution_info_t execStatus, ur_event_callback_t pfnNotify, void *pUserData); -ur_result_t urEnqueueEventsWait(ur_queue_handle_t hQueue, +ur_result_t urEnqueueEventsWait(::ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueEventsWaitWithBarrier( - ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); -ur_result_t urEnqueueMemBufferRead(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, bool blockingRead, + ::ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferRead(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingRead, size_t offset, size_t size, void *pDst, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemBufferWrite( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingWrite, size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemBufferReadRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingRead, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingRead, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pDst, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemBufferWriteRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingWrite, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueMemBufferCopy(ur_queue_handle_t hQueue, - ur_mem_handle_t hBufferSrc, - ur_mem_handle_t hBufferDst, size_t srcOffset, - size_t dstOffset, size_t size, - uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferCopy(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, + size_t srcOffset, size_t dstOffset, + size_t size, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemBufferCopyRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBufferSrc, - ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueMemBufferFill(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferFill(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, const void *pPattern, size_t patternSize, size_t offset, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemImageRead( - ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingRead, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImage, bool blockingRead, ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); ur_result_t urEnqueueMemImageWrite( - ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingWrite, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImage, bool blockingWrite, ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); ur_result_t -urEnqueueMemImageCopy(ur_queue_handle_t hQueue, ur_mem_handle_t hImageSrc, - ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, +urEnqueueMemImageCopy(::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImageSrc, + ::ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueMemBufferMap(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, bool blockingMap, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferMap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingMap, ur_map_flags_t mapFlags, size_t offset, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent, void **ppRetMap); -ur_result_t urEnqueueMemUnmap(ur_queue_handle_t hQueue, ur_mem_handle_t hMem, - void *pMappedPtr, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMFill(ur_queue_handle_t hQueue, void *pMem, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent, + void **ppRetMap); +ur_result_t urEnqueueMemUnmap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hMem, void *pMappedPtr, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill(::ur_queue_handle_t hQueue, void *pMem, size_t patternSize, const void *pPattern, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMMemcpy(ur_queue_handle_t hQueue, bool blocking, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy(::ur_queue_handle_t hQueue, bool blocking, void *pDst, const void *pSrc, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMPrefetch(ur_queue_handle_t hQueue, const void *pMem, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMPrefetch(::ur_queue_handle_t hQueue, const void *pMem, size_t size, ur_usm_migration_flags_t flags, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMAdvise(ur_queue_handle_t hQueue, const void *pMem, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMAdvise(::ur_queue_handle_t hQueue, const void *pMem, size_t size, ur_usm_advice_flags_t advice, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMFill2D(ur_queue_handle_t hQueue, void *pMem, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill2D(::ur_queue_handle_t hQueue, void *pMem, size_t pitch, size_t patternSize, const void *pPattern, size_t width, size_t height, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMMemcpy2D(ur_queue_handle_t hQueue, bool blocking, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy2D(::ur_queue_handle_t hQueue, bool blocking, void *pDst, size_t dstPitch, const void *pSrc, size_t srcPitch, size_t width, size_t height, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueDeviceGlobalVariableWrite( - ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, - bool blockingWrite, size_t count, size_t offset, const void *pSrc, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingWrite, size_t count, size_t offset, + const void *pSrc, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); ur_result_t urEnqueueDeviceGlobalVariableRead( - ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, - bool blockingRead, size_t count, size_t offset, void *pDst, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueReadHostPipe(ur_queue_handle_t hQueue, - ur_program_handle_t hProgram, + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingRead, size_t count, size_t offset, + void *pDst, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueReadHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, const char *pipe_symbol, bool blocking, void *pDst, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEnqueueWriteHostPipe(ur_queue_handle_t hQueue, - ur_program_handle_t hProgram, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueWriteHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, const char *pipe_symbol, bool blocking, void *pSrc, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueKernelLaunchWithArgsExp( - ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, + ::ur_queue_handle_t hQueue, ::ur_kernel_handle_t hKernel, uint32_t workDim, const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, const ur_kernel_launch_ext_properties_t *launchPropList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urKernelCreate(ur_program_handle_t hProgram, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urKernelCreate(::ur_program_handle_t hProgram, const char *pKernelName, - ur_kernel_handle_t *phKernel); -ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, + ::ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetInfo(::ur_kernel_handle_t hKernel, ur_kernel_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urKernelGetGroupInfo(ur_kernel_handle_t hKernel, - ur_device_handle_t hDevice, +ur_result_t urKernelGetGroupInfo(::ur_kernel_handle_t hKernel, + ::ur_device_handle_t hDevice, ur_kernel_group_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, - ur_device_handle_t hDevice, +ur_result_t urKernelGetSubGroupInfo(::ur_kernel_handle_t hKernel, + ::ur_device_handle_t hDevice, ur_kernel_sub_group_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urKernelRetain(ur_kernel_handle_t hKernel); -ur_result_t urKernelRelease(ur_kernel_handle_t hKernel); +ur_result_t urKernelRetain(::ur_kernel_handle_t hKernel); +ur_result_t urKernelRelease(::ur_kernel_handle_t hKernel); ur_result_t -urKernelSetExecInfo(ur_kernel_handle_t hKernel, ur_kernel_exec_info_t propName, - size_t propSize, +urKernelSetExecInfo(::ur_kernel_handle_t hKernel, + ur_kernel_exec_info_t propName, size_t propSize, const ur_kernel_exec_info_properties_t *pProperties, const void *pPropValue); ur_result_t urKernelSetSpecializationConstants( - ur_kernel_handle_t hKernel, uint32_t count, + ::ur_kernel_handle_t hKernel, uint32_t count, const ur_specialization_constant_info_t *pSpecConstants); -ur_result_t urKernelGetNativeHandle(ur_kernel_handle_t hKernel, - ur_native_handle_t *phNativeKernel); +ur_result_t urKernelGetNativeHandle(::ur_kernel_handle_t hKernel, + ::ur_native_handle_t *phNativeKernel); ur_result_t -urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, - ur_context_handle_t hContext, - ur_program_handle_t hProgram, +urKernelCreateWithNativeHandle(::ur_native_handle_t hNativeKernel, + ::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, const ur_kernel_native_properties_t *pProperties, - ur_kernel_handle_t *phKernel); -ur_result_t urKernelGetSuggestedLocalWorkSize(ur_kernel_handle_t hKernel, - ur_queue_handle_t hQueue, + ::ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetSuggestedLocalWorkSize(::ur_kernel_handle_t hKernel, + ::ur_queue_handle_t hQueue, uint32_t numWorkDim, const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, size_t *pSuggestedLocalWorkSize); ur_result_t urKernelGetSuggestedLocalWorkSizeWithArgs( - ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t numWorkDim, - const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, - uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, + ::ur_kernel_handle_t hKernel, ::ur_queue_handle_t hQueue, + uint32_t numWorkDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, size_t *pSuggestedLocalWorkSize); ur_result_t urKernelSuggestMaxCooperativeGroupCount( - ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, uint32_t workDim, - const size_t *pLocalWorkSize, size_t dynamicSharedMemorySize, - uint32_t *pGroupCountRet); + ::ur_kernel_handle_t hKernel, ::ur_device_handle_t hDevice, + uint32_t workDim, const size_t *pLocalWorkSize, + size_t dynamicSharedMemorySize, uint32_t *pGroupCountRet); ur_result_t urEnqueueUSMDeviceAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); ur_result_t urEnqueueUSMSharedAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); ur_result_t urEnqueueUSMHostAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent); -ur_result_t urEnqueueUSMFreeExp(ur_queue_handle_t hQueue, - ur_usm_pool_handle_t pPool, void *pMem, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFreeExp(::ur_queue_handle_t hQueue, + ::ur_usm_pool_handle_t pPool, void *pMem, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urUSMPoolCreateExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urUSMPoolCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, ur_usm_pool_desc_t *pPoolDesc, - ur_usm_pool_handle_t *pPool); -ur_result_t urUSMPoolDestroyExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t hPool); -ur_result_t urUSMPoolGetDefaultDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t *pPool); -ur_result_t urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolDestroyExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDefaultDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolGetInfoExp(::ur_usm_pool_handle_t hPool, ur_usm_pool_info_t propName, void *pPropValue, size_t *pPropSizeRet); -ur_result_t urUSMPoolSetInfoExp(ur_usm_pool_handle_t hPool, +ur_result_t urUSMPoolSetInfoExp(::ur_usm_pool_handle_t hPool, ur_usm_pool_info_t propName, void *pPropValue, size_t propSize); -ur_result_t urUSMPoolSetDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t hPool); -ur_result_t urUSMPoolGetDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t *pPool); -ur_result_t urUSMPoolTrimToExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t hPool, +ur_result_t urUSMPoolSetDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolTrimToExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool, size_t minBytesToKeep); -ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t widthInBytes, - size_t height, size_t elementSizeBytes, - void **ppMem, size_t *pResultPitch); + ::ur_usm_pool_handle_t pool, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch); ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_native_handle_t hImage); ur_result_t urBindlessImagesSampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_native_handle_t hImage); ur_result_t urBindlessImagesImageAllocateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_mem_native_handle_t *phImageMem); + ::ur_exp_image_mem_native_handle_t *phImageMem); ur_result_t -urBindlessImagesImageFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem); +urBindlessImagesImageFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem); ur_result_t urBindlessImagesUnsampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_native_handle_t *phImage); + ::ur_exp_image_native_handle_t *phImage); ur_result_t urBindlessImagesSampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, const ur_sampler_desc_t *pSamplerDesc, - ur_exp_image_native_handle_t *phImage); + ::ur_exp_image_native_handle_t *phImage); ur_result_t urBindlessImagesImageCopyExp( - ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + ::ur_queue_handle_t hQueue, const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, const ur_image_format_t *pSrcImageFormat, const ur_image_format_t *pDstImageFormat, ur_exp_image_copy_region_t *pCopyRegion, ur_exp_image_copy_flags_t imageCopyFlags, ur_exp_image_copy_input_types_t imageCopyInputTypes, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urBindlessImagesImageGetInfoExp( - ur_context_handle_t hContext, ur_exp_image_mem_native_handle_t hImageMem, - ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t +urBindlessImagesImageGetInfoExp(::ur_context_handle_t hContext, + ::ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, + size_t *pPropSizeRet); ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); ur_result_t urBindlessImagesMipmapGetLevelExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, - ur_exp_image_mem_native_handle_t *phImageMem); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ::ur_exp_image_mem_native_handle_t *phImageMem); ur_result_t -urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hMem); +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hMem); ur_result_t urBindlessImagesImportExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, size_t size, ur_exp_external_mem_type_t memHandleType, ur_exp_external_mem_desc_t *pExternalMemDesc, - ur_exp_external_mem_handle_t *phExternalMem); + ::ur_exp_external_mem_handle_t *phExternalMem); ur_result_t urBindlessImagesMapExternalArrayExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_external_mem_handle_t hExternalMem, - ur_exp_image_mem_native_handle_t *phImageMem); + ::ur_exp_external_mem_handle_t hExternalMem, + ::ur_exp_image_mem_native_handle_t *phImageMem); ur_result_t urBindlessImagesMapExternalLinearMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, - uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **ppRetMem); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + uint64_t offset, uint64_t size, ::ur_exp_external_mem_handle_t hExternalMem, + void **ppRetMem); ur_result_t urBindlessImagesReleaseExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_mem_handle_t hExternalMem); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_mem_handle_t hExternalMem); ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, void *pMem); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem); ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( - ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, + ::ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, ur_bool_t *pSupportedRet); ur_result_t urBindlessImagesImportExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, ur_exp_external_semaphore_type_t semHandleType, ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, - ur_exp_external_semaphore_handle_t *phExternalSemaphore); + ::ur_exp_external_semaphore_handle_t *phExternalSemaphore); ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_semaphore_handle_t hExternalSemaphore); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_semaphore_handle_t hExternalSemaphore); ur_result_t urBindlessImagesWaitExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, bool hasWaitValue, uint64_t waitValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); ur_result_t urBindlessImagesSignalExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, bool hasSignalValue, uint64_t signalValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); -ur_result_t urDeviceWaitExp(ur_device_handle_t hDevice); -ur_result_t urProgramDynamicLinkExp(ur_context_handle_t hContext, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urDeviceWaitExp(::ur_device_handle_t hDevice); +ur_result_t urProgramDynamicLinkExp(::ur_context_handle_t hContext, uint32_t count, - const ur_program_handle_t *phPrograms); + const ::ur_program_handle_t *phPrograms); ur_result_t urEnqueueTimestampRecordingExp( - ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); -ur_result_t urIPCGetMemHandleExp(ur_context_handle_t hContext, void *pMem, + ::ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urIPCGetMemHandleExp(::ur_context_handle_t hContext, void *pMem, void **ppIPCMemHandleData, size_t *pIPCMemHandleDataSizeRet); -ur_result_t urIPCPutMemHandleExp(ur_context_handle_t hContext, +ur_result_t urIPCPutMemHandleExp(::ur_context_handle_t hContext, void *pIPCMemHandleData); -ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urIPCOpenMemHandleExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem); -ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); -ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, - ur_physical_mem_handle_t hPhysMem, +ur_result_t urIPCCloseMemHandleExp(::ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_physical_mem_handle_t hPhysMem, void **ppIPCPhysMemHandleData, size_t *pIPCPhysMemHandleDataSizeRet); -ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, +ur_result_t urIPCPutPhysMemHandleExp(::ur_context_handle_t hContext, const void *pIPCPhysMemHandleData); -ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urIPCOpenPhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const void *pIPCPhysMemHandleData, size_t ipcPhysMemHandleDataSize, - ur_physical_mem_handle_t *phPhysMem); -ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, - ur_physical_mem_handle_t hPhysMem); -ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent, + ::ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_physical_mem_handle_t hPhysMem); +ur_result_t urIPCGetEventHandleExp(::ur_event_handle_t hEvent, void **ppIPCEventHandleData, size_t *pIPCEventHandleDataSizeRet); -ur_result_t urIPCPutEventHandleExp(ur_context_handle_t hContext, +ur_result_t urIPCPutEventHandleExp(::ur_context_handle_t hContext, void *pIPCEventHandleData); -ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext, +ur_result_t urIPCOpenEventHandleExp(::ur_context_handle_t hContext, const void *pIPCEventHandleData, size_t ipcEventHandleDataSize, - ur_event_handle_t *phEvent); + ::ur_event_handle_t *phEvent); ur_result_t urMemoryExportAllocExportableMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, - size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); -ur_result_t urMemoryExportFreeExportableMemoryExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - void *pMem); + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t +urMemoryExportFreeExportableMemoryExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, void *pMem); ur_result_t urMemoryExportExportMemoryHandleExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, ur_exp_external_mem_type_t handleTypeToExport, void *pMem, void *pMemHandleRet); -ur_result_t urProgramBuildExp(ur_program_handle_t hProgram, uint32_t numDevices, - ur_device_handle_t *phDevices, +ur_result_t urProgramBuildExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, const char *pOptions); -ur_result_t urProgramCompileExp(ur_program_handle_t hProgram, +ur_result_t urProgramCompileExp(::ur_program_handle_t hProgram, uint32_t numDevices, - ur_device_handle_t *phDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, const char *pOptions); -ur_result_t urProgramLinkExp(ur_context_handle_t hContext, uint32_t numDevices, - ur_device_handle_t *phDevices, - ur_exp_program_flags_t flags, uint32_t count, - const ur_program_handle_t *phPrograms, - const char *pOptions, - ur_program_handle_t *phProgram); -ur_result_t urUSMContextMemcpyExp(ur_context_handle_t hContext, void *pDst, +ur_result_t +urProgramLinkExp(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, + uint32_t count, const ::ur_program_handle_t *phPrograms, + const char *pOptions, ::ur_program_handle_t *phProgram); +ur_result_t urUSMContextMemcpyExp(::ur_context_handle_t hContext, void *pDst, const void *pSrc, size_t size); ur_result_t urUSMHostAllocRegisterExp( - ur_context_handle_t hContext, void *pHostMem, size_t size, + ::ur_context_handle_t hContext, void *pHostMem, size_t size, const ur_exp_usm_host_alloc_register_properties_t *pProperties); -ur_result_t urUSMHostAllocUnregisterExp(ur_context_handle_t hContext, +ur_result_t urUSMHostAllocUnregisterExp(::ur_context_handle_t hContext, void *pHostMem); -ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *pMem, +ur_result_t urUSMImportExp(::ur_context_handle_t hContext, void *pMem, size_t size); -ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *pMem); -ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice); -ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice); -ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice, +ur_result_t urUSMReleaseExp(::ur_context_handle_t hContext, void *pMem); +ur_result_t urUsmP2PEnablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice); +ur_result_t urUsmP2PDisablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice); +ur_result_t urUsmP2PPeerAccessGetInfoExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice, ur_exp_peer_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); ur_result_t -urCommandBufferCreateExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +urCommandBufferCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - ur_exp_command_buffer_handle_t *phCommandBuffer); + ::ur_exp_command_buffer_handle_t *phCommandBuffer); ur_result_t -urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer); +urCommandBufferRetainExp(::ur_exp_command_buffer_handle_t hCommandBuffer); ur_result_t -urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer); +urCommandBufferReleaseExp(::ur_exp_command_buffer_handle_t hCommandBuffer); ur_result_t -urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer); +urCommandBufferFinalizeExp(::ur_exp_command_buffer_handle_t hCommandBuffer); ur_result_t urCommandBufferAppendKernelLaunchExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, - uint32_t workDim, const size_t *pGlobalWorkOffset, - const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, - uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternatives, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, - uint32_t workDim, const size_t *pGlobalWorkOffset, - const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, - uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, - uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternatives, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendUSMMemcpyExp( - ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, const void *pSrc, - size_t size, uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, + const void *pSrc, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendUSMFillExp( - ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory, const void *pPattern, size_t patternSize, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferCopyExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, - ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hSrcMem, + ::ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferWriteExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, size_t offset, size_t size, const void *pSrc, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferReadExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, size_t offset, size_t size, void *pDst, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferCopyRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, - ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hSrcMem, + ::ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferWriteRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferReadRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pDst, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendMemBufferFillExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, const void *pPattern, size_t patternSize, size_t offset, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendUSMPrefetchExp( - ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, size_t size, ur_usm_migration_flags_t flags, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendUSMAdviseExp( - ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, size_t size, ur_usm_advice_flags_t advice, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, - ur_exp_command_buffer_command_handle_t *phCommand); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); ur_result_t urCommandBufferAppendNativeCommandExp( - ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_exp_command_buffer_handle_t hCommandBuffer, ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - void *pData, ur_exp_command_buffer_handle_t hChildCommandBuffer, + void *pData, ::ur_exp_command_buffer_handle_t hChildCommandBuffer, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ur_exp_command_buffer_sync_point_t *pSyncPoint); ur_result_t urEnqueueCommandBufferExp( - ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + ::ur_queue_handle_t hQueue, ::ur_exp_command_buffer_handle_t hCommandBuffer, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urCommandBufferUpdateKernelLaunchExp( - ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates, + ::ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates, const ur_exp_command_buffer_update_kernel_launch_desc_t *pUpdateKernelLaunch); ur_result_t urCommandBufferUpdateSignalEventExp( - ur_exp_command_buffer_command_handle_t hCommand, - ur_event_handle_t *phSignalEvent); + ::ur_exp_command_buffer_command_handle_t hCommand, + ::ur_event_handle_t *phSignalEvent); ur_result_t urCommandBufferUpdateWaitEventsExp( - ur_exp_command_buffer_command_handle_t hCommand, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList); + ::ur_exp_command_buffer_command_handle_t hCommand, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList); ur_result_t -urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, +urCommandBufferGetInfoExp(::ur_exp_command_buffer_handle_t hCommandBuffer, ur_exp_command_buffer_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet); -ur_result_t -urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, - ur_native_handle_t *phNativeCommandBuffer); +ur_result_t urCommandBufferGetNativeHandleExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_native_handle_t *phNativeCommandBuffer); ur_result_t urEnqueueHostTaskExp( - ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, + ::ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, void *data, const ur_exp_host_task_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueEventsWaitWithBarrierExt( - ur_queue_handle_t hQueue, + ::ur_queue_handle_t hQueue, const ur_exp_enqueue_ext_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); ur_result_t urEnqueueNativeCommandExp( - ur_queue_handle_t hQueue, + ::ur_queue_handle_t hQueue, ur_exp_enqueue_native_command_function_t pfnNativeEnqueue, void *data, - uint32_t numMemsInMemList, const ur_mem_handle_t *phMemList, + uint32_t numMemsInMemList, const ::ur_mem_handle_t *phMemList, const ur_exp_enqueue_native_command_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urEventCreateExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEventCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, const ur_exp_event_desc_t *pEventDesc, - ur_event_handle_t *phEvent); -ur_result_t urGraphCreateExp(ur_context_handle_t hContext, - ur_exp_graph_handle_t *phGraph); -ur_result_t urQueueBeginGraphCaptureExp(ur_queue_handle_t hQueue); -ur_result_t urQueueBeginCaptureIntoGraphExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t hGraph); -ur_result_t urQueueEndGraphCaptureExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t *phGraph); + ::ur_event_handle_t *phEvent); +ur_result_t urGraphCreateExp(::ur_context_handle_t hContext, + ::ur_exp_graph_handle_t *phGraph); +ur_result_t urQueueBeginGraphCaptureExp(::ur_queue_handle_t hQueue); +ur_result_t urQueueBeginCaptureIntoGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t hGraph); +ur_result_t urQueueEndGraphCaptureExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph); ur_result_t -urGraphInstantiateGraphExp(ur_exp_graph_handle_t hGraph, - ur_exp_executable_graph_handle_t *phExecGraph); -ur_result_t urEnqueueGraphExp(ur_queue_handle_t hQueue, - ur_exp_executable_graph_handle_t hGraph, +urGraphInstantiateGraphExp(::ur_exp_graph_handle_t hGraph, + ::ur_exp_executable_graph_handle_t *phExecGraph); +ur_result_t urEnqueueGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_executable_graph_handle_t hGraph, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); -ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t hGraph); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urGraphDestroyExp(::ur_exp_graph_handle_t hGraph); ur_result_t urGraphExecutableGraphDestroyExp( - ur_exp_executable_graph_handle_t hExecutableGraph); -ur_result_t urQueueIsGraphCaptureEnabledExp(ur_queue_handle_t hQueue, + ::ur_exp_executable_graph_handle_t hExecutableGraph); +ur_result_t urQueueIsGraphCaptureEnabledExp(::ur_queue_handle_t hQueue, bool *pResult); -ur_result_t urQueueGetGraphExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t *phGraph); -ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t hGraph, bool *pResult); -ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t hGraph, uint64_t *pGraphId); +ur_result_t urQueueGetGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph); +ur_result_t urGraphIsEmptyExp(::ur_exp_graph_handle_t hGraph, bool *pResult); +ur_result_t urGraphGetIdExp(::ur_exp_graph_handle_t hGraph, uint64_t *pGraphId); ur_result_t urGraphSetDestructionCallbackExp( - ur_exp_graph_handle_t hGraph, + ::ur_exp_graph_handle_t hGraph, ur_exp_graph_destruction_callback_t pfnCallback, void *pUserData); -ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t hGraph, +ur_result_t urGraphDumpContentsExp(::ur_exp_graph_handle_t hGraph, const char *filePath); -ur_result_t urGraphGetNativeHandleExp(ur_exp_graph_handle_t hGraph, - ur_native_handle_t *phNativeGraph); +ur_result_t urGraphGetNativeHandleExp(::ur_exp_graph_handle_t hGraph, + ::ur_native_handle_t *phNativeGraph); ur_result_t urGraphExecutableGraphGetNativeHandleExp( - ur_exp_executable_graph_handle_t hExecutableGraph, - ur_native_handle_t *phNativeExecutableGraph); + ::ur_exp_executable_graph_handle_t hExecutableGraph, + ::ur_native_handle_t *phNativeExecutableGraph); #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); #endif @@ -909,4 +942,4 @@ ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); struct ddi_getter { const static ur_dditable_t *value(); }; -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader_common_forwarders.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader_common_forwarders.hpp new file mode 100644 index 0000000000000..579af2d39f03f --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader_common_forwarders.hpp @@ -0,0 +1,503 @@ +//===--------- ur_interface_loader_common_forwarders.hpp - L0 Adapter ----===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Several official UR entry points are implemented once, in the common +// `ur::level_zero` namespace (see common/api.hpp). The generated interface +// loader declares every entry point inside `ur::level_zero::v1`. For the common +// entry points those `ur::level_zero::v1` symbols would otherwise be undefined. + +#pragma once + +#include + +#include "common/api.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v1 { + +inline ur_result_t urDeviceGet(::ur_platform_handle_t hPlatform, + ::ur_device_type_t DeviceType, + uint32_t NumEntries, + ::ur_device_handle_t *phDevices, + uint32_t *pNumDevices) { + return ::ur::level_zero::urDeviceGet(hPlatform, DeviceType, NumEntries, + phDevices, pNumDevices); +} +inline ur_result_t urDeviceRetain(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceRetain(hDevice); +} +inline ur_result_t urDeviceRelease(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceRelease(hDevice); +} +inline ur_result_t +urDevicePartition(::ur_device_handle_t hDevice, + const ::ur_device_partition_properties_t *pProperties, + uint32_t NumDevices, ::ur_device_handle_t *phSubDevices, + uint32_t *pNumDevicesRet) { + return ::ur::level_zero::urDevicePartition(hDevice, pProperties, NumDevices, + phSubDevices, pNumDevicesRet); +} +inline ur_result_t urDeviceSelectBinary(::ur_device_handle_t hDevice, + const ::ur_device_binary_t *pBinaries, + uint32_t NumBinaries, + uint32_t *pSelectedBinary) { + return ::ur::level_zero::urDeviceSelectBinary(hDevice, pBinaries, NumBinaries, + pSelectedBinary); +} +inline ur_result_t +urDeviceGetNativeHandle(::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeDevice) { + return ::ur::level_zero::urDeviceGetNativeHandle(hDevice, phNativeDevice); +} +inline ur_result_t urDeviceCreateWithNativeHandle( + ::ur_native_handle_t hNativeDevice, ::ur_adapter_handle_t hAdapter, + const ::ur_device_native_properties_t *pProperties, + ::ur_device_handle_t *phDevice) { + return ::ur::level_zero::urDeviceCreateWithNativeHandle( + hNativeDevice, hAdapter, pProperties, phDevice); +} +inline ur_result_t urDeviceGetGlobalTimestamps(::ur_device_handle_t hDevice, + uint64_t *pDeviceTimestamp, + uint64_t *pHostTimestamp) { + return ::ur::level_zero::urDeviceGetGlobalTimestamps( + hDevice, pDeviceTimestamp, pHostTimestamp); +} +inline ur_result_t urDeviceWaitExp(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceWaitExp(hDevice); +} +inline ur_result_t urPlatformGet(::ur_adapter_handle_t hAdapter, + uint32_t NumEntries, + ::ur_platform_handle_t *phPlatforms, + uint32_t *pNumPlatforms) { + return ::ur::level_zero::urPlatformGet(hAdapter, NumEntries, phPlatforms, + pNumPlatforms); +} +inline ur_result_t urPlatformGetInfo(::ur_platform_handle_t hPlatform, + ur_platform_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + return ::ur::level_zero::urPlatformGetInfo(hPlatform, propName, propSize, + pPropValue, pPropSizeRet); +} +inline ur_result_t urPlatformGetApiVersion(::ur_platform_handle_t hPlatform, + ur_api_version_t *pVersion) { + return ::ur::level_zero::urPlatformGetApiVersion(hPlatform, pVersion); +} +inline ur_result_t +urPlatformGetNativeHandle(::ur_platform_handle_t hPlatform, + ::ur_native_handle_t *phNativePlatform) { + return ::ur::level_zero::urPlatformGetNativeHandle(hPlatform, + phNativePlatform); +} +inline ur_result_t urPlatformCreateWithNativeHandle( + ::ur_native_handle_t hNativePlatform, ::ur_adapter_handle_t hAdapter, + const ur_platform_native_properties_t *pProperties, + ::ur_platform_handle_t *phPlatform) { + return ::ur::level_zero::urPlatformCreateWithNativeHandle( + hNativePlatform, hAdapter, pProperties, phPlatform); +} +inline ur_result_t urPlatformGetBackendOption(::ur_platform_handle_t hPlatform, + const char *pFrontendOption, + const char **ppPlatformOption) { + return ::ur::level_zero::urPlatformGetBackendOption( + hPlatform, pFrontendOption, ppPlatformOption); +} +inline ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapter) { + return ::ur::level_zero::urAdapterRelease(hAdapter); +} +inline ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapter) { + return ::ur::level_zero::urAdapterRetain(hAdapter); +} +inline ur_result_t urAdapterGetLastError(::ur_adapter_handle_t hAdapter, + const char **ppMessage, + int32_t *pError) { + return ::ur::level_zero::urAdapterGetLastError(hAdapter, ppMessage, pError); +} +inline ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapter, + ur_adapter_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urAdapterGetInfo(hAdapter, propName, propSize, + pPropValue, pPropSizeRet); +} +inline ur_result_t +urAdapterSetLoggerCallback(::ur_adapter_handle_t hAdapter, + ur_logger_callback_t pfnLoggerCallback, + void *pUserData, ur_logger_level_t level) { + return ::ur::level_zero::urAdapterSetLoggerCallback( + hAdapter, pfnLoggerCallback, pUserData, level); +} +inline ur_result_t +urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapter, + ur_logger_level_t level) { + return ::ur::level_zero::urAdapterSetLoggerCallbackLevel(hAdapter, level); +} +inline ur_result_t urSamplerCreate(::ur_context_handle_t Context, + const ur_sampler_desc_t *Props, + ::ur_sampler_handle_t *Sampler) { + return ::ur::level_zero::urSamplerCreate(Context, Props, Sampler); +} +inline ur_result_t urSamplerRetain(::ur_sampler_handle_t Sampler) { + return ::ur::level_zero::urSamplerRetain(Sampler); +} +inline ur_result_t urSamplerRelease(::ur_sampler_handle_t Sampler) { + return ::ur::level_zero::urSamplerRelease(Sampler); +} +inline ur_result_t urSamplerGetInfo(::ur_sampler_handle_t Sampler, + ur_sampler_info_t PropName, + size_t PropValueSize, void *PropValue, + size_t *PropSizeRet) { + return ::ur::level_zero::urSamplerGetInfo(Sampler, PropName, PropValueSize, + PropValue, PropSizeRet); +} +inline ur_result_t +urSamplerGetNativeHandle(::ur_sampler_handle_t Sampler, + ::ur_native_handle_t *NativeSampler) { + return ::ur::level_zero::urSamplerGetNativeHandle(Sampler, NativeSampler); +} +inline ur_result_t urSamplerCreateWithNativeHandle( + ::ur_native_handle_t NativeSampler, ::ur_context_handle_t Context, + const ur_sampler_native_properties_t *Properties, + ::ur_sampler_handle_t *Sampler) { + return ::ur::level_zero::urSamplerCreateWithNativeHandle( + NativeSampler, Context, Properties, Sampler); +} +inline ur_result_t +urProgramCreateWithIL(::ur_context_handle_t hContext, const void *pIL, + size_t length, const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithIL(hContext, pIL, length, + pProperties, phProgram); +} +inline ur_result_t +urProgramCreateWithBinary(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, size_t *pLengths, + const uint8_t **ppBinaries, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithBinary( + hContext, numDevices, phDevices, pLengths, ppBinaries, pProperties, + phProgram); +} +inline ur_result_t urProgramBuild(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions) { + return ::ur::level_zero::urProgramBuild(hContext, hProgram, pOptions); +} +inline ur_result_t urProgramCompile(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions) { + return ::ur::level_zero::urProgramCompile(hContext, hProgram, pOptions); +} +inline ur_result_t urProgramLink(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms, + const char *pOptions, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramLink(hContext, count, phPrograms, pOptions, + phProgram); +} +inline ur_result_t urProgramRetain(::ur_program_handle_t hProgram) { + return ::ur::level_zero::urProgramRetain(hProgram); +} +inline ur_result_t urProgramRelease(::ur_program_handle_t hProgram) { + return ::ur::level_zero::urProgramRelease(hProgram); +} +inline ur_result_t urProgramGetFunctionPointer(::ur_device_handle_t hDevice, + ::ur_program_handle_t hProgram, + const char *pFunctionName, + void **ppFunctionPointer) { + return ::ur::level_zero::urProgramGetFunctionPointer( + hDevice, hProgram, pFunctionName, ppFunctionPointer); +} +inline ur_result_t urProgramGetGlobalVariablePointer( + ::ur_device_handle_t hDevice, ::ur_program_handle_t hProgram, + const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, + void **ppGlobalVariablePointerRet) { + return ::ur::level_zero::urProgramGetGlobalVariablePointer( + hDevice, hProgram, pGlobalVariableName, pGlobalVariableSizeRet, + ppGlobalVariablePointerRet); +} +inline ur_result_t urProgramGetInfo(::ur_program_handle_t hProgram, + ur_program_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urProgramGetInfo(hProgram, propName, propSize, + pPropValue, pPropSizeRet); +} +inline ur_result_t urProgramGetBuildInfo(::ur_program_handle_t hProgram, + ::ur_device_handle_t hDevice, + ur_program_build_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + return ::ur::level_zero::urProgramGetBuildInfo( + hProgram, hDevice, propName, propSize, pPropValue, pPropSizeRet); +} +inline ur_result_t urProgramSetSpecializationConstants( + ::ur_program_handle_t hProgram, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants) { + return ::ur::level_zero::urProgramSetSpecializationConstants(hProgram, count, + pSpecConstants); +} +inline ur_result_t +urProgramGetNativeHandle(::ur_program_handle_t hProgram, + ::ur_native_handle_t *phNativeProgram) { + return ::ur::level_zero::urProgramGetNativeHandle(hProgram, phNativeProgram); +} +inline ur_result_t urProgramCreateWithNativeHandle( + ::ur_native_handle_t hNativeProgram, ::ur_context_handle_t hContext, + const ur_program_native_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithNativeHandle( + hNativeProgram, hContext, pProperties, phProgram); +} +inline ur_result_t +urProgramDynamicLinkExp(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms) { + return ::ur::level_zero::urProgramDynamicLinkExp(hContext, count, phPrograms); +} +inline ur_result_t urProgramBuildExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions) { + return ::ur::level_zero::urProgramBuildExp(hProgram, numDevices, phDevices, + flags, pOptions); +} +inline ur_result_t urProgramCompileExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions) { + return ::ur::level_zero::urProgramCompileExp(hProgram, numDevices, phDevices, + flags, pOptions); +} +inline ur_result_t +urProgramLinkExp(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, + uint32_t count, const ::ur_program_handle_t *phPrograms, + const char *pOptions, ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramLinkExp(hContext, numDevices, phDevices, + flags, count, phPrograms, pOptions, + phProgram); +} +inline ur_result_t urVirtualMemGranularityGetInfo( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t allocationSize, ur_virtual_mem_granularity_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urVirtualMemGranularityGetInfo( + hContext, hDevice, allocationSize, propName, propSize, pPropValue, + pPropSizeRet); +} +inline ur_result_t urVirtualMemReserve(::ur_context_handle_t hContext, + const void *pStart, size_t size, + void **ppStart) { + return ::ur::level_zero::urVirtualMemReserve(hContext, pStart, size, ppStart); +} +inline ur_result_t urVirtualMemFree(::ur_context_handle_t hContext, + const void *pStart, size_t size) { + return ::ur::level_zero::urVirtualMemFree(hContext, pStart, size); +} +inline ur_result_t urVirtualMemMap(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ::ur_physical_mem_handle_t hPhysicalMem, + size_t offset, + ur_virtual_mem_access_flags_t flags) { + return ::ur::level_zero::urVirtualMemMap(hContext, pStart, size, hPhysicalMem, + offset, flags); +} +inline ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContext, + const void *pStart, size_t size) { + return ::ur::level_zero::urVirtualMemUnmap(hContext, pStart, size); +} +inline ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_access_flags_t flags) { + return ::ur::level_zero::urVirtualMemSetAccess(hContext, pStart, size, flags); +} +inline ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + return ::ur::level_zero::urVirtualMemGetInfo( + hContext, pStart, size, propName, propSize, pPropValue, pPropSizeRet); +} +inline ur_result_t urMemoryExportAllocExportableMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem) { + return ::ur::level_zero::urMemoryExportAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); +} +inline ur_result_t urMemoryExportFreeExportableMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem) { + return ::ur::level_zero::urMemoryExportFreeExportableMemoryExp(hContext, + hDevice, pMem); +} +inline ur_result_t urMemoryExportExportMemoryHandleExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet) { + return ::ur::level_zero::urMemoryExportExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); +} +inline ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch) { + return ::ur::level_zero::urUSMPitchedAllocExp( + hContext, hDevice, pUSMDesc, pool, widthInBytes, height, elementSizeBytes, + ppMem, pResultPitch); +} +inline ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + return ::ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp( + hContext, hDevice, hImage); +} +inline ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + return ::ur::level_zero::urBindlessImagesSampledImageHandleDestroyExp( + hContext, hDevice, hImage); +} +inline ur_result_t urBindlessImagesImageAllocateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesImageAllocateExp( + hContext, hDevice, pImageFormat, pImageDesc, phImageMem); +} +inline ur_result_t +urBindlessImagesImageFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem) { + return ::ur::level_zero::urBindlessImagesImageFreeExp(hContext, hDevice, + hImageMem); +} +inline ur_result_t urBindlessImagesUnsampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_native_handle_t *phImage) { + return ::ur::level_zero::urBindlessImagesUnsampledImageCreateExp( + hContext, hDevice, hImageMem, pImageFormat, pImageDesc, phImage); +} +inline ur_result_t urBindlessImagesSampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_sampler_desc_t *pSamplerDesc, + ur_exp_image_native_handle_t *phImage) { + return ::ur::level_zero::urBindlessImagesSampledImageCreateExp( + hContext, hDevice, hImageMem, pImageFormat, pImageDesc, pSamplerDesc, + phImage); +} +inline ur_result_t urBindlessImagesImageGetInfoExp( + ::ur_context_handle_t hContext, ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urBindlessImagesImageGetInfoExp( + hContext, hImageMem, propName, pPropValue, pPropSizeRet); +} +inline ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageMemoryHandleTypeSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageUnsampledHandleSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageSampledHandleSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesMipmapGetLevelExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesMipmapGetLevelExp( + hContext, hDevice, hImageMem, mipmapLevel, phImageMem); +} +inline ur_result_t urBindlessImagesImportExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, size_t size, + ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ::ur_exp_external_mem_handle_t *phExternalMem) { + return ::ur::level_zero::urBindlessImagesImportExternalMemoryExp( + hContext, hDevice, size, memHandleType, pExternalMemDesc, phExternalMem); +} +inline ur_result_t urBindlessImagesMapExternalArrayExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_external_mem_handle_t hExternalMem, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesMapExternalArrayExp( + hContext, hDevice, pImageFormat, pImageDesc, hExternalMem, phImageMem); +} +inline ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + uint64_t offset, uint64_t size, ::ur_exp_external_mem_handle_t hExternalMem, + void **ppRetMem) { + return ::ur::level_zero::urBindlessImagesMapExternalLinearMemoryExp( + hContext, hDevice, offset, size, hExternalMem, ppRetMem); +} +inline ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_mem_handle_t hExternalMem) { + return ::ur::level_zero::urBindlessImagesReleaseExternalMemoryExp( + hContext, hDevice, hExternalMem); +} +inline ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem) { + return ::ur::level_zero::urBindlessImagesFreeMappedLinearMemoryExp( + hContext, hDevice, pMem); +} +inline ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( + ::ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, + ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesSupportsImportingHandleTypeExp( + hDevice, memHandleType, pSupportedRet); +} +inline ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ::ur_exp_external_semaphore_handle_t *phExternalSemaphore) { + return ::ur::level_zero::urBindlessImagesImportExternalSemaphoreExp( + hContext, hDevice, semHandleType, pExternalSemaphoreDesc, + phExternalSemaphore); +} +inline ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_semaphore_handle_t hExternalSemaphore) { + return ::ur::level_zero::urBindlessImagesReleaseExternalSemaphoreExp( + hContext, hDevice, hExternalSemaphore); +} +inline ur_result_t +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hMem) { + return ::ur::level_zero::urBindlessImagesMipmapFreeExp(hContext, hDevice, + hMem); +} + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/ur_level_zero.cpp b/unified-runtime/source/adapters/level_zero/ur_level_zero.cpp index 741da2b6eeb0a..16cfa778ff383 100644 --- a/unified-runtime/source/adapters/level_zero/ur_level_zero.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_level_zero.cpp @@ -12,6 +12,3 @@ #include #include "ur_level_zero.hpp" - -// Define the static class field -std::mutex ZeCall::GlobalLock; diff --git a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp index 9cd954311283c..66309d29d2feb 100644 --- a/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_level_zero.hpp @@ -24,14 +24,14 @@ #include #include "common.hpp" +#include "common/device.hpp" +#include "common/physical_mem.hpp" +#include "common/platform.hpp" +#include "common/sampler.hpp" #include "context.hpp" -#include "device.hpp" #include "event.hpp" #include "kernel.hpp" #include "memory.hpp" -#include "physical_mem.hpp" -#include "platform.hpp" #include "program.hpp" #include "queue.hpp" -#include "sampler.hpp" #include "usm.hpp" diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index 83e603317fed9..4694ee5411ceb 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -28,22 +28,9 @@ #include #include -namespace umf { -ur_result_t getProviderNativeError(const char *providerName, - int32_t nativeError) { - if (strcmp(providerName, "Level Zero") == 0) { - // L0 provider stores native errors of ur_result_t type - return static_cast(nativeError); - } - - return UR_RESULT_ERROR_UNKNOWN; -} -} // namespace umf - -usm::DisjointPoolAllConfigs DisjointPoolConfigInstance = - InitializeDisjointPoolConfig(); +namespace ur::level_zero::v1 { -usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig() { +static usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig() { // Prefer the UR-specific env var, fall back to the PI-specific one. const char *PoolTraceVal = std::getenv("UR_L0_USM_ALLOCATOR_TRACE"); if (!PoolTraceVal) { @@ -67,6 +54,9 @@ usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig() { return usm::parseDisjointPoolConfig(PoolConfigVal, PoolTrace); } +usm::DisjointPoolAllConfigs DisjointPoolConfigInstance = + InitializeDisjointPoolConfig(); + enum class USMAllocationForceResidencyType { // Do not force memory residency at allocation time. None = 0, @@ -138,8 +128,8 @@ static ur_result_t USMAllocationMakeResident( std::list Devices; if (!Device) { // Host allocation, make it resident on all devices in the context - Devices.insert(Devices.end(), Context->Devices.begin(), - Context->Devices.end()); + Devices.insert(Devices.end(), Context->getDevices().begin(), + Context->getDevices().end()); } else { Devices.push_back(Device); if (ForceResidency == USMAllocationForceResidencyType::P2PDevices) { @@ -152,7 +142,7 @@ static ur_result_t USMAllocationMakeResident( // Query for P2P devices and update the cache std::list P2PDevices; ze_bool_t P2P; - for (const auto &D : Context->Devices) { + for (const auto &D : Context->getDevices()) { if (D == Device) continue; ZE2UR_CALL(zeDeviceCanAccessPeer, @@ -295,22 +285,21 @@ static ur_result_t USMHostAllocImpl(void **ResultPtr, return UR_RESULT_SUCCESS; } -namespace ur::level_zero { - ur_result_t urUSMHostAlloc( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *USMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t Pool, + ::ur_usm_pool_handle_t PoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t Size, /// [out] pointer to USM host memory object void **RetMem) { + auto Context = v1_cast(ContextOpque); ur_usm_pool_handle_t UrPool = nullptr; - if (Pool) { - UrPool = Pool; + if (PoolOpque) { + UrPool = v1_cast(PoolOpque); } else { UrPool = &Context->DefaultPool; } @@ -321,58 +310,61 @@ ur_result_t urUSMHostAlloc( ur_result_t urUSMDeviceAlloc( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] handle of the device object - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *USMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t Pool, + ::ur_usm_pool_handle_t PoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t Size, /// [out] pointer to USM device memory object void **RetMem) { + auto Context = v1_cast(ContextOpque); ur_usm_pool_handle_t UrPool = nullptr; - if (Pool) { - UrPool = Pool; + if (PoolOpque) { + UrPool = v1_cast(PoolOpque); } else { UrPool = &Context->DefaultPool; } - return UrPool->allocate(Context, Device, USMDesc, UR_USM_TYPE_DEVICE, Size, - RetMem); + return UrPool->allocate(Context, common_cast(DeviceOpque), USMDesc, + UR_USM_TYPE_DEVICE, Size, RetMem); } ur_result_t urUSMSharedAlloc( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] handle of the device object - ur_device_handle_t Device, + ::ur_device_handle_t DeviceOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *USMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t Pool, + ::ur_usm_pool_handle_t PoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t Size, /// [out] pointer to USM shared memory object void **RetMem) { + auto Context = v1_cast(ContextOpque); ur_usm_pool_handle_t UrPool = nullptr; - if (Pool) { - UrPool = Pool; + if (PoolOpque) { + UrPool = v1_cast(PoolOpque); } else { UrPool = &Context->DefaultPool; } - return UrPool->allocate(Context, Device, USMDesc, UR_USM_TYPE_SHARED, Size, - RetMem); + return UrPool->allocate(Context, common_cast(DeviceOpque), USMDesc, + UR_USM_TYPE_SHARED, Size, RetMem); } ur_result_t /// [in] handle of the context object -urUSMFree(ur_context_handle_t Context, +urUSMFree(::ur_context_handle_t ContextOpque, /// [in] pointer to USM memory object void *Mem) { + auto Context = v1_cast(ContextOpque); ur_platform_handle_t Plt = Context->getPlatform(); std::scoped_lock Lock( @@ -383,7 +375,7 @@ urUSMFree(ur_context_handle_t Context, ur_result_t urUSMGetMemAllocInfo( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] pointer to USM memory object const void *Ptr, /// [in] the name of the USM allocation property to query @@ -394,6 +386,7 @@ ur_result_t urUSMGetMemAllocInfo( void *PropValue, /// [out][optional] bytes returned in USM allocation property size_t *PropValueSizeRet) { + auto Context = v1_cast(ContextOpque); ze_device_handle_t ZeDeviceHandle; ZeStruct ZeMemoryAllocationProperties; @@ -468,16 +461,17 @@ ur_result_t urUSMGetMemAllocInfo( ur_result_t urUSMPoolCreate( /// [in] handle of the context object - ur_context_handle_t Context, + ::ur_context_handle_t ContextOpque, /// [in] pointer to USM pool descriptor. Can be chained with /// ::ur_usm_pool_limits_desc_t ur_usm_pool_desc_t *PoolDesc, /// [out] pointer to USM memory pool - ur_usm_pool_handle_t *Pool) { + ::ur_usm_pool_handle_t *PoolOpque) { + auto Context = v1_cast(ContextOpque); + auto Pool = v1_cast(PoolOpque); try { - *Pool = reinterpret_cast( - new ur_usm_pool_handle_t_(Context, PoolDesc)); + *Pool = new v1::ur_usm_pool_handle_t_(Context, PoolDesc); std::scoped_lock ContextLock(Context->Mutex); Context->UsmPoolHandles.insert(Context->UsmPoolHandles.cend(), *Pool); @@ -494,14 +488,15 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool -urUSMPoolRetain(ur_usm_pool_handle_t Pool) { - Pool->RefCount.retain(); +urUSMPoolRetain(::ur_usm_pool_handle_t PoolOpque) { + v1_cast(PoolOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t /// [in] pointer to USM memory pool -urUSMPoolRelease(ur_usm_pool_handle_t Pool) { +urUSMPoolRelease(::ur_usm_pool_handle_t PoolOpque) { + auto Pool = v1_cast(PoolOpque); if (Pool->RefCount.release()) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); @@ -512,7 +507,7 @@ urUSMPoolRelease(ur_usm_pool_handle_t Pool) { ur_result_t urUSMPoolGetInfo( /// [in] handle of the USM memory pool - ur_usm_pool_handle_t Pool, + ::ur_usm_pool_handle_t PoolOpque, /// [in] name of the pool property to query ur_usm_pool_info_t PropName, /// [in] size in bytes of the pool property value provided @@ -521,6 +516,7 @@ ur_result_t urUSMPoolGetInfo( void *PropValue, /// [out] size in bytes returned in pool property value size_t *PropSizeRet) { + auto Pool = v1_cast(PoolOpque); UrReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet); switch (PropName) { @@ -536,9 +532,10 @@ ur_result_t urUSMPoolGetInfo( } } -ur_result_t urUSMImportExp(ur_context_handle_t Context, void *HostPtr, +ur_result_t urUSMImportExp(::ur_context_handle_t ContextOpque, void *HostPtr, size_t Size) { - UR_ASSERT(Context, UR_RESULT_ERROR_INVALID_CONTEXT); + UR_ASSERT(ContextOpque, UR_RESULT_ERROR_INVALID_CONTEXT); + auto Context = v1_cast(ContextOpque); // Promote the host ptr to USM host memory. if (ZeUSMImport.Supported && HostPtr != nullptr) { @@ -560,8 +557,9 @@ ur_result_t urUSMImportExp(ur_context_handle_t Context, void *HostPtr, return UR_RESULT_SUCCESS; } -ur_result_t urUSMReleaseExp(ur_context_handle_t Context, void *HostPtr) { - UR_ASSERT(Context, UR_RESULT_ERROR_INVALID_CONTEXT); +ur_result_t urUSMReleaseExp(::ur_context_handle_t ContextOpque, void *HostPtr) { + UR_ASSERT(ContextOpque, UR_RESULT_ERROR_INVALID_CONTEXT); + auto Context = v1_cast(ContextOpque); // Release the imported memory. if (ZeUSMImport.Supported && HostPtr != nullptr) @@ -571,19 +569,21 @@ ur_result_t urUSMReleaseExp(ur_context_handle_t Context, void *HostPtr) { } ur_result_t UR_APICALL urUSMPoolCreateExp( - ur_context_handle_t Context, ///< [in] handle of the context object - ur_device_handle_t Device, ///< [in] handle of the device object - ur_usm_pool_desc_t *PoolDesc, ///< [in] pointer to USM pool descriptor. - ///< Can be chained with - ///< ::ur_usm_pool_limits_desc_t - ur_usm_pool_handle_t *Pool ///< [out] pointer to USM memory pool + ::ur_context_handle_t ContextOpque, ///< [in] handle of the context object + ::ur_device_handle_t DeviceOpque, ///< [in] handle of the device object + ur_usm_pool_desc_t *PoolDesc, ///< [in] pointer to USM pool descriptor. + ///< Can be chained with + ///< ::ur_usm_pool_limits_desc_t + ::ur_usm_pool_handle_t *PoolOpque ///< [out] pointer to USM memory pool ) { + auto Context = v1_cast(ContextOpque); + auto Device = common_cast(DeviceOpque); try { - *Pool = reinterpret_cast( - new ur_usm_pool_handle_t_(Context, Device, PoolDesc)); + auto Pool = new v1::ur_usm_pool_handle_t_(Context, Device, PoolDesc); + *PoolOpque = v1_cast(Pool); std::scoped_lock ContextLock(Context->Mutex); - Context->UsmPoolHandles.insert(Context->UsmPoolHandles.cend(), *Pool); + Context->UsmPoolHandles.insert(Context->UsmPoolHandles.cend(), Pool); } catch (const UsmAllocationException &Ex) { return Ex.getError(); @@ -596,9 +596,10 @@ ur_result_t UR_APICALL urUSMPoolCreateExp( return UR_RESULT_SUCCESS; } -ur_result_t UR_APICALL urUSMPoolDestroyExp(ur_context_handle_t /*Context*/, - ur_device_handle_t /*Device*/, - ur_usm_pool_handle_t Pool) { +ur_result_t UR_APICALL urUSMPoolDestroyExp(::ur_context_handle_t /*Context*/, + ::ur_device_handle_t /*Device*/, + ::ur_usm_pool_handle_t PoolOpque) { + auto Pool = v1_cast(PoolOpque); std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -606,7 +607,7 @@ ur_result_t UR_APICALL urUSMPoolDestroyExp(ur_context_handle_t /*Context*/, return UR_RESULT_SUCCESS; } -ur_result_t UR_APICALL urUSMPoolSetInfoExp(ur_usm_pool_handle_t /*Pool*/, +ur_result_t UR_APICALL urUSMPoolSetInfoExp(::ur_usm_pool_handle_t /*Pool*/, ur_usm_pool_info_t PropName, void * /*PropValue*/, size_t PropSize) { @@ -630,20 +631,21 @@ ur_result_t UR_APICALL urUSMPoolSetInfoExp(ur_usm_pool_handle_t /*Pool*/, } ur_result_t UR_APICALL urUSMPoolGetDefaultDevicePoolExp( - ur_context_handle_t Context, ur_device_handle_t /*Device*/, - ur_usm_pool_handle_t *Pool) { - + ::ur_context_handle_t ContextOpque, ::ur_device_handle_t /*Device*/, + ::ur_usm_pool_handle_t *PoolOpque) { + auto Context = v1_cast(ContextOpque); // Default async pool should contain an internal pool for all detected // devices. - *Pool = &Context->AsyncPool; + *PoolOpque = v1_cast(&Context->AsyncPool); return UR_RESULT_SUCCESS; } -ur_result_t UR_APICALL urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, +ur_result_t UR_APICALL urUSMPoolGetInfoExp(::ur_usm_pool_handle_t hPoolOpque, ur_usm_pool_info_t PropName, void *PropValue, size_t *PropSizeRet) { + auto hPool = v1_cast(hPoolOpque); size_t value = 0; switch (PropName) { case UR_USM_POOL_INFO_RELEASE_THRESHOLD_EXP: @@ -678,27 +680,28 @@ ur_result_t UR_APICALL urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, return UR_RESULT_SUCCESS; } -ur_result_t UR_APICALL urUSMPoolGetDevicePoolExp(ur_context_handle_t, - ur_device_handle_t, - ur_usm_pool_handle_t *) { +ur_result_t UR_APICALL urUSMPoolGetDevicePoolExp(::ur_context_handle_t, + ::ur_device_handle_t, + ::ur_usm_pool_handle_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMPoolSetDevicePoolExp(ur_context_handle_t, - ur_device_handle_t, - ur_usm_pool_handle_t) { +ur_result_t UR_APICALL urUSMPoolSetDevicePoolExp(::ur_context_handle_t, + ::ur_device_handle_t, + ::ur_usm_pool_handle_t) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMPoolTrimToExp(ur_context_handle_t, - ur_device_handle_t, - ur_usm_pool_handle_t, size_t) { +ur_result_t UR_APICALL urUSMPoolTrimToExp(::ur_context_handle_t, + ::ur_device_handle_t, + ::ur_usm_pool_handle_t, size_t) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMContextMemcpyExp(ur_context_handle_t Context, +ur_result_t UR_APICALL urUSMContextMemcpyExp(::ur_context_handle_t ContextOpque, void *pDst, const void *pSrc, size_t Size) { + auto Context = v1_cast(ContextOpque); // zeCommandListAppendMemoryCopy must not be called from simultaneous // threads with the same command list handle, so we need exclusive lock. std::scoped_lock Lock(Context->ImmediateCommandListMutex); @@ -708,18 +711,16 @@ ur_result_t UR_APICALL urUSMContextMemcpyExp(ur_context_handle_t Context, } ur_result_t UR_APICALL urUSMHostAllocRegisterExp( - ur_context_handle_t /*hContext*/, void * /*pHostMem*/, size_t /*size*/, + ::ur_context_handle_t /*hContext*/, void * /*pHostMem*/, size_t /*size*/, const ur_exp_usm_host_alloc_register_properties_t * /*pProperties*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ur_result_t UR_APICALL urUSMHostAllocUnregisterExp( - ur_context_handle_t /*hContext*/, void * /*pHostMem*/) { + ::ur_context_handle_t /*hContext*/, void * /*pHostMem*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero - static ur_result_t USMFreeImpl(ur_context_handle_t Context, void *Ptr) { ur_result_t Res = UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { @@ -744,8 +745,9 @@ static ur_result_t USMQueryPageSize(ur_context_handle_t Context, void *Ptr, return UR_RESULT_SUCCESS; } -umf_result_t L0MemoryProvider::initialize(ur_context_handle_t Ctx, - ur_device_handle_t Dev) { +umf_result_t +L0MemoryProvider::initialize(ur_context_handle_t Ctx, + ur::level_zero::ur_device_handle_t Dev) { Context = Ctx; Device = Dev; @@ -958,7 +960,8 @@ umf_result_t L0MemoryProvider::ext_ctl(umf_ctl_query_source_t /*Source*/, ur_result_t L0SharedMemoryProvider::allocateImpl(void **ResultPtr, size_t Size, uint32_t Alignment) { - return USMSharedAllocImpl(ResultPtr, Context, Device, /*host flags*/ 0, + return USMSharedAllocImpl(ResultPtr, Context, Device, + /*host flags*/ 0, /*device flags*/ 0, Size, Alignment); } @@ -1008,26 +1011,28 @@ MakeProvider(ProviderParams *Params = nullptr) { umf_result_t Ret = UMF_RESULT_SUCCESS; umf::provider_unique_handle_t &&L0Provider = nullptr; + auto Context = v1_cast(Params->hContext); + auto Device = common_cast(Params->hDevice); + switch (Params->type) { case UR_USM_TYPE_HOST: std::tie(Ret, L0Provider) = - umf::memoryProviderMakeUnique(Params->hContext, - Params->hDevice); + umf::memoryProviderMakeUnique(Context, Device); break; case UR_USM_TYPE_DEVICE: std::tie(Ret, L0Provider) = - umf::memoryProviderMakeUnique( - Params->hContext, Params->hDevice); + umf::memoryProviderMakeUnique(Context, + Device); break; case UR_USM_TYPE_SHARED: if (Params->deviceReadOnly) { std::tie(Ret, L0Provider) = umf::memoryProviderMakeUnique( - Params->hContext, Params->hDevice); + Context, Device); } else { std::tie(Ret, L0Provider) = - umf::memoryProviderMakeUnique( - Params->hContext, Params->hDevice); + umf::memoryProviderMakeUnique(Context, + Device); } break; default: @@ -1049,14 +1054,16 @@ MakeProvider(ProviderParams *Params = nullptr) { UsmPool::UsmPool(ur_usm_pool_handle_t UrPool, umf::pool_unique_handle_t UmfPool) : UrPool(UrPool), UmfPool(std::move(UmfPool)), AsyncPool( - [](ur_event_handle_t Event) { return urEventReleaseInternal(Event); }, + [](::ur_event_handle_t Event) { + return urEventReleaseInternal(v1_cast(Event)); + }, [Context = UrPool->Context](void *Ptr) { return USMFreeHelper(Context, Ptr); }) {} -ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, - ur_usm_pool_desc_t *PoolDesc, - bool IsProxy) +v1::ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, + ur_usm_pool_desc_t *PoolDesc, + bool IsProxy) : Context(Context) { // TODO: handle zero-init flag 'UR_USM_POOL_FLAG_ZERO_INITIALIZE_BLOCK' auto DisjointPoolConfigs = DisjointPoolConfigInstance; @@ -1068,9 +1075,10 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, } auto DevicesAndSubDevices = - CollectDevicesForUsmPoolCreation(Context->Devices); + CollectDevicesForUsmPoolCreation(Context->getDevices()); auto Descriptors = usm::pool_descriptor::createFromDevices( - this, Context, DevicesAndSubDevices); + reinterpret_cast<::ur_usm_pool_handle_t>(this), v1_cast(Context), + common_cast(DevicesAndSubDevices)); for (auto &Desc : Descriptors) { umf::pool_unique_handle_t Pool = nullptr; if (IsProxy) { @@ -1098,9 +1106,9 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, } } -ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, - ur_device_handle_t Device, - ur_usm_pool_desc_t *PoolDesc) +v1::ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_usm_pool_desc_t *PoolDesc) : Context(Context) { // TODO: handle zero-init flag 'UR_USM_POOL_FLAG_ZERO_INITIALIZE_BLOCK' auto DisjointPoolConfigs = DisjointPoolConfigInstance; @@ -1111,28 +1119,31 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, } } + auto PoolOpque = v1_cast(this); + auto ContextOpque = v1_cast(Context); + auto DeviceOpque = common_cast(Device); // Create pool descriptor for single device provided std::vector Descriptors; { auto &Desc = Descriptors.emplace_back(); - Desc.poolHandle = this; - Desc.hContext = Context; - Desc.hDevice = Device; + Desc.poolHandle = PoolOpque; + Desc.hContext = ContextOpque; + Desc.hDevice = DeviceOpque; Desc.type = UR_USM_TYPE_DEVICE; } { auto &Desc = Descriptors.emplace_back(); - Desc.poolHandle = this; - Desc.hContext = Context; - Desc.hDevice = Device; + Desc.poolHandle = PoolOpque; + Desc.hContext = ContextOpque; + Desc.hDevice = DeviceOpque; Desc.type = UR_USM_TYPE_SHARED; Desc.deviceReadOnly = false; } { auto &Desc = Descriptors.emplace_back(); - Desc.poolHandle = this; - Desc.hContext = Context; - Desc.hDevice = Device; + Desc.poolHandle = PoolOpque; + Desc.hContext = ContextOpque; + Desc.hDevice = DeviceOpque; Desc.type = UR_USM_TYPE_SHARED; Desc.deviceReadOnly = true; } @@ -1157,19 +1168,20 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t Context, } } -UsmPool *ur_usm_pool_handle_t_::getPool(const usm::pool_descriptor &Desc) { +UsmPool *v1::ur_usm_pool_handle_t_::getPool(const usm::pool_descriptor &Desc) { auto PoolOpt = PoolManager.getPool(Desc); return PoolOpt.has_value() ? PoolOpt.value() : nullptr; } std::optional> -ur_usm_pool_handle_t_::allocateEnqueued(ur_queue_handle_t Queue, - ur_device_handle_t Device, - ur_usm_type_t Type, size_t Size) { +v1::ur_usm_pool_handle_t_::allocateEnqueued(ur_queue_handle_t Queue, + ur_device_handle_t Device, + ur_usm_type_t Type, size_t Size) { bool DeviceReadOnly = false; - auto *Pool = getPool( - usm::pool_descriptor{this, Queue->Context, Device, Type, DeviceReadOnly}); + auto *Pool = getPool(usm::pool_descriptor{ + reinterpret_cast<::ur_usm_pool_handle_t>(this), v1_cast(Queue->Context), + common_cast(Device), Type, DeviceReadOnly}); if (!Pool) { return std::nullopt; } @@ -1180,7 +1192,7 @@ ur_usm_pool_handle_t_::allocateEnqueued(ur_queue_handle_t Queue, return std::nullopt; } - auto *Event = Allocation->Event; + auto *Event = v1_cast(Allocation->Event); if (Event->Completed || (Allocation->Queue == Queue && Queue->isInOrderQueue())) { urEventReleaseInternal(Event); @@ -1190,11 +1202,11 @@ ur_usm_pool_handle_t_::allocateEnqueued(ur_queue_handle_t Queue, return std::make_pair(Allocation->Ptr, Event); } -ur_result_t ur_usm_pool_handle_t_::allocate(ur_context_handle_t Context, - ur_device_handle_t Device, - const ur_usm_desc_t *USMDesc, - ur_usm_type_t Type, size_t Size, - void **RetMem) { +ur_result_t v1::ur_usm_pool_handle_t_::allocate(ur_context_handle_t Context, + ur_device_handle_t Device, + const ur_usm_desc_t *USMDesc, + ur_usm_type_t Type, size_t Size, + void **RetMem) { uint32_t Alignment = USMDesc ? USMDesc->align : 0; // L0 supports alignment up to 64KB and silently ignores higher values. // We flag alignment > 64KB as an invalid value. @@ -1231,13 +1243,14 @@ ur_result_t ur_usm_pool_handle_t_::allocate(ur_context_handle_t Context, // We are going to defer memory release if there are kernels with indirect // access, that is why explicitly retain context to be sure that it is // released after all memory allocations in this context are released. - UR_CALL(ur::level_zero::urContextRetain(Context)); + UR_CALL(ur::level_zero::v1::urContextRetain(v1_cast(Context))); } else { ContextLock.lock(); } - auto *Pool = getPool( - usm::pool_descriptor{this, Context, Device, Type, DeviceReadOnly}); + auto *Pool = getPool(usm::pool_descriptor{ + reinterpret_cast<::ur_usm_pool_handle_t>(this), v1_cast(Context), + common_cast(Device), Type, DeviceReadOnly}); if (!Pool) { return UR_RESULT_ERROR_INVALID_ARGUMENT; } @@ -1270,8 +1283,8 @@ ur_result_t ur_usm_pool_handle_t_::allocate(ur_context_handle_t Context, return UR_RESULT_SUCCESS; } -ur_result_t ur_usm_pool_handle_t_::free(void *Mem, - umf_memory_pool_handle_t UmfPool) { +ur_result_t v1::ur_usm_pool_handle_t_::free(void *Mem, + umf_memory_pool_handle_t UmfPool) { size_t Size = 0; auto UmfRet = umfPoolMallocUsableSize(UmfPool, Mem, &Size); if (UmfRet != UMF_RESULT_SUCCESS && @@ -1289,8 +1302,8 @@ ur_result_t ur_usm_pool_handle_t_::free(void *Mem, return UR_RESULT_SUCCESS; } -UsmPool * -ur_usm_pool_handle_t_::getPoolByHandle(const umf_memory_pool_handle_t UmfPool) { +UsmPool *v1::ur_usm_pool_handle_t_::getPoolByHandle( + const umf_memory_pool_handle_t UmfPool) { UsmPool *Pool = nullptr; PoolManager.forEachPool([&](UsmPool *p) { if (p->UmfPool.get() == UmfPool) { @@ -1302,21 +1315,21 @@ ur_usm_pool_handle_t_::getPoolByHandle(const umf_memory_pool_handle_t UmfPool) { return Pool; } -void ur_usm_pool_handle_t_::cleanupPools() { +void v1::ur_usm_pool_handle_t_::cleanupPools() { PoolManager.forEachPool([&](UsmPool *p) { p->AsyncPool.cleanup(); return true; }); } -void ur_usm_pool_handle_t_::cleanupPoolsForQueue(ur_queue_handle_t Queue) { +void v1::ur_usm_pool_handle_t_::cleanupPoolsForQueue(ur_queue_handle_t Queue) { PoolManager.forEachPool([&](UsmPool *p) { p->AsyncPool.cleanupForQueue(Queue); return true; }); } -size_t ur_usm_pool_handle_t_::getTotalReservedSize() { +size_t v1::ur_usm_pool_handle_t_::getTotalReservedSize() { size_t TotalAllocatedSize = 0; umf_result_t UmfRet = UMF_RESULT_SUCCESS; PoolManager.forEachPool([&](UsmPool *p) { @@ -1340,7 +1353,7 @@ size_t ur_usm_pool_handle_t_::getTotalReservedSize() { return UmfRet == UMF_RESULT_SUCCESS ? TotalAllocatedSize : 0; } -size_t ur_usm_pool_handle_t_::getPeakReservedSize() { +size_t v1::ur_usm_pool_handle_t_::getPeakReservedSize() { size_t MaxPeakSize = 0; umf_result_t Ret = UMF_RESULT_SUCCESS; PoolManager.forEachPool([&](UsmPool *p) { @@ -1364,13 +1377,15 @@ size_t ur_usm_pool_handle_t_::getPeakReservedSize() { return Ret == UMF_RESULT_SUCCESS ? MaxPeakSize : 0; } -size_t ur_usm_pool_handle_t_::getTotalUsedSize() { +size_t v1::ur_usm_pool_handle_t_::getTotalUsedSize() { return AllocStats.getCurrent(); } -size_t ur_usm_pool_handle_t_::getPeakUsedSize() { return AllocStats.getPeak(); } +size_t v1::ur_usm_pool_handle_t_::getPeakUsedSize() { + return AllocStats.getPeak(); +} -bool ur_usm_pool_handle_t_::hasPool(const umf_memory_pool_handle_t Pool) { +bool v1::ur_usm_pool_handle_t_::hasPool(const umf_memory_pool_handle_t Pool) { return getPoolByHandle(Pool) != nullptr; } @@ -1466,3 +1481,5 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, } return Ret; } + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/usm.hpp b/unified-runtime/source/adapters/level_zero/usm.hpp index 865bea9a558d7..79f32d230d260 100644 --- a/unified-runtime/source/adapters/level_zero/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/usm.hpp @@ -9,15 +9,17 @@ #pragma once #include "common.hpp" +#include "common/enqueued_pool.hpp" #include "common/ur_ref_count.hpp" -#include "enqueued_pool.hpp" #include "event.hpp" #include "unified-runtime/ur_api.h" #include "ur_pool_manager.hpp" #include "usm.hpp" #include -usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig(); +namespace ur::level_zero::v1 { + +extern usm::DisjointPoolAllConfigs DisjointPoolConfigInstance; struct UsmPool { UsmPool(ur_usm_pool_handle_t UrPool, umf::pool_unique_handle_t UmfPool); @@ -57,7 +59,7 @@ struct AllocationStats { std::atomic_size_t PeakAllocatedMemorySize{0}; }; -struct ur_usm_pool_handle_t_ : ur_object { +struct ur_usm_pool_handle_t_ : ur_object_t { ur_usm_pool_handle_t_(ur_context_handle_t Context, ur_usm_pool_desc_t *PoolDesc, bool IsProxy = false); ur_usm_pool_handle_t_(ur_context_handle_t Context, ur_device_handle_t Device, @@ -103,7 +105,7 @@ class UsmAllocationException { class USMMemoryProviderBase { protected: ur_context_handle_t Context; - ur_device_handle_t Device; + ur::level_zero::ur_device_handle_t Device; ur_result_t &getLastStatusRef() { static thread_local ur_result_t LastStatus = UR_RESULT_SUCCESS; @@ -120,7 +122,8 @@ class USMMemoryProviderBase { *ErrCode = static_cast(getLastStatusRef()); return UMF_RESULT_SUCCESS; }; - virtual umf_result_t initialize(ur_context_handle_t, ur_device_handle_t) { + virtual umf_result_t initialize(ur_context_handle_t, + ur::level_zero::ur_device_handle_t) { return UMF_RESULT_ERROR_NOT_SUPPORTED; }; virtual umf_result_t alloc(size_t, size_t, void **) { @@ -171,7 +174,7 @@ class L0MemoryProvider : public USMMemoryProviderBase { public: umf_result_t initialize(ur_context_handle_t Ctx, - ur_device_handle_t Dev) override; + ur::level_zero::ur_device_handle_t Dev) override; umf_result_t alloc(size_t Size, size_t Align, void **Ptr) override; umf_result_t free(void *Ptr, size_t Size) override; umf_result_t get_min_page_size(const void *, size_t *) override; @@ -273,3 +276,5 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, bool OwnZeMemHandle = true); extern const bool UseUSMAllocator; + +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/usm_p2p.cpp b/unified-runtime/source/adapters/level_zero/usm_p2p.cpp index 7b094d42c1f81..c644ff6e56ed3 100644 --- a/unified-runtime/source/adapters/level_zero/usm_p2p.cpp +++ b/unified-runtime/source/adapters/level_zero/usm_p2p.cpp @@ -10,10 +10,10 @@ #include "logger/ur_logger.hpp" #include "ur_level_zero.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v1 { -ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice) { +ur_result_t urUsmP2PEnablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice) { UR_LOG(INFO, "ignored enabling peer access from {} to memory of {}, because P2P is " @@ -22,8 +22,8 @@ ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, return UR_RESULT_SUCCESS; } -ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice) { +ur_result_t urUsmP2PDisablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice) { UR_LOG(INFO, "ignored disabling peer access from {} to memory of {}, because P2P " @@ -32,11 +32,13 @@ ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, return UR_RESULT_SUCCESS; } -ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice, - ur_exp_peer_info_t propName, - size_t propSize, void *pPropValue, - size_t *pPropSizeRet) { +ur_result_t +urUsmP2PPeerAccessGetInfoExp(::ur_device_handle_t commandDeviceOpque, + ::ur_device_handle_t peerDeviceOpque, + ur_exp_peer_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + auto commandDevice = common_cast(commandDeviceOpque); + auto peerDevice = common_cast(peerDeviceOpque); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); @@ -71,4 +73,4 @@ ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, return ReturnValue(propertyValue); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v1 diff --git a/unified-runtime/source/adapters/level_zero/v2/adapter.cpp b/unified-runtime/source/adapters/level_zero/v2/adapter.cpp new file mode 100644 index 0000000000000..3290b9f792540 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/adapter.cpp @@ -0,0 +1,98 @@ +//===--------- adapter.cpp - Level Zero Adapter v2 -----------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include "../common/adapter.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { + +ur_result_t urAdapterGet(uint32_t NumEntries, ::ur_adapter_handle_t *Adapters, + uint32_t *NumAdapters) try { + std::lock_guard Lock(GlobalAdapterMutex); + + // L0v1 adapter may have already been selected. Return adapter only if it + // carries our DDI. If the GlobalAdapter is not set, then create a new one + // and return it. + if (GlobalAdapter) { + if (GlobalAdapter->ddi_table != ur::level_zero::v2::ddi_getter::value()) { + if (NumAdapters) + *NumAdapters = 0; + return UR_RESULT_SUCCESS; + } + if (NumEntries && Adapters) { + *Adapters = common_cast(GlobalAdapter); + GlobalAdapter->RefCount.retain(); + } + if (NumAdapters) + *NumAdapters = 1; + return UR_RESULT_SUCCESS; + } + + auto *adapter = new ur::level_zero::ur_adapter_handle_t_(); + if (adapter->Platforms.empty()) { + delete adapter; + + releaseStaticLoaderResourcesIfNeeded(); + if (NumAdapters) + *NumAdapters = 0; + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + adapter->version = 2; + + if (!ur_getenv("UR_ADAPTERS_FORCE_LOAD").has_value()) { + auto [useV2, reason] = shouldUseV2Adapter(); + if (!useV2) { + UR_LOG(INFO, "Skipping L0 V2 adapter: {}", reason); + delete adapter; +#ifndef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 + releaseStaticLoaderResourcesIfNeeded(); +#endif + if (NumAdapters) + *NumAdapters = 0; + return UR_RESULT_SUCCESS; + } + } + + // Stamp L0v2 DDI table on the created adapter and its platforms. + adapter->ddi_table = ddi_getter::value(); + for (auto &platform : adapter->Platforms) + platform->ddi_table = adapter->ddi_table; + GlobalAdapter = adapter; + +#if defined(_WIN32) + // On Windows the adapter library is torn down before the UR loader, + // so register an atexit cleanup that frees GlobalAdapter ourselves. + std::atexit(ur::level_zero::globalAdapterOnDemandCleanup); +#endif + + if (NumEntries && Adapters) { + *Adapters = common_cast(GlobalAdapter); + if (GlobalAdapter->RefCount.retain() == 0) { + adapterStateInit(); + } + } + + if (NumAdapters) + *NumAdapters = 1; + + return UR_RESULT_SUCCESS; +} catch (ur_result_t result) { + return result; +} catch (...) { + return UR_RESULT_ERROR_UNKNOWN; +} + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/api.cpp b/unified-runtime/source/adapters/level_zero/v2/api.cpp index 93b58c981cf02..4bf75014446d3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/api.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/api.cpp @@ -14,52 +14,53 @@ #include "../common.hpp" #include "logger/ur_logger.hpp" -std::mutex ZeCall::GlobalLock; - -namespace ur::level_zero { +namespace ur::level_zero::v2 { ur_result_t -urContextSetExtendedDeleter(ur_context_handle_t hContext, - ur_context_extended_deleter_t pfnDeleter, - void *pUserData) { +urContextSetExtendedDeleter(::ur_context_handle_t /*hContextOpque*/, + ur_context_extended_deleter_t /*pfnDeleter*/, + void * /*pUserData*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ur_result_t urKernelSetSpecializationConstants( - ur_kernel_handle_t hKernel, uint32_t count, - const ur_specialization_constant_info_t *pSpecConstants) { + ::ur_kernel_handle_t /*hKernelOpque*/, uint32_t /*count*/, + const ur_specialization_constant_info_t * /*pSpecConstants*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urEventSetCallback(ur_event_handle_t hEvent, - ur_execution_info_t execStatus, - ur_event_callback_t pfnNotify, void *pUserData) { +ur_result_t urEventSetCallback(::ur_event_handle_t /*hEventOpque*/, + ur_execution_info_t /*execStatus*/, + ur_event_callback_t /*pfnNotify*/, + void * /*pUserData*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMPoolGetDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t *pPool) { +ur_result_t UR_APICALL +urUSMPoolGetDevicePoolExp(::ur_context_handle_t /*hContextOpque*/, + ::ur_device_handle_t /*hDeviceOpque*/, + ::ur_usm_pool_handle_t * /*pPoolOpque*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMPoolSetDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t hPool) { +ur_result_t UR_APICALL +urUSMPoolSetDevicePoolExp(::ur_context_handle_t /*hContextOpque*/, + ::ur_device_handle_t /*hDeviceOpque*/, + ::ur_usm_pool_handle_t /*hPoolOpque*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t UR_APICALL urUSMPoolTrimToExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_usm_pool_handle_t hPool, - size_t minBytesToKeep) { +ur_result_t UR_APICALL urUSMPoolTrimToExp( + ::ur_context_handle_t /*hContextOpque*/, + ::ur_device_handle_t /*hDeviceOpque*/, + ::ur_usm_pool_handle_t /*hPoolOpque*/, size_t /*minBytesToKeep*/) { UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index 818f77416a57c..16abe29332c15 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -8,26 +8,30 @@ //===----------------------------------------------------------------------===// #include "command_buffer.hpp" -#include "../command_buffer_command.hpp" -#include "../helpers/kernel_helpers.hpp" -#include "../ur_interface_loader.hpp" +#include "../common/command_buffer_command.hpp" +#include "../common/device.hpp" +#include "../common/helpers/kernel_helpers.hpp" #include "command_list_manager.hpp" #include "logger/ur_logger.hpp" #include "queue_handle.hpp" +#include "ur_interface_loader.hpp" +namespace ur::level_zero::v2 { namespace { -ur_result_t getZeKernelWrapped(ur_kernel_handle_t kernel, +ur_result_t getZeKernelWrapped(::ur_kernel_handle_t kernelOpque, ze_kernel_handle_t &zeKernel, - ur_device_handle_t device) { - zeKernel = kernel->getZeHandle(device); + ur::level_zero::ur_device_handle_t device) { + zeKernel = v2_cast(kernelOpque)->getZeHandle(device); return UR_RESULT_SUCCESS; } -ur_result_t getMemPtr(ur_mem_handle_t memObj, +ur_result_t getMemPtr(::ur_mem_handle_t memObjOpque, const ur_kernel_arg_mem_obj_properties_t *properties, - char **&zeHandlePtr, ur_device_handle_t device, + char **&zeHandlePtr, + ur::level_zero::ur_device_handle_t device, device_ptr_storage_t *ptrStorage) { + auto memObj = v2_cast(memObjOpque); char *ptr; if (memObj->isImage()) { auto imageObj = memObj->getImage(); @@ -72,7 +76,7 @@ ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_( context, device, std::forward(commandList)), context(context), device(device), - eventPool(context->getEventPoolCache(PoolCacheType::Regular) + eventPool(context->getEventPoolCache(v2::PoolCacheType::Regular) .borrow(device->Id.value(), isInOrder ? v2::EVENT_FLAGS_COUNTER : 0)) {} @@ -106,6 +110,15 @@ ur_event_handle_t *ur_exp_command_buffer_handle_t_::getWaitListFromSyncPoints( return syncPointWaitList.data(); } +kernel_command_handle * +CreateKernelCommandHandle(::ur_exp_command_buffer_handle_t commandBuffer, + ::ur_kernel_handle_t kernel, uint64_t commandId, + uint32_t workDim, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *kernelAlternatives) { + return new kernel_command_handle(commandBuffer, kernel, commandId, workDim, + numKernelAlternatives, kernelAlternatives); +} + ur_result_t ur_exp_command_buffer_handle_t_::createCommandHandle( locked &commandListLocked, ur_kernel_handle_t hKernel, uint32_t workDim, const size_t *pGlobalWorkSize, @@ -116,11 +129,12 @@ ur_result_t ur_exp_command_buffer_handle_t_::createCommandHandle( ze_command_list_handle_t zeCommandList = commandListLocked->getZeCommandList(); std::unique_ptr newCommand; - UR_CALL(createCommandHandleUnlocked(this, zeCommandList, hKernel, workDim, - pGlobalWorkSize, numKernelAlternatives, - kernelAlternatives, platform, - getZeKernelWrapped, device, newCommand)); - *command = newCommand.get(); + UR_CALL(createCommandHandleUnlocked( + v2_cast(this), zeCommandList, v2_cast(hKernel), workDim, pGlobalWorkSize, + numKernelAlternatives, v2_cast(kernelAlternatives), platform, + getZeKernelWrapped, CreateKernelCommandHandle, device, newCommand)); + *command = reinterpret_cast( + newCommand.get()); commandHandles.push_back(std::move(newCommand)); return UR_RESULT_SUCCESS; @@ -193,7 +207,8 @@ ur_result_t ur_exp_command_buffer_handle_t_::applyUpdateCommands( return UR_RESULT_ERROR_INVALID_OPERATION; } UR_CALL(validateCommandDescUnlocked( - this, device, context->getPlatform()->ZeDriverGlobalOffsetExtensionFound, + v2_cast(this), device, + context->getPlatform()->ZeDriverGlobalOffsetExtensionFound, numUpdateCommands, updateCommands)); if (currentExecution) { @@ -233,12 +248,12 @@ ur_event_handle_t ur_exp_command_buffer_handle_t_::createEventIfRequested( return event; } -namespace ur::level_zero { - -ur_result_t -urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, - const ur_exp_command_buffer_desc_t *commandBufferDesc, - ur_exp_command_buffer_handle_t *commandBuffer) try { +ur_result_t urCommandBufferCreateExp( + ::ur_context_handle_t contextOpque, ::ur_device_handle_t deviceOpque, + const ur_exp_command_buffer_desc_t *commandBufferDesc, + ::ur_exp_command_buffer_handle_t *commandBufferOpque) try { + auto context = v2_cast(contextOpque); + auto device = common_cast(deviceOpque); checkImmediateAppendSupport(context); if (commandBufferDesc->isUpdatable && @@ -258,7 +273,7 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, context->getCommandListCache().getRegularCommandList(device->ZeDevice, listDesc); - *commandBuffer = new ur_exp_command_buffer_handle_t_( + *v2_cast(commandBufferOpque) = new ur_exp_command_buffer_handle_t_( context, device, std::move(zeCommandList), commandBufferDesc); return UR_RESULT_SUCCESS; @@ -266,16 +281,17 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, return exceptionToResult(std::current_exception()); } -ur_result_t -urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - hCommandBuffer->RefCount.retain(); +ur_result_t urCommandBufferRetainExp( + ::ur_exp_command_buffer_handle_t hCommandBufferOpque) try { + v2_cast(hCommandBufferOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t -urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { +ur_result_t urCommandBufferReleaseExp( + ::ur_exp_command_buffer_handle_t hCommandBufferOpque) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); if (!hCommandBuffer->RefCount.release()) return UR_RESULT_SUCCESS; @@ -289,8 +305,9 @@ urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { return exceptionToResult(std::current_exception()); } -ur_result_t -urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { +ur_result_t urCommandBufferFinalizeExp( + ::ur_exp_command_buffer_handle_t hCommandBufferOpque) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); UR_ASSERT(hCommandBuffer, UR_RESULT_ERROR_INVALID_NULL_POINTER); UR_CALL(hCommandBuffer->finalizeCommandBuffer()); return UR_RESULT_SUCCESS; @@ -299,17 +316,22 @@ urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { } ur_result_t urCommandBufferAppendKernelLaunchExp( - ur_exp_command_buffer_handle_t commandBuffer, ur_kernel_handle_t hKernel, - uint32_t workDim, const size_t *pGlobalWorkOffset, - const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, - uint32_t numKernelAlternatives, ur_kernel_handle_t *kernelAlternatives, + ::ur_exp_command_buffer_handle_t commandBufferOpque, + ::ur_kernel_handle_t hKernelOpque, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *kernelAlternativesOpque, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *syncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*eventWaitList*/, + const ::ur_event_handle_t * /*eventWaitList*/, ur_exp_command_buffer_sync_point_t *retSyncPoint, - ur_event_handle_t * /*event*/, - ur_exp_command_buffer_command_handle_t *command) try { + ::ur_event_handle_t * /*event*/, + ::ur_exp_command_buffer_command_handle_t *commandOpque) try { + auto commandBuffer = v2_cast(commandBufferOpque); + auto hKernel = v2_cast(hKernelOpque); + auto kernelAlternatives = v2_cast(kernelAlternativesOpque); + auto command = common_cast(commandOpque); if (command != nullptr && !commandBuffer->isUpdatable) { return UR_RESULT_ERROR_INVALID_OPERATION; @@ -342,18 +364,23 @@ ur_result_t urCommandBufferAppendKernelLaunchExp( } ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, - uint32_t workDim, const size_t *pGlobalWorkOffset, - const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, - uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, - uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_kernel_handle_t hKernelOpque, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternativesOpque, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /* numEventsInWaitList */, - const ur_event_handle_t * /* phEventWaitList */, + const ::ur_event_handle_t * /* phEventWaitList */, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /* phEvent */, - ur_exp_command_buffer_command_handle_t *phCommand) try { + ::ur_event_handle_t * /* phEvent */, + ::ur_exp_command_buffer_command_handle_t *phCommandOpque) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hKernel = v2_cast(hKernelOpque); + auto phKernelAlternatives = v2_cast(phKernelAlternativesOpque); + auto phCommand = common_cast(phCommandOpque); UR_ASSERT(hKernel, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(hKernel->getProgramHandle(), UR_RESULT_ERROR_INVALID_NULL_POINTER); @@ -391,14 +418,15 @@ ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( } ur_result_t urCommandBufferAppendUSMMemcpyExp( - ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, const void *pSrc, - size_t size, uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, void *pDst, + const void *pSrc, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); // Responsibility of UMD to offload to copy engine auto commandListLocked = hCommandBuffer->commandListManager.lock(); @@ -418,15 +446,19 @@ ur_result_t urCommandBufferAppendUSMMemcpyExp( } ur_result_t urCommandBufferAppendMemBufferCopyExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, - ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hSrcMemOpque, ::ur_mem_handle_t hDstMemOpque, + size_t srcOffset, size_t dstOffset, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hSrcMem = v2_cast(hSrcMemOpque); + auto hDstMem = v2_cast(hDstMemOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp // sync mechanic can be ignored, because all lists are in-order @@ -448,15 +480,17 @@ ur_result_t urCommandBufferAppendMemBufferCopyExp( } ur_result_t urCommandBufferAppendMemBufferWriteExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, - size_t offset, size_t size, const void *pSrc, - uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hBufferOpque, size_t offset, size_t size, + const void *pSrc, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hBuffer = v2_cast(hBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp // sync mechanic can be ignored, because all lists are in-order @@ -478,14 +512,17 @@ ur_result_t urCommandBufferAppendMemBufferWriteExp( } ur_result_t urCommandBufferAppendMemBufferReadExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, - size_t offset, size_t size, void *pDst, uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hBufferOpque, size_t offset, size_t size, void *pDst, + uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hBuffer = v2_cast(hBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp // Responsibility of UMD to offload to copy engine @@ -506,17 +543,20 @@ ur_result_t urCommandBufferAppendMemBufferReadExp( } ur_result_t urCommandBufferAppendMemBufferCopyRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, - ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin, - ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, - size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hSrcMemOpque, ::ur_mem_handle_t hDstMemOpque, + ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, + ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, + size_t dstRowPitch, size_t dstSlicePitch, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hSrcMem = v2_cast(hSrcMemOpque); + auto hDstMem = v2_cast(hDstMemOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp // sync mechanic can be ignored, because all lists are in-order @@ -539,17 +579,19 @@ ur_result_t urCommandBufferAppendMemBufferCopyRectExp( } ur_result_t urCommandBufferAppendMemBufferWriteRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, - ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, - ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, - size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, - uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hBufferOpque, ur_rect_offset_t bufferOffset, + ur_rect_offset_t hostOffset, ur_rect_region_t region, size_t bufferRowPitch, + size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, + void *pSrc, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hBuffer = v2_cast(hBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp @@ -572,17 +614,19 @@ ur_result_t urCommandBufferAppendMemBufferWriteRectExp( } ur_result_t urCommandBufferAppendMemBufferReadRectExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, - ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, - ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, - size_t hostRowPitch, size_t hostSlicePitch, void *pDst, - uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hBufferOpque, ur_rect_offset_t bufferOffset, + ur_rect_offset_t hostOffset, ur_rect_region_t region, size_t bufferRowPitch, + size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, + void *pDst, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hBuffer = v2_cast(hBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp @@ -605,15 +649,16 @@ ur_result_t urCommandBufferAppendMemBufferReadRectExp( } ur_result_t urCommandBufferAppendUSMFillExp( - ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, void *pMemory, const void *pPattern, size_t patternSize, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); auto commandListLocked = hCommandBuffer->commandListManager.lock(); auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints( @@ -631,15 +676,17 @@ ur_result_t urCommandBufferAppendUSMFillExp( } ur_result_t urCommandBufferAppendMemBufferFillExp( - ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, - const void *pPattern, size_t patternSize, size_t offset, size_t size, - uint32_t numSyncPointsInWaitList, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_mem_handle_t hBufferOpque, const void *pPattern, size_t patternSize, + size_t offset, size_t size, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); + auto hBuffer = v2_cast(hBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp auto commandListLocked = hCommandBuffer->commandListManager.lock(); @@ -659,15 +706,16 @@ ur_result_t urCommandBufferAppendMemBufferFillExp( } ur_result_t urCommandBufferAppendUSMPrefetchExp( - ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, const void *pMemory, size_t size, ur_usm_migration_flags_t flags, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp @@ -688,14 +736,15 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp( } ur_result_t urCommandBufferAppendUSMAdviseExp( - ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, const void *pMemory, size_t size, ur_usm_advice_flags_t advice, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, + const ::ur_event_handle_t * /*phEventWaitList*/, ur_exp_command_buffer_sync_point_t *pSyncPoint, - ur_event_handle_t * /*phEvent*/, - ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + ::ur_event_handle_t * /*phEvent*/, + ::ur_exp_command_buffer_command_handle_t * /*phCommand*/) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); // the same issue as in urCommandBufferAppendKernelLaunchExp auto commandListLocked = hCommandBuffer->commandListManager.lock(); @@ -715,10 +764,11 @@ ur_result_t urCommandBufferAppendUSMAdviseExp( } ur_result_t -urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, +urCommandBufferGetInfoExp(::ur_exp_command_buffer_handle_t hCommandBufferOpque, ur_exp_command_buffer_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) try { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { @@ -743,12 +793,13 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, } ur_result_t urCommandBufferAppendNativeCommandExp( - ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - void *pData, ur_exp_command_buffer_handle_t, + void *pData, ::ur_exp_command_buffer_handle_t, uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ur_exp_command_buffer_sync_point_t *pSyncPoint) { + auto hCommandBuffer = v2_cast(hCommandBufferOpque); // Barrier on all commands before user defined commands. auto commandListLocked = hCommandBuffer->commandListManager.lock(); @@ -772,11 +823,11 @@ ur_result_t urCommandBufferAppendNativeCommandExp( return UR_RESULT_SUCCESS; } -ur_result_t -urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, - ur_native_handle_t *phNativeCommandBuffer) { - - auto commandListLocked = hCommandBuffer->commandListManager.lock(); +ur_result_t urCommandBufferGetNativeHandleExp( + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + ::ur_native_handle_t *phNativeCommandBuffer) { + auto commandListLocked = + v2_cast(hCommandBufferOpque)->commandListManager.lock(); ze_command_list_handle_t ZeCommandList = commandListLocked->getZeCommandList(); *phNativeCommandBuffer = reinterpret_cast(ZeCommandList); @@ -784,32 +835,34 @@ urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, } ur_result_t urCommandBufferUpdateKernelLaunchExp( - ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numUpdateCommands, + ::ur_exp_command_buffer_handle_t hCommandBufferOpque, + uint32_t numUpdateCommands, const ur_exp_command_buffer_update_kernel_launch_desc_t *pUpdateKernelLaunch) { - UR_CALL(hCommandBuffer->applyUpdateCommands(numUpdateCommands, - pUpdateKernelLaunch)); + UR_CALL(v2_cast(hCommandBufferOpque) + ->applyUpdateCommands(numUpdateCommands, pUpdateKernelLaunch)); return UR_RESULT_SUCCESS; } ur_result_t urCommandBufferUpdateSignalEventExp( - ur_exp_command_buffer_command_handle_t hCommand, - ur_event_handle_t *phEvent) { + ::ur_exp_command_buffer_command_handle_t hCommandOpque, + ::ur_event_handle_t *phEventOpque) { // needs to be implemented together with signal event handling - (void)hCommand; - (void)phEvent; + (void)hCommandOpque; + (void)phEventOpque; return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ur_result_t urCommandBufferUpdateWaitEventsExp( - ur_exp_command_buffer_command_handle_t hCommand, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList) { + ::ur_exp_command_buffer_command_handle_t hCommandOpque, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitListOpque) { // needs to be implemented together with wait event handling - (void)hCommand; + (void)hCommandOpque; (void)numEventsInWaitList; - (void)phEventWaitList; + (void)phEventWaitListOpque; return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp index e1dbe1afd3b98..3a5005aca6a56 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp @@ -10,7 +10,7 @@ #include -#include "../helpers/mutable_helpers.hpp" +#include "../common/helpers/mutable_helpers.hpp" #include "command_list_manager.hpp" #include "common.hpp" #include "common/ur_ref_count.hpp" @@ -20,9 +20,9 @@ #include "queue_api.hpp" #include -struct kernel_command_handle; +namespace ur::level_zero::v2 { -struct ur_exp_command_buffer_handle_t_ : public ur_object { +struct ur_exp_command_buffer_handle_t_ : public v2::ur_object_t { ur_exp_command_buffer_handle_t_( ur_context_handle_t context, ur_device_handle_t device, v2::raii::command_list_unique_handle &&commandList, @@ -80,7 +80,8 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { const ur_context_handle_t context; const ur_device_handle_t device; - std::vector> + std::vector< + std::unique_ptr> commandHandles; // Indicates if command-buffer was finalized. @@ -90,3 +91,5 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { v2::raii::cache_borrowed_event_pool eventPool; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp index dbcb19df31cee..913eb02e7a6fc 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp @@ -10,7 +10,9 @@ #include "command_list_cache.hpp" #include "context.hpp" -#include "../device.hpp" +#include "../common/device.hpp" + +namespace ur::level_zero::v2 { // UR copy of the deprecated // zex_intel_queue_copy_operations_offload_hint_exp_desc_t @@ -40,25 +42,31 @@ typedef struct _ur_zex_intel_queue_copy_operations_offload_hint_exp_desc_t { static_cast(0x0003001B) #endif // ZEX_INTEL_STRUCTURE_TYPE_QUEUE_COPY_OPERATIONS_OFFLOAD_HINT_EXP_PROPERTIES +} // namespace ur::level_zero::v2 + +namespace ur::level_zero { template <> ze_structure_type_ext_t getZexStructureType< - ur_zex_intel_queue_copy_operations_offload_hint_exp_desc_t>() { + ur::level_zero::v2:: + ur_zex_intel_queue_copy_operations_offload_hint_exp_desc_t>() { return UR_ZEX_INTEL_STRUCTURE_TYPE_QUEUE_COPY_OPERATIONS_OFFLOAD_HINT_EXP_PROPERTIES; } +} // namespace ur::level_zero + +namespace ur::level_zero::v2 { -bool v2::immediate_command_list_descriptor_t::operator==( +bool immediate_command_list_descriptor_t::operator==( const immediate_command_list_descriptor_t &rhs) const { return ZeDevice == rhs.ZeDevice && IsInOrder == rhs.IsInOrder && Mode == rhs.Mode && Priority == rhs.Priority && Index == rhs.Index; } -bool v2::regular_command_list_descriptor_t::operator==( +bool regular_command_list_descriptor_t::operator==( const regular_command_list_descriptor_t &rhs) const { return ZeDevice == rhs.ZeDevice && Ordinal == rhs.Ordinal && IsInOrder == rhs.IsInOrder && Mutable == rhs.Mutable; } -namespace v2 { inline size_t command_list_descriptor_hash_t::operator()( const command_list_descriptor_t &desc) const { if (auto ImmCmdDesc = @@ -284,4 +292,4 @@ size_t command_list_cache_t::getNumRegularCommandLists() { return NumLists; } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.hpp b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.hpp index efee243272f0d..39a89f06cac5a 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.hpp @@ -18,7 +18,7 @@ #include "common.hpp" -namespace v2 { +namespace ur::level_zero::v2 { namespace raii { using command_list_unique_handle = std::unique_ptr<::_ze_command_list_handle_t, @@ -117,4 +117,4 @@ struct command_list_cache_t { raii::ze_command_list_handle_t createCommandList(const command_list_descriptor_t &desc); }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index aa33bf82966fc..de4defaad57cd 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -8,16 +8,18 @@ //===----------------------------------------------------------------------===// #include "command_list_manager.hpp" -#include "../helpers/kernel_helpers.hpp" -#include "../helpers/memory_helpers.hpp" -#include "../sampler.hpp" -#include "../ur_interface_loader.hpp" +#include "../common/helpers/kernel_helpers.hpp" +#include "../common/helpers/memory_helpers.hpp" +#include "../common/sampler.hpp" #include "command_buffer.hpp" #include "common.hpp" #include "context.hpp" #include "graph.hpp" #include "kernel.hpp" #include "memory.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { thread_local std::vector waitList; // The wait_list_view is a wrapper for eventsWaitLists, which: @@ -111,9 +113,9 @@ void wait_list_view::addEvent(ur_event_handle_t phEvent) { } ur_command_list_manager::ur_command_list_manager( - ur_context_handle_t context, ur_device_handle_t device, + ur_context_handle_t hContext, ur_device_handle_t hDevice, v2::raii::command_list_unique_handle &&commandList) - : hContext(context), hDevice(device), + : hContext(hContext), hDevice(hDevice), zeCommandList(std::move(commandList)) {} v2::raii::command_list_unique_handle && @@ -449,8 +451,9 @@ ur_result_t ur_command_list_manager::appendUSMAdvise( (zeCommandList.get(), numWaitEvents, pWaitEvents)); } - ZE2UR_CALL(zeCommandListAppendMemAdvise, - (zeCommandList.get(), hDevice->ZeDevice, pMem, size, zeAdvice)); + ZE2UR_CALL( + zeCommandListAppendMemAdvise, + (zeCommandList.get(), hDevice.get()->ZeDevice, pMem, size, zeAdvice)); if (zeSignalEvent) { ZE2UR_CALL(zeCommandListAppendSignalEvent, @@ -579,8 +582,9 @@ ur_result_t ur_command_list_manager::appendUSMMemcpy2D( ur_rect_offset_t zeroOffset{0, 0, 0}; ur_rect_region_t region{width, height, 0}; - ur_usm_handle_t srcHandle(hContext.get(), 0, pSrc); - ur_usm_handle_t dstHandle(hContext.get(), 0, pDst); + auto hContextInternal = hContext.get(); + ur_usm_handle_t srcHandle(hContextInternal, 0, pSrc); + ur_usm_handle_t dstHandle(hContextInternal, 0, pDst); return appendRegionCopyUnlocked(&srcHandle, &dstHandle, blocking, zeroOffset, zeroOffset, region, srcPitch, 0, dstPitch, 0, @@ -600,8 +604,7 @@ ur_result_t ur_command_list_manager::appendTimestampRecordingExp( phEvent->recordStartTimestamp(); - auto [timestampPtr, zeSignalEvent] = - (phEvent)->getEventEndTimestampAndHandle(); + auto [timestampPtr, zeSignalEvent] = phEvent->getEventEndTimestampAndHandle(); ZE2UR_CALL(zeCommandListAppendWriteGlobalTimestamp, (getZeCommandList(), timestampPtr, zeSignalEvent, numWaitEvents, @@ -825,7 +828,7 @@ ur_result_t ur_command_list_manager::appendDeviceGlobalVariableWrite( // TODO: make getZeModuleHandle thread-safe ze_module_handle_t zeModule = - hProgram->getZeModuleHandle(this->hDevice->ZeDevice); + hProgram->getZeModuleHandle(this->hDevice.get()->ZeDevice); // Find global variable pointer auto globalVarPtr = getGlobalPointerFromModule(zeModule, offset, count, name); @@ -844,7 +847,7 @@ ur_result_t ur_command_list_manager::appendDeviceGlobalVariableRead( // TODO: make getZeModuleHandle thread-safe ze_module_handle_t zeModule = - hProgram->getZeModuleHandle(this->hDevice->ZeDevice); + hProgram->getZeModuleHandle(this->hDevice.get()->ZeDevice); // Find global variable pointer auto globalVarPtr = getGlobalPointerFromModule(zeModule, offset, count, name); @@ -873,18 +876,19 @@ ur_result_t ur_command_list_manager::appendUSMAllocHelper( ur_queue_t_ *Queue, ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *, wait_list_view &waitListView, void **ppMem, ur_event_handle_t phEvent, ur_usm_type_t type) { + auto hContextInternal = hContext.get(); if (!pPool) { - pPool = hContext->getAsyncPool(); + pPool = hContextInternal->getAsyncPool(); } auto device = (type == UR_USM_TYPE_HOST) ? nullptr : hDevice.get(); ur_event_handle_t originAllocEvent = nullptr; auto asyncAlloc = pPool->allocateEnqueued( - hContext.get(), Queue, Queue->isInOrder(), device, type, size); + hContextInternal, Queue, Queue->isInOrder(), device, type, size); if (!asyncAlloc) { auto Ret = - pPool->allocate(hContext.get(), device, nullptr, type, size, ppMem); + pPool->allocate(hContextInternal, device, nullptr, type, size, ppMem); if (Ret) { return Ret; } @@ -942,7 +946,7 @@ ur_result_t ur_command_list_manager::appendUSMFreeExp( return UR_RESULT_ERROR_INVALID_MEM_OBJECT; } - UsmPool *usmPool = nullptr; + v2::UsmPool *usmPool = nullptr; umfRet = umfPoolGetTag(hPool, (void **)&usmPool); if (umfRet != UMF_RESULT_SUCCESS || !usmPool) { // This should never happen @@ -974,7 +978,7 @@ ur_result_t ur_command_list_manager::appendUSMFreeExp( } // Insert must be done after the signal event is appended. - usmPool->asyncPool.insert(pMem, size, phEvent, Queue); + usmPool->asyncPool.insert(pMem, size, v2_cast(phEvent), Queue); return UR_RESULT_SUCCESS; } @@ -1002,7 +1006,7 @@ ur_result_t ur_command_list_manager::bindlessImagesWaitExternalSemaphoreExp( ur_exp_external_semaphore_handle_t hSemaphore, bool hasWaitValue, uint64_t waitValue, wait_list_view &waitListView, ur_event_handle_t phEvent) { - auto hPlatform = hContext->getPlatform(); + auto hPlatform = hContext.get()->getPlatform(); if (hPlatform->ZeExternalSemaphoreExt.Supported == false) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not supported!"), @@ -1031,7 +1035,7 @@ ur_result_t ur_command_list_manager::bindlessImagesSignalExternalSemaphoreExp( ur_exp_external_semaphore_handle_t hSemaphore, bool hasSignalValue, uint64_t signalValue, wait_list_view &waitListView, ur_event_handle_t phEvent) { - auto hPlatform = hContext->getPlatform(); + auto hPlatform = hContext.get()->getPlatform(); if (hPlatform->ZeExternalSemaphoreExt.Supported == false) { UR_LOG_LEGACY(ERR, logger::LegacyMessage("[UR][L0] {} function not supported!"), @@ -1152,14 +1156,15 @@ ur_result_t ur_command_list_manager::appendKernelLaunchWithArgsExpOld( case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: // TODO: import helper for converting ur flags to internal equivalent UR_CALL(hKernel->addPendingMemoryAllocation( - {pArgs[argIndex].value.memObjTuple.hMem, + {v2_cast(pArgs[argIndex].value.memObjTuple.hMem), ur_mem_buffer_t::device_access_mode_t::read_write, pArgs[argIndex].index})); break; case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { - UR_CALL( - hKernel->setArgValue(hDevice, argIndex, sizeof(void *), nullptr, - &pArgs[argIndex].value.sampler->ZeSampler)); + UR_CALL(hKernel->setArgValue( + hDevice, argIndex, sizeof(void *), nullptr, + &ur::level_zero::common_cast(pArgs[argIndex].value.sampler) + ->ZeSampler)); break; } default: @@ -1261,13 +1266,15 @@ ur_result_t ur_command_list_manager::appendKernelLaunchWithArgsExpNew( // compute zePtr for the given memory handle and store it in // hKernel->kernelMemObj[argIndex] UR_CALL(hKernel->computeZePtr( - pArgs[argIndex].value.memObjTuple.hMem, hDevice.get(), + v2_cast(pArgs[argIndex].value.memObjTuple.hMem), hDevice.get(), ur_mem_buffer_t::device_access_mode_t::read_write, getZeCommandList(), waitListView, &hKernel->kernelMemObj[argIndex])); hKernel->kernelArgs[argIndex] = &hKernel->kernelMemObj[argIndex]; break; case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: - hKernel->kernelArgs[argIndex] = &pArgs[argIndex].value.sampler->ZeSampler; + hKernel->kernelArgs[argIndex] = + &ur::level_zero::common_cast(pArgs[argIndex].value.sampler) + ->ZeSampler; break; default: return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -1300,7 +1307,7 @@ ur_result_t ur_command_list_manager::appendKernelLaunchWithArgsExp( cooperativeKernelLaunchRequested = true; } - ur_platform_handle_t hPlatform = hContext->getPlatform(); + ur_platform_handle_t hPlatform = hContext.get()->getPlatform(); bool KernelWithArgsSupported = hPlatform->ZeCommandListAppendLaunchKernelWithArgumentsExt.Supported; bool CooperativeCompatible = @@ -1330,12 +1337,12 @@ ur_result_t ur_command_list_manager::appendKernelLaunchWithArgsExp( } ur_result_t ur_command_list_manager::beginGraphCapture() { - if (!checkGraphExtensionSupport(hContext.get())) { + auto hContextInternal = hContext.get(); + if (!checkGraphExtensionSupport(hContextInternal)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - ZE2UR_CALL(hContext.get() - ->getPlatform() + ZE2UR_CALL(hContextInternal->getPlatform() ->ZeGraphExt.zeCommandListBeginGraphCaptureExp, (getZeCommandList(), nullptr)); graphCapture.enableCapture(); @@ -1345,12 +1352,12 @@ ur_result_t ur_command_list_manager::beginGraphCapture() { ur_result_t ur_command_list_manager::beginCaptureIntoGraph(ur_exp_graph_handle_t hGraph) { - if (!checkGraphExtensionSupport(hContext.get())) { + auto hContextInternal = hContext.get(); + if (!checkGraphExtensionSupport(hContextInternal)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - ZE2UR_CALL(hContext.get() - ->getPlatform() + ZE2UR_CALL(hContextInternal->getPlatform() ->ZeGraphExt.zeCommandListBeginCaptureIntoGraphExp, (getZeCommandList(), hGraph->getZeHandle(), nullptr)); graphCapture.enableCapture(hGraph); @@ -1360,7 +1367,8 @@ ur_command_list_manager::beginCaptureIntoGraph(ur_exp_graph_handle_t hGraph) { ur_result_t ur_command_list_manager::endGraphCapture(ur_exp_graph_handle_t *phGraph) { - if (!checkGraphExtensionSupport(hContext.get())) { + auto hContextInternal = hContext.get(); + if (!checkGraphExtensionSupport(hContextInternal)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -1371,11 +1379,11 @@ ur_command_list_manager::endGraphCapture(ur_exp_graph_handle_t *phGraph) { graphCapture.disableCapture(); if (!graph) { - std::scoped_lock lock(hContext.get()->GraphMapMutex); - graph = hContext.get()->getGraphFromZeHandle(zeGraph); + std::scoped_lock lock(hContextInternal->GraphMapMutex); + graph = hContextInternal->getGraphFromZeHandle(zeGraph); if (!graph) { - graph = new ur_exp_graph_handle_t_(hContext.get(), zeGraph); - hContext.get()->registerGraph(zeGraph, graph); + graph = new ur_exp_graph_handle_t_(hContextInternal, zeGraph); + hContextInternal->registerGraph(zeGraph, graph); } } *phGraph = graph; @@ -1387,13 +1395,14 @@ ur_result_t ur_command_list_manager::appendGraph(ur_exp_executable_graph_handle_t hGraph, wait_list_view &waitListView, ur_event_handle_t hEvent) { - if (!checkGraphExtensionSupport(hContext.get())) { + auto hContextInternal = hContext.get(); + if (!checkGraphExtensionSupport(hContextInternal)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } auto zeSignalEvent = getSignalEvent(hEvent, UR_COMMAND_ENQUEUE_GRAPH_EXP); ZE2UR_CALL( - hContext.get()->getPlatform()->ZeGraphExt.zeCommandListAppendGraphExp, + hContextInternal->getPlatform()->ZeGraphExt.zeCommandListAppendGraphExp, (getZeCommandList(), hGraph->getZeHandle(), nullptr, zeSignalEvent, waitListView.num, waitListView.handles)); @@ -1401,7 +1410,8 @@ ur_command_list_manager::appendGraph(ur_exp_executable_graph_handle_t hGraph, } ur_result_t ur_command_list_manager::queryGraphCaptureActive(bool *pResult) { - if (!checkGraphExtensionSupport(hContext.get())) { + auto hContextInternal = hContext.get(); + if (!checkGraphExtensionSupport(hContextInternal)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -1415,9 +1425,10 @@ ur_result_t ur_command_list_manager::queryGraphCaptureActive(bool *pResult) { } ur_result_t ur_command_list_manager::getGraph(ur_exp_graph_handle_t *phGraph) { + auto hContextInternal = hContext.get(); auto zeGetGraph = - hContext.get()->getPlatform()->ZeGraphExt.zeCommandListGetGraphExp; - if (!checkGraphExtensionSupport(hContext.get()) || !zeGetGraph) { + hContextInternal->getPlatform()->ZeGraphExt.zeCommandListGetGraphExp; + if (!checkGraphExtensionSupport(hContextInternal) || !zeGetGraph) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -1439,11 +1450,11 @@ ur_result_t ur_command_list_manager::getGraph(ur_exp_graph_handle_t *phGraph) { ur_exp_graph_handle_t hUrGraph = nullptr; { - std::scoped_lock lock(hContext.get()->GraphMapMutex); - hUrGraph = hContext.get()->getGraphFromZeHandle(hZeGraph); + std::scoped_lock lock(hContextInternal->GraphMapMutex); + hUrGraph = hContextInternal->getGraphFromZeHandle(hZeGraph); if (!hUrGraph) { - hUrGraph = new ur_exp_graph_handle_t_(hContext.get(), hZeGraph); - hContext.get()->registerGraph(hZeGraph, hUrGraph); + hUrGraph = new ur_exp_graph_handle_t_(hContextInternal, hZeGraph); + hContextInternal->registerGraph(hZeGraph, hUrGraph); if (graphCapture.isActive()) { graphCapture.enableCapture(hUrGraph); } @@ -1459,7 +1470,7 @@ ur_result_t ur_command_list_manager::appendHostTaskExp( const ur_exp_host_task_properties_t *pProperties, wait_list_view &waitListView, ur_event_handle_t phEvent) { - ur_platform_handle_t hPlatform = hContext->getPlatform(); + ur_platform_handle_t hPlatform = hContext.get()->getPlatform(); if (!hPlatform->ZeHostTaskExt.Supported) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -1473,3 +1484,4 @@ ur_result_t ur_command_list_manager::appendHostTaskExp( return UR_RESULT_SUCCESS; } +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp index 34dff932c7eca..8ba0bd797256e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp @@ -17,6 +17,7 @@ #include #include +namespace ur::level_zero::v2 { struct ur_mem_buffer_t; struct wait_list_view { @@ -65,8 +66,8 @@ struct graph_capture_tracking_data_t { }; struct ur_command_list_manager { - ur_command_list_manager(ur_context_handle_t context, - ur_device_handle_t device, + ur_command_list_manager(ur_context_handle_t hContext, + ur_device_handle_t hDevice, v2::raii::command_list_unique_handle &&commandList); ur_command_list_manager(const ur_command_list_manager &src) = delete; ur_command_list_manager(ur_command_list_manager &&src) = default; @@ -338,3 +339,5 @@ struct ur_command_list_manager { graph_capture_tracking_data_t graphCapture; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/common.hpp b/unified-runtime/source/adapters/level_zero/v2/common.hpp index 489bea1dba987..07ea79ee20aac 100644 --- a/unified-runtime/source/adapters/level_zero/v2/common.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/common.hpp @@ -10,12 +10,157 @@ #pragma once #include +#include +#include #include -#include "../common.hpp" +#include "../common/device.hpp" +#include "../common/helpers/shared_helpers.hpp" #include "logger/ur_logger.hpp" +#include "ur_interface_loader.hpp" +#include + +namespace ur::level_zero::v2 { + +struct ur_object_t : ur::handle_base { + ur_shared_mutex Mutex; + bool OwnNativeHandle = false; +}; + +// Forward declarations for v2-only concrete handle types. +struct ur_context_handle_t_; +typedef struct ur_context_handle_t_ *ur_context_handle_t; +struct ur_event_handle_t_; +typedef struct ur_event_handle_t_ *ur_event_handle_t; +struct ur_usm_pool_handle_t_; +typedef struct ur_usm_pool_handle_t_ *ur_usm_pool_handle_t; +struct ur_kernel_handle_t_; +typedef struct ur_kernel_handle_t_ *ur_kernel_handle_t; +struct ur_queue_handle_t_; +typedef struct ur_queue_handle_t_ *ur_queue_handle_t; +struct ur_mem_handle_t_; +typedef struct ur_mem_handle_t_ *ur_mem_handle_t; +struct ur_exp_command_buffer_handle_t_; +typedef struct ur_exp_command_buffer_handle_t_ *ur_exp_command_buffer_handle_t; +struct ur_exp_graph_handle_t_; +typedef struct ur_exp_graph_handle_t_ *ur_exp_graph_handle_t; +struct ur_exp_executable_graph_handle_t_; +typedef struct ur_exp_executable_graph_handle_t_ + *ur_exp_executable_graph_handle_t; + +// Cast from an opaque UR handle to the v2 concrete type. The loader only ever +// reads offset 0 (ddi_table). +namespace detail { +// Maps an opaque handle typedef to its corresponding v2 internal struct. +template struct v2_handle_traits; +template <> struct v2_handle_traits<::ur_context_handle_t> { + using type = ur_context_handle_t_; +}; +template <> struct v2_handle_traits<::ur_event_handle_t> { + using type = ur_event_handle_t_; +}; +template <> struct v2_handle_traits<::ur_usm_pool_handle_t> { + using type = ur_usm_pool_handle_t_; +}; +template <> struct v2_handle_traits<::ur_kernel_handle_t> { + using type = ur_kernel_handle_t_; +}; +template <> struct v2_handle_traits<::ur_queue_handle_t> { + using type = ur_queue_handle_t_; +}; +template <> struct v2_handle_traits<::ur_mem_handle_t> { + using type = ur_mem_handle_t_; +}; +template <> struct v2_handle_traits<::ur_exp_command_buffer_handle_t> { + using type = ur_exp_command_buffer_handle_t_; +}; +template <> struct v2_handle_traits<::ur_exp_graph_handle_t> { + using type = ur_exp_graph_handle_t_; +}; +template <> struct v2_handle_traits<::ur_exp_executable_graph_handle_t> { + using type = ur_exp_executable_graph_handle_t_; +}; +template <> struct v2_handle_traits<::ur_device_handle_t> { + using type = ur::level_zero::ur_device_handle_t_; +}; +template <> struct v2_handle_traits<::ur_program_handle_t> { + using type = ur::level_zero::ur_program_handle_t_; +}; + +template +using v2_internal_t = typename v2_handle_traits::type; + +// Reverse mapping: v2 internal struct -> opaque handle. +template struct v2_opaque_handle_for; +template <> struct v2_opaque_handle_for { + using type = ::ur_context_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_event_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_usm_pool_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_kernel_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_queue_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_mem_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_exp_command_buffer_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_exp_graph_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_exp_executable_graph_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_device_handle_t; +}; +template <> struct v2_opaque_handle_for { + using type = ::ur_program_handle_t; +}; +} // namespace detail + +// Opaque handle -> v2 internal pointer. +template +inline detail::v2_internal_t *v2_cast(Opaque h) { + return reinterpret_cast *>(h); +} + +// Opaque handle array -> v2 internal pointer array. +template +inline detail::v2_internal_t **v2_cast(Opaque *ph) { + return reinterpret_cast **>(ph); +} + +// const Opaque handle array -> const v2 internal pointer array. +template +inline detail::v2_internal_t *const *v2_cast(const Opaque *ph) { + return reinterpret_cast *const *>(ph); +} + +// V2 internal pointer -> opaque handle (reverse direction). +template +inline typename detail::v2_opaque_handle_for::type +v2_cast(Internal *p) { + return reinterpret_cast< + typename detail::v2_opaque_handle_for::type>(p); +} + +// V2 internal pointer array -> opaque handle array. +template +inline typename detail::v2_opaque_handle_for::type * +v2_cast(Internal **p) { + return reinterpret_cast< + typename detail::v2_opaque_handle_for::type *>(p); +} -namespace v2 { namespace raii { template +template struct ur_handle { - ur_handle(RawHandle handle = nullptr) : handle(handle) { + ur_handle(Handle handle = nullptr) : handle(handle) { if (handle) { - retain(handle); + retain(v2_cast(handle)); } } @@ -140,7 +286,7 @@ struct ur_handle { } if (this->handle) { - release(this->handle); + release(v2_cast(this->handle)); } this->handle = rhs.handle; @@ -151,24 +297,24 @@ struct ur_handle { ~ur_handle() { if (handle) { - release(handle); + release(v2_cast(handle)); } } - RawHandle get() const { return handle; } + Handle get() const { return handle; } - RawHandle operator->() const { return get(); } + Handle operator->() const { return get(); } private: - RawHandle handle; + Handle handle; }; using ur_context_handle_t = - ur_handle<::ur_context_handle_t, ur::level_zero::urContextRetain, - ur::level_zero::urContextRelease>; + ur_handle; using ur_device_handle_t = - ur_handle<::ur_device_handle_t, ur::level_zero::urDeviceRetain, - ur::level_zero::urDeviceRelease>; - + ur_handle; } // namespace raii -} // namespace v2 + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index d63c3aff5b747..a49c7cc6e4e19 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -7,13 +7,15 @@ // //===----------------------------------------------------------------------===// -#include "../adapter.hpp" -#include "../device.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" #include "context.hpp" #include "event_provider_counter.hpp" #include "event_provider_normal.hpp" +namespace ur::level_zero::v2 { + template static void sortAndUnique(std::vector &values) { std::sort(values.begin(), values.end()); values.erase(std::unique(values.begin(), values.end()), values.end()); @@ -39,16 +41,17 @@ static bool isCompletePlatformDeviceList(uint32_t deviceCount, sortAndUnique(requestedDevices); uint32_t platformDeviceCount = 0; - ur_result_t result = ur::level_zero::urDeviceGet( - hPlatform, UR_DEVICE_TYPE_ALL, 0, nullptr, &platformDeviceCount); + ur_result_t result = + ur::level_zero::urDeviceGet(common_cast(hPlatform), UR_DEVICE_TYPE_ALL, 0, + nullptr, &platformDeviceCount); if (result != UR_RESULT_SUCCESS || platformDeviceCount == 0) { return false; } std::vector platformDevices(platformDeviceCount); - result = ur::level_zero::urDeviceGet(hPlatform, UR_DEVICE_TYPE_ALL, - platformDeviceCount, - platformDevices.data(), nullptr); + result = ur::level_zero::urDeviceGet( + common_cast(hPlatform), UR_DEVICE_TYPE_ALL, platformDeviceCount, + common_cast(platformDevices.data()), nullptr); if (result != UR_RESULT_SUCCESS) { return false; } @@ -66,8 +69,9 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, uint32_t numDevices, const ur_device_handle_t *phDevices, bool ownZeContext) - : hContext(hContext, ownZeContext), - hDevices(uniqueDevices(numDevices, phDevices)), + : ur_context_common_t(hContext, uniqueDevices(numDevices, phDevices), + phDevices[0]->Platform), + hContext(hContext, ownZeContext), commandListCache( hContext, {phDevices[0]->Platform->ZeCopyOffloadExtensionSupported, phDevices[0]->Platform->ZeMutableCmdListExt.Supported, @@ -88,7 +92,7 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, this, phDevices[0]->Platform->getNumDevices(), [context = this, platform = phDevices[0]->Platform]( DeviceId deviceId, - v2::event_flags_t flags) -> std::unique_ptr { + event_flags_t flags) -> std::unique_ptr { auto device = platform->getDeviceById(deviceId); // TODO: just use per-context id? @@ -115,24 +119,6 @@ ur_result_t ur_context_handle_t_::retain() { return UR_RESULT_SUCCESS; } -ur_platform_handle_t ur_context_handle_t_::getPlatform() const { - return hDevices[0]->Platform; -} - -const std::vector & -ur_context_handle_t_::getDevices() const { - return hDevices; -} - -bool ur_context_handle_t_::isValidDevice(ur_device_handle_t hDevice) const { - while (hDevice) { - if (std::find(hDevices.begin(), hDevices.end(), hDevice) != hDevices.end()) - return true; - hDevice = hDevice->RootDevice; - } - return false; -} - ur_usm_pool_handle_t ur_context_handle_t_::getDefaultUSMPool() { return &defaultUSMPool; } @@ -233,96 +219,94 @@ ur_context_handle_t_::getDevicesWhichCanAccessAllocationsPresentOn( return retVal; } -namespace ur::level_zero { ur_result_t urContextCreate(uint32_t deviceCount, - const ur_device_handle_t *phDevices, + const ::ur_device_handle_t *phDevicesOpque, const ur_context_properties_t * /*pProperties*/, - ur_context_handle_t *phContext) { + ::ur_context_handle_t *phContextOpque) try { + auto phContext = v2_cast(phContextOpque); + auto phDevices = common_cast(phDevicesOpque); *phContext = nullptr; - try { - for (uint32_t i = 0; i < deviceCount; ++i) { - UR_ASSERT(phDevices[i], UR_RESULT_ERROR_INVALID_NULL_HANDLE); - } + for (uint32_t i = 0; i < deviceCount; ++i) { + UR_ASSERT(phDevices[i], UR_RESULT_ERROR_INVALID_NULL_HANDLE); + } - ur_platform_handle_t hPlatform = phDevices[0]->Platform; - ZeStruct contextDesc{}; - - if (isCompletePlatformDeviceList(deviceCount, phDevices)) { - if (auto zeContext = zeDriverGetDefaultContext(hPlatform->ZeDriver)) { - *phContext = - new ur_context_handle_t_(zeContext, deviceCount, phDevices, false); - { - std::scoped_lock Lock(hPlatform->ContextsMutex); - hPlatform->Contexts.push_back(*phContext); - } - return UR_RESULT_SUCCESS; + ur_platform_handle_t hPlatform = phDevices[0]->Platform; + ZeStruct contextDesc{}; + + if (isCompletePlatformDeviceList(deviceCount, phDevices)) { + if (auto zeContext = zeDriverGetDefaultContext(hPlatform->ZeDriver)) { + *phContext = new v2::ur_context_handle_t_(zeContext, deviceCount, + phDevices, false); + { + std::scoped_lock Lock(hPlatform->ContextsMutex); + hPlatform->Contexts.push_back(v2_cast(*phContext)); } + return UR_RESULT_SUCCESS; } + } - ze_context_handle_t rawZeContext{}; - ZE2UR_CALL(zeContextCreate, - (hPlatform->ZeDriver, &contextDesc, &rawZeContext)); - UR_LOG(INFO, "ZE context created with {} devices", deviceCount); - - // Wrap immediately so any exception thrown by the ur_context_handle_t_ - // constructor (after hContext member is initialised) does not double-free - // the Level Zero context handle. - *phContext = - new ur_context_handle_t_(rawZeContext, deviceCount, phDevices, true); - { - std::scoped_lock Lock(hPlatform->ContextsMutex); - hPlatform->Contexts.push_back(*phContext); - } - return UR_RESULT_SUCCESS; - } catch (...) { - UR_LOG(ERR, "creating context failed"); - delete *phContext; - *phContext = nullptr; - return exceptionToResult(std::current_exception()); + ze_context_handle_t rawZeContext{}; + ZE2UR_CALL(zeContextCreate, + (hPlatform->ZeDriver, &contextDesc, &rawZeContext)); + UR_LOG(INFO, "ZE context created with {} devices", deviceCount); + + // Wrap immediately so any exception thrown by the ur_context_handle_t_ + // constructor (after hContext member is initialised) does not double-free + // the Level Zero context handle. + *phContext = + new v2::ur_context_handle_t_(rawZeContext, deviceCount, phDevices, true); + { + std::scoped_lock Lock(hPlatform->ContextsMutex); + hPlatform->Contexts.push_back(v2_cast(*phContext)); } + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); } -ur_result_t urContextGetNativeHandle(ur_context_handle_t hContext, - ur_native_handle_t *phNativeContext) try { - *phNativeContext = - reinterpret_cast(hContext->getZeHandle()); +ur_result_t +urContextGetNativeHandle(::ur_context_handle_t hContextOpque, + ::ur_native_handle_t *phNativeContext) try { + *phNativeContext = reinterpret_cast( + v2_cast(hContextOpque)->getZeHandle()); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urContextCreateWithNativeHandle( - ur_native_handle_t hNativeContext, ur_adapter_handle_t, uint32_t numDevices, - const ur_device_handle_t *phDevices, + ::ur_native_handle_t hNativeContext, ::ur_adapter_handle_t, + uint32_t numDevices, const ::ur_device_handle_t *phDevicesOpque, const ur_context_native_properties_t *pProperties, - ur_context_handle_t *phContext) try { + ::ur_context_handle_t *phContextOpque) try { + auto phContext = v2_cast(phContextOpque); + auto phDevices = common_cast(phDevicesOpque); *phContext = nullptr; auto zeContext = reinterpret_cast(hNativeContext); auto ownZeHandle = pProperties ? pProperties->isNativeHandleOwned : false; - *phContext = - new ur_context_handle_t_(zeContext, numDevices, phDevices, ownZeHandle); + *phContext = new v2::ur_context_handle_t_(zeContext, numDevices, phDevices, + ownZeHandle); { auto hPlatform = phDevices[0]->Platform; std::scoped_lock Lock(hPlatform->ContextsMutex); - hPlatform->Contexts.push_back(*phContext); + hPlatform->Contexts.push_back(v2_cast(*phContext)); } return UR_RESULT_SUCCESS; } catch (...) { - delete *phContext; - *phContext = nullptr; return exceptionToResult(std::current_exception()); } -ur_result_t urContextRetain(ur_context_handle_t hContext) try { - return hContext->retain(); +ur_result_t urContextRetain(::ur_context_handle_t hContextOpque) try { + return v2_cast(hContextOpque)->retain(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urContextRelease(ur_context_handle_t hContext) try { +ur_result_t urContextRelease(::ur_context_handle_t hContextOpque) try { + auto hContext = v2_cast(hContextOpque); if (!hContext->RefCount.release()) return UR_RESULT_SUCCESS; @@ -330,7 +314,7 @@ ur_result_t urContextRelease(ur_context_handle_t hContext) try { { std::scoped_lock Lock(Platform->ContextsMutex); auto &Contexts = Platform->Contexts; - auto It = std::find(Contexts.begin(), Contexts.end(), hContext); + auto It = std::find(Contexts.begin(), Contexts.end(), hContextOpque); if (It != Contexts.end()) { Contexts.erase(It); } @@ -341,11 +325,10 @@ ur_result_t urContextRelease(ur_context_handle_t hContext) try { return exceptionToResult(std::current_exception()); } -ur_result_t urContextGetInfo(ur_context_handle_t hContext, +ur_result_t urContextGetInfo(::ur_context_handle_t hContextOpque, ur_context_info_t contextInfoType, size_t propSize, void *pContextInfo, size_t *pPropSizeRet) try { - // No locking needed here, we only read const members - + auto hContext = v2_cast(hContextOpque); UrReturnHelper ReturnValue(propSize, pContextInfo, pPropSizeRet); switch ( (uint32_t)contextInfoType) { // cast to avoid warnings on EXT enum values @@ -368,4 +351,4 @@ ur_result_t urContextGetInfo(ur_context_handle_t hContext, } catch (...) { return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/context.hpp b/unified-runtime/source/adapters/level_zero/v2/context.hpp index 29e305e8c69f7..fc6270a6fd986 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.hpp @@ -11,6 +11,9 @@ #include +#include + +#include "../common/interfaces.hpp" #include "command_list_cache.hpp" #include "common.hpp" #include "common/ur_ref_count.hpp" @@ -18,18 +21,16 @@ #include "logger/ur_logger.hpp" #include "usm.hpp" +namespace ur::level_zero::v2 { + enum class PoolCacheType { Immediate, Regular }; -struct ur_context_handle_t_ : ur_object { +struct ur_context_handle_t_ : ur_context_common_t { ur_context_handle_t_(ze_context_handle_t hContext, uint32_t numDevices, const ur_device_handle_t *phDevices, bool ownZeContext); ur_result_t retain(); - inline ze_context_handle_t getZeHandle() const { return hContext.get(); } - ur_platform_handle_t getPlatform() const; - - const std::vector &getDevices() const; ur_usm_pool_handle_t getDefaultUSMPool(); ur_usm_pool_handle_t getAsyncPool(); @@ -52,9 +53,9 @@ struct ur_context_handle_t_ : ur_object { std::vector getDevicesWhichCanAccessAllocationsPresentOn(ur_device_handle_t hDevice); - v2::event_pool &getNativeEventsPool() { return nativeEventsPool; } - v2::command_list_cache_t &getCommandListCache() { return commandListCache; } - v2::event_pool_cache &getEventPoolCache(PoolCacheType type) { + event_pool &getNativeEventsPool() { return nativeEventsPool; } + command_list_cache_t &getCommandListCache() { return commandListCache; } + event_pool_cache &getEventPoolCache(PoolCacheType type) { switch (type) { case PoolCacheType::Immediate: return eventPoolCacheImmediate; @@ -64,12 +65,9 @@ struct ur_context_handle_t_ : ur_object { UR_FFAILURE("Requested invalid event pool cache type"); } - v2::event_pool_cache &getReusableEventPoolCache() { + event_pool_cache &getReusableEventPoolCache() { return reusableEventPoolCache; } - // Checks if Device is covered by this context. - // For that the Device or its root devices need to be in the context. - bool isValidDevice(ur_device_handle_t Device) const; ur_exp_graph_handle_t getGraphFromZeHandle(ze_graph_handle_t zeGraph) { auto it = zeToUrGraphMap.find(zeGraph); @@ -89,19 +87,15 @@ struct ur_context_handle_t_ : ur_object { ur_shared_mutex GraphMapMutex; private: - const v2::raii::ze_context_handle_t hContext; - const std::vector - hDevices; // possibly without subdevices, only what was passed to ctor, - // context may have user-defined, limited subset of available - // devices - v2::command_list_cache_t commandListCache; - v2::event_pool_cache eventPoolCacheImmediate; - v2::event_pool_cache eventPoolCacheRegular; - v2::event_pool_cache reusableEventPoolCache; + const raii::ze_context_handle_t hContext; + command_list_cache_t commandListCache; + event_pool_cache eventPoolCacheImmediate; + event_pool_cache eventPoolCacheRegular; + event_pool_cache reusableEventPoolCache; // pool used for urEventCreateWithNativeHandle when native handle is NULL // (uses non-counter based events to allow for signaling from host) - v2::event_pool nativeEventsPool; + event_pool nativeEventsPool; ur_usm_pool_handle_t_ defaultUSMPool; ur_usm_pool_handle_t_ asyncPool; @@ -112,3 +106,5 @@ struct ur_context_handle_t_ : ur_object { // managers. Caller must protect accesses with GraphMapMutex. std::unordered_map zeToUrGraphMap; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/device.cpp b/unified-runtime/source/adapters/level_zero/v2/device.cpp new file mode 100644 index 0000000000000..cff868e87d0b4 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/device.cpp @@ -0,0 +1,91 @@ +//===--------- device.cpp - Level Zero Adapter v2 ------------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +#include "../common/device.hpp" +#include "../common/platform.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { + +ur_result_t urDeviceGetInfo(::ur_device_handle_t hDeviceOpque, + ::ur_device_info_t propName, size_t propSize, + void *propValue, size_t *propSizeRet) { + auto hDevice = ur::level_zero::common_cast(hDeviceOpque); + UrReturnHelper ReturnValue(propSize, propValue, propSizeRet); + + switch (propName) { + case UR_DEVICE_INFO_GRAPH_RECORD_AND_REPLAY_SUPPORT_EXP: { + if (!hDevice->Platform->ZeGraphExt.Supported) { + return ReturnValue(false); + } + + // The experimental variant of the extension reports its capabilities + // through a structure with a different type value; an older driver would + // not recognize the stable one and would leave graphFlags unset. The + // structure layout (stype, pNext, graphFlags) is identical between the two + // variants, so the stable type can be reused with the experimental value. + constexpr ze_structure_type_t ZeStructTypeRecordReplayGraphExpProperties = + static_cast(0x00030029); + ze_record_replay_graph_ext_properties_t GraphProperties{}; + GraphProperties.stype = + hDevice->Platform->ZeGraphExt.UsesLegacyExperimentalApi + ? ZeStructTypeRecordReplayGraphExpProperties + : ZE_STRUCTURE_TYPE_RECORD_REPLAY_GRAPH_EXT_PROPERTIES; + GraphProperties.pNext = nullptr; + ZeStruct DeviceProperties; + DeviceProperties.pNext = &GraphProperties; + ZE2UR_CALL(zeDeviceGetProperties, (hDevice->ZeDevice, &DeviceProperties)); + + constexpr ze_record_replay_graph_ext_flags_t GraphModeMask = + ZE_RECORD_REPLAY_GRAPH_EXT_FLAG_IMMUTABLE_GRAPH | + ZE_RECORD_REPLAY_GRAPH_EXT_FLAG_MUTABLE_GRAPH; + return ReturnValue(static_cast( + (GraphProperties.graphFlags & GraphModeMask) != 0)); + } + case UR_DEVICE_INFO_ENQUEUE_HOST_TASK_SUPPORT_EXP: + return ReturnValue( + static_cast(hDevice->Platform->ZeHostTaskExt.Supported)); + case UR_DEVICE_INFO_REUSABLE_EVENTS_SUPPORT_EXP: + return ReturnValue(static_cast(true)); + case UR_DEVICE_INFO_PER_EVENT_PROFILING_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_USM_HOST_ALLOC_REGISTER_SUPPORT_EXP: + // Registering existing host memory as a USM host allocation relies on the + // external system memory mapping extension being supported by the driver. + return ReturnValue( + hDevice->Platform->ZeExternalMemoryMappingExtensionSupported); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: +#if defined(__linux__) + return ReturnValue(true); +#else + return ReturnValue(false); +#endif + case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP: { +#if defined(__linux__) + constexpr uint32_t MinDriverBuild = 38646; + ZeStruct ZeDriverProperties; + ZE2UR_CALL(zeDriverGetProperties, + (hDevice->Platform->ZeDriver, &ZeDriverProperties)); + const uint32_t DriverBuild = ZeDriverProperties.driverVersion & 0xFFFF; + return ReturnValue(static_cast(hDevice->isBMGOrNewer() && + DriverBuild >= MinDriverBuild)); +#else + return ReturnValue(false); +#endif + } + default: + return ur::level_zero::urDeviceGetInfo(hDeviceOpque, propName, propSize, + propValue, propSizeRet); + } +} + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index 0c8ccade81c4a..ffef28393d829 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -10,6 +10,7 @@ #include #include +#include "../common/device.hpp" #include "context.hpp" #include "event.hpp" #include "event_pool.hpp" @@ -17,8 +18,9 @@ #include "queue_api.hpp" #include "queue_handle.hpp" -#include "../device.hpp" -#include "../ur_interface_loader.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { static uint64_t adjustEndEventTimestamp(uint64_t adjustedStartTimestamp, uint64_t endTimestamp, @@ -83,7 +85,7 @@ void event_profiling_data_t::recordStartTimestamp(ur_device_handle_t hDevice) { uint64_t deviceStartTimestamp = 0; UR_CALL_THROWS(ur::level_zero::urDeviceGetGlobalTimestamps( - hDevice, &deviceStartTimestamp, nullptr)); + common_cast(hDevice), &deviceStartTimestamp, nullptr)); assert(adjustedEventStartTimestamp == 0); adjustedEventStartTimestamp = deviceStartTimestamp; @@ -251,21 +253,21 @@ ur_event_handle_t_::ur_event_handle_t_( , nullptr) {} -namespace ur::level_zero { -ur_result_t urEventRetain(ur_event_handle_t hEvent) try { - return hEvent->retain(); +ur_result_t urEventRetain(::ur_event_handle_t hEventOpque) try { + return v2_cast(hEventOpque)->retain(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEventRelease(ur_event_handle_t hEvent) try { - return hEvent->release(); +ur_result_t urEventRelease(::ur_event_handle_t hEventOpque) try { + return v2_cast(hEventOpque)->release(); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEventWait(uint32_t numEvents, - const ur_event_handle_t *phEventWaitList) try { + const ::ur_event_handle_t *phEventWaitListOpque) try { + auto phEventWaitList = v2_cast(phEventWaitListOpque); for (uint32_t i = 0; i < numEvents; ++i) { phEventWaitList[i]->onWaitListUse(); ZE2UR_CALL(zeEventHostSynchronize, @@ -276,14 +278,15 @@ ur_result_t urEventWait(uint32_t numEvents, return exceptionToResult(std::current_exception()); } -ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, - size_t propValueSize, void *pPropValue, - size_t *pPropValueSizeRet) try { +ur_result_t urEventGetInfo(::ur_event_handle_t hEventOpque, + ur_event_info_t propName, size_t propValueSize, + void *pPropValue, size_t *pPropValueSizeRet) try { + auto event = v2_cast(hEventOpque); UrReturnHelper returnValue(propValueSize, pPropValue, pPropValueSizeRet); switch (propName) { case UR_EVENT_INFO_COMMAND_EXECUTION_STATUS: { - auto zeStatus = ZE_CALL_NOCHECK(zeEventQueryStatus, (hEvent->getZeEvent())); + auto zeStatus = ZE_CALL_NOCHECK(zeEventQueryStatus, (event->getZeEvent())); if (zeStatus == ZE_RESULT_NOT_READY) { return returnValue(UR_EVENT_STATUS_SUBMITTED); @@ -292,18 +295,18 @@ ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, } } case UR_EVENT_INFO_REFERENCE_COUNT: { - return returnValue(hEvent->RefCount.getCount()); + return returnValue(event->RefCount.getCount()); } case UR_EVENT_INFO_COMMAND_QUEUE: { - auto urQueueHandle = reinterpret_cast(hEvent->getQueue()) - + auto urQueueHandle = reinterpret_cast(event->getQueue()) - ur_queue_handle_t_::queue_offset; return returnValue(urQueueHandle); } case UR_EVENT_INFO_CONTEXT: { - return returnValue(hEvent->getContext()); + return returnValue(event->getContext()); } case UR_EVENT_INFO_COMMAND_TYPE: { - return returnValue(hEvent->getCommandType()); + return returnValue(event->getCommandType()); } default: UR_LOG(ERR, "Unsupported ParamName in urEventGetInfo: ParamName={}(0x{})", @@ -318,7 +321,7 @@ ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, ur_result_t urEventGetProfilingInfo( /// [in] handle of the event object - ur_event_handle_t hEvent, + ::ur_event_handle_t hEventOpque, /// [in] the name of the profiling property to query ur_profiling_info_t propName, /// [in] size in bytes of the profiling property value @@ -328,11 +331,12 @@ ur_result_t urEventGetProfilingInfo( /// [out][optional] pointer to the actual size in bytes returned in /// propValue size_t *pPropValueSizeRet) try { - std::scoped_lock lock(hEvent->Mutex); + auto event = v2_cast(hEventOpque); + std::scoped_lock lock(event->Mutex); // The event must either have profiling enabled or be recording timestamps. - bool isTimestampedEvent = hEvent->isTimestamped(); - if (!hEvent->isProfilingEnabled() && !isTimestampedEvent) { + bool isTimestampedEvent = event->isTimestamped(); + if (!event->isProfilingEnabled() && !isTimestampedEvent) { return UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE; } @@ -341,7 +345,7 @@ ur_result_t urEventGetProfilingInfo( // For timestamped events we have the timestamps ready directly on the event // handle, so we short-circuit the return. if (isTimestampedEvent) { - uint64_t contextStartTime = hEvent->getEventStartTimestmap(); + uint64_t contextStartTime = event->getEventStartTimestmap(); switch (propName) { case UR_PROFILING_INFO_COMMAND_QUEUED: case UR_PROFILING_INFO_COMMAND_SUBMIT: @@ -349,7 +353,7 @@ ur_result_t urEventGetProfilingInfo( case UR_PROFILING_INFO_COMMAND_END: case UR_PROFILING_INFO_COMMAND_START: case UR_PROFILING_INFO_COMMAND_COMPLETE: { - return returnValue(hEvent->getEventEndTimestamp()); + return returnValue(event->getEventEndTimestamp()); } default: UR_LOG(ERR, "urEventGetProfilingInfo: not supported ParamName"); @@ -357,7 +361,7 @@ ur_result_t urEventGetProfilingInfo( } } - auto hDevice = hEvent->getDevice(); + auto hDevice = event->getDevice(); if (!hDevice) { // no command has been enqueued with this event yet return UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE; @@ -370,14 +374,14 @@ ur_result_t urEventGetProfilingInfo( switch (propName) { case UR_PROFILING_INFO_COMMAND_START: { - ZE2UR_CALL(zeEventQueryKernelTimestamp, (hEvent->getZeEvent(), &tsResult)); + ZE2UR_CALL(zeEventQueryKernelTimestamp, (event->getZeEvent(), &tsResult)); uint64_t contextStartTime = (tsResult.global.kernelStart & timestampMaxValue) * zeTimerResolution; return returnValue(contextStartTime); } case UR_PROFILING_INFO_COMMAND_END: case UR_PROFILING_INFO_COMMAND_COMPLETE: { - ZE2UR_CALL(zeEventQueryKernelTimestamp, (hEvent->getZeEvent(), &tsResult)); + ZE2UR_CALL(zeEventQueryKernelTimestamp, (event->getZeEvent(), &tsResult)); uint64_t contextStartTime = (tsResult.global.kernelStart & timestampMaxValue); @@ -406,19 +410,22 @@ ur_result_t urEventGetProfilingInfo( return exceptionToResult(std::current_exception()); } -ur_result_t urEventGetNativeHandle(ur_event_handle_t hEvent, - ur_native_handle_t *phNativeEvent) try { - *phNativeEvent = reinterpret_cast(hEvent->getZeEvent()); +ur_result_t urEventGetNativeHandle(::ur_event_handle_t hEventOpque, + ::ur_native_handle_t *phNativeEvent) try { + *phNativeEvent = + reinterpret_cast(v2_cast(hEventOpque)->getZeEvent()); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t -urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent, - ur_context_handle_t hContext, +urEventCreateWithNativeHandle(::ur_native_handle_t hNativeEvent, + ::ur_context_handle_t hContextOpque, const ur_event_native_properties_t *pProperties, - ur_event_handle_t *phEvent) try { + ::ur_event_handle_t *phEventOpque) try { + auto hContext = v2_cast(hContextOpque); + auto phEvent = v2_cast(phEventOpque); if (!hNativeEvent) { assert((hContext->getNativeEventsPool().getFlags() & v2::EVENT_FLAGS_COUNTER) == 0); @@ -426,22 +433,26 @@ urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent, *phEvent = hContext->getNativeEventsPool().allocate(); ZE2UR_CALL(zeEventHostSignal, ((*phEvent)->getZeEvent())); } else { - *phEvent = new ur_event_handle_t_(hContext, hNativeEvent, pProperties); + *phEvent = new v2::ur_event_handle_t_(hContext, hNativeEvent, pProperties); } return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEventCreateExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urEventCreateExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, const ur_exp_event_desc_t *pEventDesc, - ur_event_handle_t *phEvent) try { - UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pEventDesc && phEvent, UR_RESULT_ERROR_INVALID_NULL_POINTER); + ::ur_event_handle_t *phEventOpque) try { + UR_ASSERT(hContextOpque && hDeviceOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pEventDesc && phEventOpque, UR_RESULT_ERROR_INVALID_NULL_POINTER); UR_ASSERT(!(pEventDesc->flags & UR_EXP_EVENT_FLAGS_MASK), UR_RESULT_ERROR_INVALID_ENUMERATION); + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + auto phEvent = v2_cast(phEventOpque); + const v2::event_flags_t flags = v2::EVENT_FLAGS_COUNTER | (pEventDesc->flags & UR_EXP_EVENT_FLAG_ENABLE_PROFILING @@ -466,4 +477,4 @@ ur_result_t urEventCreateExp(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event.hpp b/unified-runtime/source/adapters/level_zero/v2/event.hpp index be7df165e5423..f7e6fe0a83c64 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.hpp @@ -19,11 +19,11 @@ #include "common/ur_ref_count.hpp" #include "event_provider.hpp" +namespace ur::level_zero::v2 { + using ur_event_generation_t = int64_t; -namespace v2 { class event_pool; -} struct event_profiling_data_t { event_profiling_data_t(ze_event_handle_t hZeEvent) : hZeEvent(hZeEvent) {} @@ -54,7 +54,7 @@ struct event_profiling_data_t { bool timestampRecorded = false; }; -struct ur_event_handle_t_ : ur_object { +struct ur_event_handle_t_ : v2::ur_object_t { public: // The variant alternative encodes how the L0 event handle is torn down: // - cache_borrowed_event: pooled event; it is returned to the pool. @@ -165,3 +165,5 @@ struct ur_event_handle_t_ : ur_object { v2::event_flags_t flags; event_profiling_data_t profilingData; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index dc93af9b5102d..038886779ff48 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -12,7 +12,7 @@ #include "queue_api.hpp" #include "unified-runtime/ur_api.h" -namespace v2 { +namespace ur::level_zero::v2 { static constexpr size_t EVENTS_BURST = 64; @@ -69,4 +69,4 @@ event_flags_t event_pool::getFlags() const { return getProvider()->eventFlags(); } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp index 773821be73af7..5887c6810b3b7 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.hpp @@ -19,11 +19,11 @@ #include #include "../common.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #include "event.hpp" #include "event_provider.hpp" -namespace v2 { +namespace ur::level_zero::v2 { class event_pool { public: @@ -74,7 +74,7 @@ createEventIfRequested(event_pool *eventPool, ur_event_handle_t *phEvent, (*phEvent) = eventPool->allocate(); (*phEvent)->setQueue(queue); - return (*phEvent); + return *phEvent; } static inline ur_event_handle_t @@ -118,4 +118,4 @@ static inline ur_event_handle_t createEventAndRetain(event_pool *eventPool, return event; } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.cpp index 3c0c7dd6b2c8e..759c79cdfaf11 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.cpp @@ -7,10 +7,10 @@ // //===----------------------------------------------------------------------===// #include "event_pool_cache.hpp" -#include "../device.hpp" -#include "../platform.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" -namespace v2 { +namespace ur::level_zero::v2 { event_pool_cache::event_pool_cache(ur_context_handle_t hContext, size_t max_devices, @@ -45,4 +45,4 @@ raii::cache_borrowed_event_pool event_pool_cache::borrow(DeviceId id, }); } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.hpp b/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.hpp index fa36bf4fd7be4..ad44674185173 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool_cache.hpp @@ -18,11 +18,11 @@ #include #include -#include "../device.hpp" +#include "../common/device.hpp" #include "event_pool.hpp" #include "event_provider.hpp" -namespace v2 { +namespace ur::level_zero::v2 { namespace raii { using cache_borrowed_event_pool = @@ -56,4 +56,5 @@ class event_pool_cache { // Indexed by event_descriptor::index() std::vector>> pools; }; -} // namespace v2 + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp b/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp index 2f847441ba610..1b49bc22e698f 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp @@ -18,7 +18,7 @@ #include #include -namespace v2 { +namespace ur::level_zero::v2 { using event_flags_t = uint32_t; enum event_flag_t { @@ -54,4 +54,4 @@ class event_provider { virtual event_flags_t eventFlags() const = 0; }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp index 2f33959e654ff..77392862ad455 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp @@ -15,10 +15,10 @@ #include "event_provider_normal.hpp" #include "loader/ze_loader.h" -#include "../device.hpp" -#include "../platform.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" -namespace v2 { +namespace ur::level_zero::v2 { provider_counter::provider_counter(ur_platform_handle_t platform, ur_context_handle_t context, @@ -117,4 +117,4 @@ std::unique_ptr createProvider(ur_platform_handle_t platform, return std::make_unique(context, queueType, flags); } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp index 760fe55ce8336..a6b7786d85da3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp @@ -22,12 +22,12 @@ #include "event.hpp" #include "event_provider.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #include #include -namespace v2 { +namespace ur::level_zero::v2 { typedef ze_result_t (*zexCounterBasedEventCreate)( ze_context_handle_t hContext, ze_device_handle_t hDevice, @@ -61,4 +61,4 @@ std::unique_ptr createProvider(ur_platform_handle_t platform, ur_device_handle_t device, event_flags_t flags); -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.cpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.cpp index fbdeb00f3bbe9..9a55e42c09486 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.cpp @@ -16,11 +16,10 @@ #include "event_provider.hpp" #include "event_provider_normal.hpp" -#include "../common/latency_tracker.hpp" - #include "../common.hpp" +#include "../common/latency_tracker.hpp" -namespace v2 { +namespace ur::level_zero::v2 { static constexpr int EVENTS_BURST = 64; provider_pool::provider_pool(ur_context_handle_t context, queue_type queue, @@ -118,4 +117,4 @@ raii::cache_borrowed_event provider_normal::allocate() { event_flags_t provider_normal::eventFlags() const { return flags; } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.hpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.hpp index 2339051424af2..47d92386cac39 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_normal.hpp @@ -22,10 +22,10 @@ #include "event.hpp" #include "event_provider.hpp" -#include "../device.hpp" -#include "../ur_interface_loader.hpp" +#include "../common/device.hpp" +#include "ur_interface_loader.hpp" -namespace v2 { +namespace ur::level_zero::v2 { class provider_pool { public: @@ -58,4 +58,4 @@ class provider_normal : public event_provider { std::vector> pools; }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/graph.cpp b/unified-runtime/source/adapters/level_zero/v2/graph.cpp index ff2f325e24eab..a93fc167207a8 100644 --- a/unified-runtime/source/adapters/level_zero/v2/graph.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/graph.cpp @@ -7,15 +7,15 @@ // //===----------------------------------------------------------------------===// -#include - -#include "../ur_interface_loader.hpp" +#include "graph.hpp" #include "common.hpp" #include "context.hpp" -#include "graph.hpp" +#include "ur_interface_loader.hpp" #include +namespace ur::level_zero::v2 { + ur_exp_graph_handle_t_::ur_exp_graph_handle_t_(ur_context_handle_t hContext) : hContext(hContext) { ZE2UR_CALL_THROWS(hContext->getPlatform()->ZeGraphExt.graphCreate, @@ -75,23 +75,24 @@ void ZE_CALLBACK_CONV destructionCallbackWrapper(void *pUserData) { } // namespace -namespace ur::level_zero { - -ur_result_t urGraphCreateExp(ur_context_handle_t hContext, - ur_exp_graph_handle_t *phGraph) try { +ur_result_t urGraphCreateExp(::ur_context_handle_t hContextOpque, + ::ur_exp_graph_handle_t *phGraphOpque) try { + auto hContext = v2_cast(hContextOpque); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - *phGraph = new ur_exp_graph_handle_t_(hContext); + auto hGraph = new ur_exp_graph_handle_t_(hContext); + *phGraphOpque = v2_cast(hGraph); std::scoped_lock lock(hContext->GraphMapMutex); - hContext->registerGraph((*phGraph)->getZeHandle(), *phGraph); + hContext->registerGraph(hGraph->getZeHandle(), hGraph); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t hGraph) try { +ur_result_t urGraphDestroyExp(::ur_exp_graph_handle_t hGraphOpque) try { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -108,21 +109,24 @@ ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t hGraph) try { } ur_result_t urGraphInstantiateGraphExp( - ur_exp_graph_handle_t hGraph, - ur_exp_executable_graph_handle_t *phExecutableGraph) try { + ::ur_exp_graph_handle_t hGraphOpque, + ::ur_exp_executable_graph_handle_t *phExecutableGraphOpque) try { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - *phExecutableGraph = new ur_exp_executable_graph_handle_t_(hContext, hGraph); + *phExecutableGraphOpque = + v2_cast(new ur_exp_executable_graph_handle_t_(hContext, hGraph)); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urGraphExecutableGraphDestroyExp( - ur_exp_executable_graph_handle_t hExecutableGraph) try { + ::ur_exp_executable_graph_handle_t hExecutableGraphOpque) try { + auto hExecutableGraph = v2_cast(hExecutableGraphOpque); ur_context_handle_t hContext = hExecutableGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -134,7 +138,9 @@ ur_result_t urGraphExecutableGraphDestroyExp( return exceptionToResult(std::current_exception()); } -ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t hGraph, bool *pIsEmpty) { +ur_result_t urGraphIsEmptyExp(::ur_exp_graph_handle_t hGraphOpque, + bool *pIsEmpty) { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -151,7 +157,9 @@ ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t hGraph, bool *pIsEmpty) { return UR_RESULT_SUCCESS; } -ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t hGraph, uint64_t *pGraphId) { +ur_result_t urGraphGetIdExp(::ur_exp_graph_handle_t hGraphOpque, + uint64_t *pGraphId) { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); auto ZeGetId = hContext->getPlatform()->ZeGraphExt.zeGraphGetIdExt; if (!checkGraphExtensionSupport(hContext) || !ZeGetId) { @@ -164,8 +172,9 @@ ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t hGraph, uint64_t *pGraphId) { } ur_result_t urGraphSetDestructionCallbackExp( - ur_exp_graph_handle_t hGraph, + ::ur_exp_graph_handle_t hGraphOpque, ur_exp_graph_destruction_callback_t pfnCallback, void *pUserData) { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); auto ZeSetCallback = hContext->getPlatform()->ZeGraphExt.zeGraphSetDestructionCallbackExp; @@ -189,8 +198,9 @@ ur_result_t urGraphSetDestructionCallbackExp( return UR_RESULT_SUCCESS; } -ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t hGraph, +ur_result_t urGraphDumpContentsExp(::ur_exp_graph_handle_t hGraphOpque, const char *filePath) { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -202,8 +212,9 @@ ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t hGraph, return UR_RESULT_SUCCESS; } -ur_result_t urGraphGetNativeHandleExp(ur_exp_graph_handle_t hGraph, +ur_result_t urGraphGetNativeHandleExp(::ur_exp_graph_handle_t hGraphOpque, ur_native_handle_t *phNativeGraph) try { + auto hGraph = v2_cast(hGraphOpque); ur_context_handle_t hContext = hGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -216,8 +227,9 @@ ur_result_t urGraphGetNativeHandleExp(ur_exp_graph_handle_t hGraph, } ur_result_t urGraphExecutableGraphGetNativeHandleExp( - ur_exp_executable_graph_handle_t hExecutableGraph, + ::ur_exp_executable_graph_handle_t hExecutableGraphOpque, ur_native_handle_t *phNativeExecutableGraph) try { + auto hExecutableGraph = v2_cast(hExecutableGraphOpque); ur_context_handle_t hContext = hExecutableGraph->getContext(); if (!checkGraphExtensionSupport(hContext)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -230,4 +242,4 @@ ur_result_t urGraphExecutableGraphGetNativeHandleExp( return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/graph.hpp b/unified-runtime/source/adapters/level_zero/v2/graph.hpp index 889b1903e4335..869b0ffc17afd 100644 --- a/unified-runtime/source/adapters/level_zero/v2/graph.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/graph.hpp @@ -8,11 +8,14 @@ //===----------------------------------------------------------------------===// #pragma once +#include "../common/platform.hpp" #include "common.hpp" #include "context.hpp" #include "unified-runtime/ur_api.h" -struct ur_exp_graph_handle_t_ : ur_object { +namespace ur::level_zero::v2 { + +struct ur_exp_graph_handle_t_ : v2::ur_object_t { public: ur_exp_graph_handle_t_(ur_context_handle_t hContext); ur_exp_graph_handle_t_(ur_context_handle_t hContext, @@ -30,7 +33,7 @@ struct ur_exp_graph_handle_t_ : ur_object { ze_graph_handle_t zeGraph = nullptr; }; -struct ur_exp_executable_graph_handle_t_ : ur_object { +struct ur_exp_executable_graph_handle_t_ : v2::ur_object_t { public: ur_exp_executable_graph_handle_t_(ur_context_handle_t hContext, ur_exp_graph_handle_t hGraph); @@ -52,3 +55,5 @@ struct ur_exp_executable_graph_handle_t_ : ur_object { inline bool checkGraphExtensionSupport(ur_context_handle_t hContext) { return hContext->getPlatform()->ZeGraphExt.Supported; } + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp b/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp index eb14aa01b31b5..500504f4d5bd2 100644 --- a/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp @@ -11,12 +11,12 @@ #include -#include "../platform.hpp" -#include "../ur_interface_loader.hpp" +#include "../common/platform.hpp" #include "context.hpp" #include "event.hpp" +#include "ur_interface_loader.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v2 { namespace { @@ -26,13 +26,14 @@ constexpr size_t kIpcEventHandleDataSize = } // namespace -ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent, +ur_result_t urIPCGetEventHandleExp(::ur_event_handle_t hEventOpque, void **ppIPCEventHandleData, size_t *pIPCEventHandleDataSizeRet) try { - UR_ASSERT(hEvent, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(hEventOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(ppIPCEventHandleData && pIPCEventHandleDataSizeRet, UR_RESULT_ERROR_INVALID_NULL_POINTER); + auto hEvent = v2_cast(hEventOpque); std::shared_lock lock(hEvent->Mutex); UR_ASSERT(hEvent->isIpcCapable() && !hEvent->isIpcImported(), @@ -52,7 +53,7 @@ ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent, return exceptionToResult(std::current_exception()); } -ur_result_t urIPCPutEventHandleExp(ur_context_handle_t /*hContext*/, +ur_result_t urIPCPutEventHandleExp(::ur_context_handle_t /*hContext*/, void *pIPCEventHandleData) try { UR_ASSERT(pIPCEventHandleData, UR_RESULT_ERROR_INVALID_NULL_POINTER); // Free the buffer allocated by urIPCGetEventHandleExp via RAII. @@ -63,16 +64,19 @@ ur_result_t urIPCPutEventHandleExp(ur_context_handle_t /*hContext*/, return exceptionToResult(std::current_exception()); } -ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext, +ur_result_t urIPCOpenEventHandleExp(::ur_context_handle_t hContextOpque, const void *pIPCEventHandleData, size_t ipcEventHandleDataSize, - ur_event_handle_t *phEvent) try { - UR_ASSERT(hContext, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pIPCEventHandleData && phEvent, + ::ur_event_handle_t *phEventOpque) try { + UR_ASSERT(hContextOpque, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pIPCEventHandleData && phEventOpque, UR_RESULT_ERROR_INVALID_NULL_POINTER); UR_ASSERT(ipcEventHandleDataSize == kIpcEventHandleDataSize, UR_RESULT_ERROR_INVALID_VALUE); + auto hContext = v2_cast(hContextOpque); + auto phEvent = v2_cast(phEventOpque); + // The driver consumes the handle by value. ze_ipc_event_counter_based_handle_t handle = *static_cast( @@ -92,4 +96,4 @@ ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index c0a65ab335df2..595093e766342 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -15,12 +15,15 @@ #include "queue_api.hpp" #include "queue_handle.hpp" -#include "../device.hpp" -#include "../helpers/kernel_helpers.hpp" -#include "../platform.hpp" +#include "../common/api.hpp" +#include "../common/device.hpp" +#include "../common/helpers/kernel_helpers.hpp" +#include "../common/platform.hpp" +#include "../common/sampler.hpp" #include "../program.hpp" -#include "../sampler.hpp" -#include "../ur_interface_loader.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { ur_single_device_kernel_t::ur_single_device_kernel_t(ur_device_handle_t hDevice, ze_kernel_handle_t hKernel, @@ -51,8 +54,9 @@ ur_result_t ur_single_device_kernel_t::release() { ur_kernel_handle_t_::ur_kernel_handle_t_(ur_program_handle_t hProgram, const char *kernelName) : hProgram(hProgram), - deviceKernels(hProgram->Context->getPlatform()->getNumDevices()) { - ur::level_zero::urProgramRetain(hProgram); + deviceKernels( + v2_cast(hProgram->Context)->getPlatform()->getNumDevices()) { + ur::level_zero::urProgramRetain(common_cast(hProgram)); for (auto &Dev : hProgram->AssociatedDevices) { auto zeDevice = Dev->ZeDevice; @@ -68,12 +72,13 @@ ur_kernel_handle_t_::ur_kernel_handle_t_(ur_program_handle_t hProgram, ze_kernel_handle_t zeKernel; ZE2UR_CALL_THROWS(zeKernelCreate, (zeModule, &zeKernelDesc, &zeKernel)); - auto urDevice = std::find_if(hProgram->Context->getDevices().begin(), - hProgram->Context->getDevices().end(), - [zeDevice = zeDevice](const auto &urDevice) { - return urDevice->ZeDevice == zeDevice; - }); - assert(urDevice != hProgram->Context->getDevices().end()); + auto urDevice = + std::find_if(v2_cast(hProgram->Context)->getDevices().begin(), + v2_cast(hProgram->Context)->getDevices().end(), + [zeDevice = zeDevice](const auto &urDevice) { + return urDevice->ZeDevice == zeDevice; + }); + assert(urDevice != v2_cast(hProgram->Context)->getDevices().end()); auto deviceId = (*urDevice)->Id.value(); deviceKernels[deviceId].emplace(*urDevice, zeKernel, true); @@ -87,7 +92,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( const ur_kernel_native_properties_t *pProperties) : hProgram(hProgram), deviceKernels(context ? context->getPlatform()->getNumDevices() : 0) { - ur::level_zero::urProgramRetain(hProgram); + ur::level_zero::urProgramRetain(common_cast(hProgram)); auto ownZeHandle = pProperties ? pProperties->isNativeHandleOwned : false; @@ -118,7 +123,7 @@ ur_result_t ur_kernel_handle_t_::release() { } } - UR_CALL_THROWS(ur::level_zero::urProgramRelease(hProgram)); + UR_CALL_THROWS(ur::level_zero::urProgramRelease(common_cast(hProgram))); delete this; @@ -309,8 +314,8 @@ ur_result_t ur_kernel_handle_t_::prepareForSubmission( auto hZeKernel = deviceKernel.hKernel.get(); if (pGlobalWorkOffset != NULL) { - UR_CALL( - setKernelGlobalOffset(hContext, hZeKernel, workDim, pGlobalWorkOffset)); + UR_CALL(setKernelGlobalOffset(v2_cast(hContext), hZeKernel, workDim, + pGlobalWorkOffset)); } ZE2UR_CALL(zeKernelSetGroupSize, @@ -370,38 +375,39 @@ std::vector ur_kernel_handle_t_::getSourceAttributes() const { return attributes; } -namespace ur::level_zero { -ur_result_t urKernelCreate(ur_program_handle_t hProgram, +ur_result_t urKernelCreate(::ur_program_handle_t hProgramOpque, const char *pKernelName, - ur_kernel_handle_t *phKernel) try { - *phKernel = new ur_kernel_handle_t_(hProgram, pKernelName); + ::ur_kernel_handle_t *phKernelOpque) try { + *v2_cast(phKernelOpque) = + new ur_kernel_handle_t_(common_cast(hProgramOpque), pKernelName); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urKernelGetNativeHandle(ur_kernel_handle_t hKernel, - ur_native_handle_t *phNativeKernel) try { +ur_result_t urKernelGetNativeHandle(::ur_kernel_handle_t hKernelOpque, + ::ur_native_handle_t *phNativeKernel) try { // Return the handle of the kernel for the first device - *phNativeKernel = - reinterpret_cast(hKernel->getNativeZeHandle()); + *phNativeKernel = reinterpret_cast( + v2_cast(hKernelOpque)->getNativeZeHandle()); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t -urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, - ur_context_handle_t hContext, - ur_program_handle_t hProgram, +urKernelCreateWithNativeHandle(::ur_native_handle_t hNativeKernel, + ::ur_context_handle_t hContextOpque, + ::ur_program_handle_t hProgramOpque, const ur_kernel_native_properties_t *pProperties, - ur_kernel_handle_t *phKernel) try { + ::ur_kernel_handle_t *phKernelOpque) try { + auto hProgram = common_cast(hProgramOpque); if (!hProgram) { return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } - *phKernel = - new ur_kernel_handle_t_(hNativeKernel, hProgram, hContext, pProperties); + *v2_cast(phKernelOpque) = new ur_kernel_handle_t_( + hNativeKernel, hProgram, v2_cast(hContextOpque), pProperties); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -409,8 +415,8 @@ urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain - ur_kernel_handle_t hKernel) try { - hKernel->RefCount.retain(); + ::ur_kernel_handle_t hKernelOpque) try { + v2_cast(hKernelOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -418,15 +424,15 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release - ur_kernel_handle_t hKernel) try { - return hKernel->release(); + ::ur_kernel_handle_t hKernelOpque) try { + return v2_cast(hKernelOpque)->release(); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urKernelSetExecInfo( /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, + ::ur_kernel_handle_t hKernelOpque, /// [in] name of the execution attribute ur_kernel_exec_info_t propName, /// [in] size in byte the attribute value @@ -436,6 +442,7 @@ ur_result_t urKernelSetExecInfo( /// [in][range(0, propSize)] pointer to memory location holding the property /// value. const void *pPropValue) try { + auto hKernel = v2_cast(hKernelOpque); std::scoped_lock guard(hKernel->Mutex); @@ -446,9 +453,9 @@ ur_result_t urKernelSetExecInfo( ur_result_t urKernelGetGroupInfo( /// [in] handle of the Kernel object - ur_kernel_handle_t hKernel, + ::ur_kernel_handle_t hKernelOpque, /// [in] handle of the Device object - ur_device_handle_t hDevice, + ::ur_device_handle_t hDeviceOpque, /// [in] name of the work Group property to query ur_kernel_group_info_t paramName, /// [in] size of the Kernel Work Group property value @@ -459,6 +466,8 @@ ur_result_t urKernelGetGroupInfo( /// [out][optional] pointer to the actual size in bytes of data being /// queried by propName. size_t *pParamValueSizeRet) try { + auto hKernel = v2_cast(hKernelOpque); + auto hDevice = common_cast(hDeviceOpque); UrReturnHelper returnValue(paramValueSize, pParamValue, pParamValueSizeRet); // No locking needed here, we only read const members @@ -530,9 +539,9 @@ ur_result_t urKernelGetGroupInfo( ur_result_t urKernelGetSubGroupInfo( /// [in] handle of the Kernel object - ur_kernel_handle_t hKernel, + ::ur_kernel_handle_t hKernelOpque, /// [in] handle of the Device object - ur_device_handle_t hDevice, + ::ur_device_handle_t hDeviceOpque, /// [in] name of the SubGroup property to query ur_kernel_sub_group_info_t propName, /// [in] size of the Kernel SubGroup property value @@ -545,7 +554,7 @@ ur_result_t urKernelGetSubGroupInfo( size_t *pPropSizeRet) try { UrReturnHelper returnValue(propSize, pPropValue, pPropSizeRet); - auto props = hKernel->getProperties(hDevice); + auto props = v2_cast(hKernelOpque)->getProperties(common_cast(hDeviceOpque)); // No locking needed here, we only read const members if (propName == UR_KERNEL_SUB_GROUP_INFO_MAX_SUB_GROUP_SIZE) { @@ -565,9 +574,10 @@ ur_result_t urKernelGetSubGroupInfo( return exceptionToResult(std::current_exception()); } -ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, +ur_result_t urKernelGetInfo(::ur_kernel_handle_t hKernelOpque, ur_kernel_info_t paramName, size_t propSize, void *pKernelInfo, size_t *pPropSizeRet) try { + auto hKernel = v2_cast(hKernelOpque); UrReturnHelper ReturnValue(propSize, pKernelInfo, pPropSizeRet); @@ -575,9 +585,9 @@ ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, switch (paramName) { case UR_KERNEL_INFO_CONTEXT: return ReturnValue( - ur_context_handle_t{hKernel->getProgramHandle()->Context}); + ::ur_context_handle_t{hKernel->getProgramHandle()->Context}); case UR_KERNEL_INFO_PROGRAM: - return ReturnValue(ur_program_handle_t{hKernel->getProgramHandle()}); + return ReturnValue(common_cast(hKernel->getProgramHandle())); case UR_KERNEL_INFO_FUNCTION_NAME: { auto kernelName = hKernel->getCommonProperties().name; return ReturnValue(static_cast(kernelName.c_str())); @@ -613,9 +623,11 @@ ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, } ur_result_t urKernelGetSuggestedLocalWorkSize( - ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t workDim, - [[maybe_unused]] const size_t *pGlobalWorkOffset, + ::ur_kernel_handle_t hKernelOpque, ::ur_queue_handle_t hQueueOpque, + uint32_t workDim, [[maybe_unused]] const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, size_t *pSuggestedLocalWorkSize) { + auto hKernel = v2_cast(hKernelOpque); + auto hQueue = v2_cast(hQueueOpque); UR_ASSERT(workDim > 0, UR_RESULT_ERROR_INVALID_WORK_DIMENSION); UR_ASSERT(workDim < 4, UR_RESULT_ERROR_INVALID_WORK_DIMENSION); UR_ASSERT(pSuggestedLocalWorkSize != nullptr, @@ -638,20 +650,22 @@ ur_result_t urKernelGetSuggestedLocalWorkSize( } ur_result_t urKernelGetSuggestedLocalWorkSizeWithArgs( - ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t workDim, - const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, - [[maybe_unused]] uint32_t numArgs, + ::ur_kernel_handle_t hKernelOpque, ::ur_queue_handle_t hQueueOpque, + uint32_t workDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, [[maybe_unused]] uint32_t numArgs, [[maybe_unused]] const ur_exp_kernel_arg_properties_t *pArgs, size_t *pSuggestedLocalWorkSize) { - return ur::level_zero::urKernelGetSuggestedLocalWorkSize( - hKernel, hQueue, workDim, pGlobalWorkOffset, pGlobalWorkSize, + return ur::level_zero::v2::urKernelGetSuggestedLocalWorkSize( + hKernelOpque, hQueueOpque, workDim, pGlobalWorkOffset, pGlobalWorkSize, pSuggestedLocalWorkSize); } ur_result_t urKernelSuggestMaxCooperativeGroupCount( - ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, uint32_t workDim, - const size_t *pLocalWorkSize, size_t dynamicSharedMemorySize, - uint32_t *pGroupCountRet) { + ::ur_kernel_handle_t hKernelOpque, ::ur_device_handle_t hDeviceOpque, + uint32_t workDim, const size_t *pLocalWorkSize, + size_t dynamicSharedMemorySize, uint32_t *pGroupCountRet) { + auto hKernel = v2_cast(hKernelOpque); + auto hDevice = common_cast(hDeviceOpque); (void)dynamicSharedMemorySize; uint32_t wg[3]; @@ -668,4 +682,4 @@ ur_result_t urKernelSuggestMaxCooperativeGroupCount( return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp index 6d3eb32449477..e96f0c973d425 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp @@ -15,6 +15,8 @@ #include "common/ur_ref_count.hpp" #include "memory.hpp" +namespace ur::level_zero::v2 { + struct ur_single_device_kernel_t { ur_single_device_kernel_t(ur_device_handle_t hDevice, ze_kernel_handle_t hKernel, bool ownZeHandle); @@ -33,7 +35,7 @@ struct ur_single_device_kernel_t { mutable ZeCache> zeKernelProperties; }; -struct ur_kernel_handle_t_ : ur_object { +struct ur_kernel_handle_t_ : v2::ur_object_t { private: struct pending_memory_allocation_t; @@ -113,7 +115,7 @@ struct ur_kernel_handle_t_ : ur_object { wait_list_view &waitListView); // Get context of the kernel. - ur_context_handle_t getContext() const { return hProgram->Context; } + ur_context_handle_t getContext() const { return v2_cast(hProgram->Context); } ur::RefCount RefCount; @@ -156,3 +158,5 @@ struct ur_kernel_handle_t_ : ur_object { // pointer to any non-null kernel in deviceKernels ur_single_device_kernel_t *nonEmptyKernel; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/lockable.hpp b/unified-runtime/source/adapters/level_zero/v2/lockable.hpp index 080a09c3e972f..eaab96d37c0f4 100644 --- a/unified-runtime/source/adapters/level_zero/v2/lockable.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/lockable.hpp @@ -10,6 +10,8 @@ #pragma once #include +namespace ur::level_zero::v2 { + template struct locked { public: locked(T *object, std::unique_lock &&lock) @@ -56,3 +58,5 @@ template struct lockable { T object_; ur_mutex mut_; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index b0601ba956af2..2e23a07bdd020 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -8,12 +8,14 @@ //===----------------------------------------------------------------------===// #include "memory.hpp" - -#include "../ur_interface_loader.hpp" #include "context.hpp" +#include "ur_interface_loader.hpp" + +#include "../common/helpers/memory_helpers.hpp" +#include "../common/image_common.hpp" +#include "../common/platform.hpp" -#include "../helpers/memory_helpers.hpp" -#include "../image_common.hpp" +namespace ur::level_zero::v2 { static bool isAccessCompatible(ur_mem_buffer_t::device_access_mode_t requested, ur_mem_buffer_t::device_access_mode_t actual) { @@ -507,11 +509,11 @@ ur_mem_sub_buffer_t::ur_mem_sub_buffer_t(ur_mem_handle_t hParent, size_t offset, device_access_mode_t accessMode) : ur_mem_buffer_t(hParent->getBuffer()->getContext(), size, accessMode), hParent(hParent), offset(offset) { - ur::level_zero::urMemRetain(hParent); + ur::level_zero::v2::urMemRetain(v2_cast(hParent)); } ur_mem_sub_buffer_t::~ur_mem_sub_buffer_t() { - ur::level_zero::urMemRelease(hParent); + ur::level_zero::v2::urMemRelease(v2_cast(hParent)); } void *ur_mem_sub_buffer_t::getDevicePtr(ur_device_handle_t hDevice, @@ -622,11 +624,12 @@ ur_mem_image_t::copy_desc_t ur_mem_image_t::getCopyRegions( return {{src.zeImage.get(), zeSrcRegion}, {src.zeImage.get(), zeDstRegion}}; } -namespace ur::level_zero { -ur_result_t urMemBufferCreate(ur_context_handle_t hContext, +ur_result_t urMemBufferCreate(::ur_context_handle_t hContextOpque, ur_mem_flags_t flags, size_t size, const ur_buffer_properties_t *pProperties, - ur_mem_handle_t *phBuffer) try { + ::ur_mem_handle_t *phBufferOpque) try { + auto hContext = v2_cast(hContextOpque); + auto phBuffer = v2_cast(phBufferOpque); if (flags & UR_MEM_FLAG_ALLOC_HOST_POINTER) { // TODO: // Having PI_MEM_FLAGS_HOST_PTR_ALLOC for buffer requires allocation of @@ -662,10 +665,12 @@ ur_result_t urMemBufferCreate(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -ur_result_t urMemBufferPartition(ur_mem_handle_t hMem, ur_mem_flags_t flags, +ur_result_t urMemBufferPartition(::ur_mem_handle_t hMemOpque, + ur_mem_flags_t flags, ur_buffer_create_type_t bufferCreateType, const ur_buffer_region_t *pRegion, - ur_mem_handle_t *phMem) try { + ::ur_mem_handle_t *phMemOpque) try { + auto hMem = v2_cast(hMemOpque); auto hBuffer = hMem->getBuffer(); UR_ASSERT(bufferCreateType == UR_BUFFER_CREATE_TYPE_REGION, @@ -679,7 +684,7 @@ ur_result_t urMemBufferPartition(ur_mem_handle_t hMem, ur_mem_flags_t flags, UR_ASSERT(isAccessCompatible(accessMode, hBuffer->getDeviceAccessMode()), UR_RESULT_ERROR_INVALID_VALUE); - *phMem = ur_mem_handle_t_::create( + *v2_cast(phMemOpque) = ur_mem_handle_t_::create( hMem, pRegion->origin, pRegion->size, accessMode); return UR_RESULT_SUCCESS; @@ -687,9 +692,13 @@ ur_result_t urMemBufferPartition(ur_mem_handle_t hMem, ur_mem_flags_t flags, return exceptionToResult(std::current_exception()); } -ur_result_t urMemBufferCreateWithNativeHandle( - ur_native_handle_t hNativeMem, ur_context_handle_t hContext, - const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem) try { +ur_result_t +urMemBufferCreateWithNativeHandle(::ur_native_handle_t hNativeMem, + ::ur_context_handle_t hContextOpque, + const ur_mem_native_properties_t *pProperties, + ::ur_mem_handle_t *phMemOpque) try { + auto hContext = v2_cast(hContextOpque); + auto phMem = v2_cast(phMemOpque); auto ptr = reinterpret_cast(hNativeMem); bool ownNativeHandle = pProperties ? pProperties->isNativeHandleOwned : false; @@ -747,9 +756,10 @@ ur_result_t urMemBufferCreateWithNativeHandle( return exceptionToResult(std::current_exception()); } -ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, +ur_result_t urMemGetInfo(::ur_mem_handle_t hMemOpque, ur_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) try { + auto hMem = v2_cast(hMemOpque); // No locking needed here, we only read const members UrReturnHelper returnValue(propSize, pPropValue, pPropSizeRet); @@ -784,14 +794,16 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, return exceptionToResult(std::current_exception()); } -ur_result_t urMemRetain(ur_mem_handle_t hMem) try { +ur_result_t urMemRetain(::ur_mem_handle_t hMemOpque) try { + auto hMem = v2_cast(hMemOpque); hMem->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urMemRelease(ur_mem_handle_t hMem) try { +ur_result_t urMemRelease(::ur_mem_handle_t hMemOpque) try { + auto hMem = v2_cast(hMemOpque); if (!hMem->RefCount.release()) return UR_RESULT_SUCCESS; @@ -801,9 +813,10 @@ ur_result_t urMemRelease(ur_mem_handle_t hMem) try { return exceptionToResult(std::current_exception()); } -ur_result_t urMemGetNativeHandle(ur_mem_handle_t hMem, - ur_device_handle_t hDevice, - ur_native_handle_t *phNativeMem) try { +ur_result_t urMemGetNativeHandle(::ur_mem_handle_t hMemOpque, + ::ur_device_handle_t hDeviceOpque, + ::ur_native_handle_t *phNativeMem) try { + auto hMem = v2_cast(hMemOpque); if (hMem->isImage()) { auto hImage = hMem->getImage(); *phNativeMem = reinterpret_cast(hImage->getZeImage()); @@ -815,26 +828,29 @@ ur_result_t urMemGetNativeHandle(ur_mem_handle_t hMem, std::scoped_lock lock(hBuffer->getMutex()); wait_list_view emptyWaitListView(nullptr, 0); - auto ptr = hBuffer->getDevicePtr( - hDevice, ur_mem_buffer_t::device_access_mode_t::read_write, 0, - hBuffer->getSize(), nullptr, emptyWaitListView); + auto ptr = + hBuffer->getDevicePtr(common_cast(hDeviceOpque), + ur_mem_buffer_t::device_access_mode_t::read_write, + 0, hBuffer->getSize(), nullptr, emptyWaitListView); *phNativeMem = reinterpret_cast(ptr); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, +ur_result_t urMemImageCreate(::ur_context_handle_t hContextOpque, + ur_mem_flags_t flags, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, void *pHost, - ur_mem_handle_t *phMem) try { + ::ur_mem_handle_t *phMemOpque) try { + auto hContext = v2_cast(hContextOpque); // TODO: implement read-only, write-only if ((flags & UR_MEM_FLAG_READ_WRITE) == 0) { die("urMemImageCreate: Level-Zero implements only read-write buffer," "no read-only or write-only yet."); } - *phMem = ur_mem_handle_t_::create( + *v2_cast(phMemOpque) = ur_mem_handle_t_::create( hContext, flags, pImageFormat, pImageDesc, pHost); return UR_RESULT_SUCCESS; } catch (...) { @@ -842,20 +858,22 @@ ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, } ur_result_t urMemImageCreateWithNativeHandle( - ur_native_handle_t hNativeMem, ur_context_handle_t hContext, + ::ur_native_handle_t hNativeMem, ::ur_context_handle_t hContextOpque, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem) try { + const ur_mem_native_properties_t *pProperties, + ::ur_mem_handle_t *phMemOpque) try { + auto hContext = v2_cast(hContextOpque); auto zeImage = reinterpret_cast(hNativeMem); bool ownNativeHandle = pProperties ? pProperties->isNativeHandleOwned : false; - *phMem = ur_mem_handle_t_::create( + *v2_cast(phMemOpque) = ur_mem_handle_t_::create( hContext, pImageFormat, pImageDesc, zeImage, ownNativeHandle); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urMemImageGetInfo(ur_mem_handle_t /*hMemory*/, +ur_result_t urMemImageGetInfo(::ur_mem_handle_t /*hMemory*/, ur_image_info_t /*propName*/, size_t /*propSize*/, void * /*pPropValue*/, size_t * /*pPropSizeRet*/) { @@ -864,7 +882,7 @@ ur_result_t urMemImageGetInfo(ur_mem_handle_t /*hMemory*/, return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urIPCGetMemHandleExp(ur_context_handle_t, void *pMem, +ur_result_t urIPCGetMemHandleExp(::ur_context_handle_t, void *pMem, void **ppIPCMemHandleData, size_t *pIPCMemHandleDataSizeRet) { umf_memory_pool_handle_t umfPool; @@ -886,18 +904,20 @@ ur_result_t urIPCGetMemHandleExp(ur_context_handle_t, void *pMem, umfHandleSize)); } -ur_result_t urIPCPutMemHandleExp(ur_context_handle_t, void *pIPCMemHandleData) { +ur_result_t urIPCPutMemHandleExp(::ur_context_handle_t, + void *pIPCMemHandleData) { return umf::umf2urResult( umfPutIPCHandle(reinterpret_cast(pIPCMemHandleData))); } -ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urIPCOpenMemHandleExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem) { - auto *pool = hContext->getDefaultUSMPool()->getPool( - usm::pool_descriptor{hContext->getDefaultUSMPool(), hContext, hDevice, - UR_USM_TYPE_DEVICE, false}); + auto hContext = v2_cast(hContextOpque); + auto *pool = hContext->getDefaultUSMPool()->getPool(usm::pool_descriptor{ + v2_cast(hContext->getDefaultUSMPool()), hContextOpque, hDeviceOpque, + UR_USM_TYPE_DEVICE, false}); if (!pool) return UR_RESULT_ERROR_INVALID_CONTEXT; umf_memory_pool_handle_t umfPool = pool->umfPool.get(); @@ -921,8 +941,8 @@ ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, ppMem)); } -ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t, void *pMem) { +ur_result_t urIPCCloseMemHandleExp(::ur_context_handle_t, void *pMem) { return umf::umf2urResult(umfCloseIPCHandle(pMem)); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.hpp b/unified-runtime/source/adapters/level_zero/v2/memory.hpp index 709bb3e71c600..f304cd3f23f50 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.hpp @@ -13,17 +13,19 @@ #include -#include "../device.hpp" -#include "../helpers/memory_helpers.hpp" -#include "../image_common.hpp" +#include "../common/device.hpp" +#include "../common/helpers/memory_helpers.hpp" +#include "../common/image_common.hpp" #include "command_list_manager.hpp" #include "common.hpp" #include "common/ur_ref_count.hpp" #include +namespace ur::level_zero::v2 { + using usm_unique_ptr_t = std::unique_ptr>; -struct ur_mem_buffer_t : ur_object { +struct ur_mem_buffer_t : v2::ur_object_t { enum class device_access_mode_t { read_write, read_only, write_only }; @@ -216,7 +218,7 @@ struct ur_mem_sub_buffer_t : ur_mem_buffer_t { size_t offset; }; -struct ur_mem_image_t : ur_object { +struct ur_mem_image_t : v2::ur_object_t { ur_mem_image_t(ur_context_handle_t hContext, ur_mem_flags_t flags, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, void *pHost); @@ -248,7 +250,7 @@ struct ur_mem_image_t : ur_object { ZeStruct zeImageDesc; }; -struct ur_mem_handle_t_ : ur::handle_base { +struct ur_mem_handle_t_ : v2::ur_object_t { template static ur_mem_handle_t_ *create(Args &&...args) { return new ur_mem_handle_t_(std::in_place_type, @@ -288,7 +290,7 @@ struct ur_mem_handle_t_ : ur::handle_base { private: template ur_mem_handle_t_(std::in_place_type_t, Args &&...args) - : ur::handle_base(), + : v2::ur_object_t(), mem(std::in_place_type, std::forward(args)...) {} std::variant { ur_mem_sub_buffer_t, ur_mem_image_t> mem; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp index a366d4843d4a8..897bd9b896948 100644 --- a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -9,7 +9,7 @@ #include "physical_mem.hpp" #include "../common.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #ifdef __linux__ #include @@ -21,12 +21,15 @@ #include "context.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v2 { -ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, - ur_device_handle_t hDevice, size_t size, +ur_result_t urPhysicalMemCreate(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, size_t size, const ur_physical_mem_properties_t *pProperties, - ur_physical_mem_handle_t *phPhysicalMem) { + ::ur_physical_mem_handle_t *phPhysicalMem) { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + ZeStruct PhysicalMemDesc; PhysicalMemDesc.flags = 0; PhysicalMemDesc.size = size; @@ -48,8 +51,8 @@ ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, &PhysicalMemDesc, &ZePhysicalMem)); try { - *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, - hDevice, size, EnableIpc); + *phPhysicalMem = common_cast(new ur_physical_mem_handle_t_( + ZePhysicalMem, hContextOpque, hDevice, size, EnableIpc)); } catch (const std::bad_alloc &) { zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; @@ -60,36 +63,39 @@ ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->RefCount.retain(); +ur_result_t urPhysicalMemRetain(::ur_physical_mem_handle_t hPhysicalMemOpque) { + common_cast(hPhysicalMemOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { +ur_result_t urPhysicalMemRelease(::ur_physical_mem_handle_t hPhysicalMemOpque) { + auto hPhysicalMem = common_cast(hPhysicalMemOpque); if (!hPhysicalMem->RefCount.release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { - ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(), - hPhysicalMem->ZePhysicalMem)); + ZE2UR_CALL(zePhysicalMemDestroy, + (v2_cast(hPhysicalMem->Context)->getZeHandle(), + hPhysicalMem->ZePhysicalMem)); } delete hPhysicalMem; return UR_RESULT_SUCCESS; } -ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, +ur_result_t urPhysicalMemGetInfo(::ur_physical_mem_handle_t hPhysicalMemOpque, ur_physical_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + auto hPhysicalMem = common_cast(hPhysicalMemOpque); + switch (propName) { case UR_PHYSICAL_MEM_INFO_CONTEXT: return ReturnValue(hPhysicalMem->Context); case UR_PHYSICAL_MEM_INFO_DEVICE: - return ReturnValue(hPhysicalMem->Device); + return ReturnValue(common_cast(hPhysicalMem->Device)); case UR_PHYSICAL_MEM_INFO_SIZE: return ReturnValue(hPhysicalMem->Size); case UR_PHYSICAL_MEM_INFO_PROPERTIES: { @@ -108,11 +114,12 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, return UR_RESULT_SUCCESS; } -ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, - ur_physical_mem_handle_t hPhysMem, +ur_result_t urIPCGetPhysMemHandleExp(::ur_context_handle_t, + ::ur_physical_mem_handle_t hPhysMemOpque, void **ppIPCPhysMemHandleData, size_t *pIPCPhysMemHandleDataSizeRet) { #ifdef __linux__ + auto hPhysMem = common_cast(hPhysMemOpque); if (!hPhysMem->EnableIpc) return UR_RESULT_ERROR_INVALID_ARGUMENT; @@ -135,8 +142,8 @@ ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, // hContext: zePhysicalMemGetProperties requires the same context that was // used to create the object. ze_result_t ZeRes = ZE_CALL_NOCHECK( - zePhysicalMemGetProperties, - (hPhysMem->Context->getZeHandle(), hPhysMem->ZePhysicalMem, &Props)); + zePhysicalMemGetProperties, (v2_cast(hPhysMem->Context)->getZeHandle(), + hPhysMem->ZePhysicalMem, &Props)); if (ZeRes != ZE_RESULT_SUCCESS) { if (ExportFd.fd >= 0) close(ExportFd.fd); @@ -163,14 +170,14 @@ ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, *pIPCPhysMemHandleDataSizeRet = sizeof(ZeIPCPhysMemHandleData); return UR_RESULT_SUCCESS; #else - (void)hPhysMem; + (void)hPhysMemOpque; (void)ppIPCPhysMemHandleData; (void)pIPCPhysMemHandleDataSizeRet; return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; #endif // __linux__ } -ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, +ur_result_t urIPCPutPhysMemHandleExp(::ur_context_handle_t, const void *pIPCPhysMemHandleData) { #ifdef __linux__ auto *HandleData = @@ -184,12 +191,15 @@ ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, #endif // __linux__ } -ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urIPCOpenPhysMemHandleExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, const void *pIPCPhysMemHandleData, size_t ipcPhysMemHandleDataSize, - ur_physical_mem_handle_t *phPhysMem) { + ::ur_physical_mem_handle_t *phPhysMem) { #ifdef __linux__ + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + if (ipcPhysMemHandleDataSize != sizeof(ZeIPCPhysMemHandleData)) return UR_RESULT_ERROR_INVALID_VALUE; @@ -237,9 +247,9 @@ ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, return ze2urResult(ZeRes); try { - *phPhysMem = new ur_physical_mem_handle_t_(ZePhysMem, hContext, hDevice, - HandleData->Size, - /*EnableIpc=*/true); + *phPhysMem = common_cast( + new ur_physical_mem_handle_t_(ZePhysMem, hContextOpque, hDevice, + HandleData->Size, /*EnableIpc=*/true)); } catch (const std::bad_alloc &) { zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; @@ -249,8 +259,8 @@ ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, } return UR_RESULT_SUCCESS; #else - (void)hContext; - (void)hDevice; + (void)hContextOpque; + (void)hDeviceOpque; (void)pIPCPhysMemHandleData; (void)ipcPhysMemHandleDataSize; (void)phPhysMem; @@ -258,12 +268,13 @@ ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, #endif // __linux__ } -ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, - ur_physical_mem_handle_t hPhysMem) { +ur_result_t urIPCClosePhysMemHandleExp(::ur_context_handle_t hContextOpque, + ::ur_physical_mem_handle_t hPhysMem) { // Delegate to urPhysicalMemRelease so the refcount is respected: if the // handle has been retained (refcount > 1) it will not be destroyed until // all references are released. - return ur::level_zero::urPhysicalMemRelease(hPhysMem); + (void)hContextOpque; + return v2::urPhysicalMemRelease(hPhysMem); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp index 9fc2ea8dc55ce..49fec957d2b93 100644 --- a/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// #pragma once -#include "../physical_mem.hpp" +#include "../common/physical_mem.hpp" // Opaque handle data exchanged between processes for physical memory IPC. // Contains the PID and file descriptor exported by zePhysicalMemGetProperties, diff --git a/unified-runtime/source/adapters/level_zero/v2/platform.cpp b/unified-runtime/source/adapters/level_zero/v2/platform.cpp new file mode 100644 index 0000000000000..8160504571ba6 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/platform.cpp @@ -0,0 +1,29 @@ +//===--------- platform.cpp - Level Zero Adapter v2 ----------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +#include "../common/platform.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { + +ur_result_t urPlatformGetInfo(::ur_platform_handle_t hPlatformOpque, + ::ur_platform_info_t paramName, size_t size, + void *paramValue, size_t *sizeRet) { + if (paramName == UR_PLATFORM_INFO_NAME) { + UrReturnHelper ReturnValue(size, paramValue, sizeRet); + return ReturnValue("Intel(R) oneAPI Unified Runtime over Level-Zero V2"); + } + return ur::level_zero::urPlatformGetInfo(hPlatformOpque, paramName, size, + paramValue, sizeRet); +} + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp index 0c501870bbf1f..a3659383e8fcb 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp @@ -17,502 +17,532 @@ #include "queue_handle.hpp" #include "ur_util.hpp" +namespace ur::level_zero::v2 { + ur_queue_t_::~ur_queue_t_() {} -namespace ur::level_zero { -ur_result_t urQueueGetInfo(ur_queue_handle_t hQueue, ur_queue_info_t propName, +ur_result_t urQueueGetInfo(::ur_queue_handle_t hQueue, ur_queue_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) try { - return hQueue->get().queueGetInfo(propName, propSize, pPropValue, - pPropSizeRet); + return v2_cast(hQueue)->get().queueGetInfo(propName, propSize, pPropValue, + pPropSizeRet); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueRetain(ur_queue_handle_t hQueue) try { - return hQueue->queueRetain(); +ur_result_t urQueueRetain(::ur_queue_handle_t hQueue) try { + return v2_cast(hQueue)->queueRetain(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueRelease(ur_queue_handle_t hQueue) try { - return hQueue->queueRelease(); +ur_result_t urQueueRelease(::ur_queue_handle_t hQueue) try { + return v2_cast(hQueue)->queueRelease(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueGetNativeHandle(ur_queue_handle_t hQueue, +ur_result_t urQueueGetNativeHandle(::ur_queue_handle_t hQueue, ur_queue_native_desc_t *pDesc, - ur_native_handle_t *phNativeQueue) try { - return hQueue->get().queueGetNativeHandle(pDesc, phNativeQueue); + ::ur_native_handle_t *phNativeQueue) try { + return v2_cast(hQueue)->get().queueGetNativeHandle(pDesc, phNativeQueue); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueFinish(ur_queue_handle_t hQueue) try { - return hQueue->get().queueFinish(); +ur_result_t urQueueFinish(::ur_queue_handle_t hQueue) try { + return v2_cast(hQueue)->get().queueFinish(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueFlush(ur_queue_handle_t hQueue) try { - return hQueue->get().queueFlush(); +ur_result_t urQueueFlush(::ur_queue_handle_t hQueue) try { + return v2_cast(hQueue)->get().queueFlush(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueEventsWait(ur_queue_handle_t hQueue, +ur_result_t urEnqueueEventsWait(::ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueEventsWait(numEventsInWaitList, phEventWaitList, - phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueEventsWait( + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueEventsWaitWithBarrier( - ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueEventsWaitWithBarrier(numEventsInWaitList, - phEventWaitList, phEvent); +ur_result_t +urEnqueueEventsWaitWithBarrier(::ur_queue_handle_t hQueue, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueEventsWaitWithBarrier( + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemBufferRead(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, bool blockingRead, +ur_result_t urEnqueueMemBufferRead(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingRead, size_t offset, size_t size, void *pDst, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferRead(hBuffer, blockingRead, offset, size, - pDst, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferRead( + v2_cast(hBuffer), blockingRead, offset, size, pDst, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemBufferWrite( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, - size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferWrite(hBuffer, blockingWrite, offset, - size, pSrc, numEventsInWaitList, - phEventWaitList, phEvent); +ur_result_t urEnqueueMemBufferWrite(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, + bool blockingWrite, size_t offset, + size_t size, const void *pSrc, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferWrite( + v2_cast(hBuffer), blockingWrite, offset, size, pSrc, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueMemBufferReadRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingRead, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingRead, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pDst, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferReadRect( - hBuffer, blockingRead, bufferOrigin, hostOrigin, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, numEventsInWaitList, - phEventWaitList, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferReadRect( + v2_cast(hBuffer), blockingRead, bufferOrigin, hostOrigin, region, + bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueMemBufferWriteRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingWrite, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferWriteRect( - hBuffer, blockingWrite, bufferOrigin, hostOrigin, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, numEventsInWaitList, - phEventWaitList, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferWriteRect( + v2_cast(hBuffer), blockingWrite, bufferOrigin, hostOrigin, region, + bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemBufferCopy(ur_queue_handle_t hQueue, - ur_mem_handle_t hBufferSrc, - ur_mem_handle_t hBufferDst, size_t srcOffset, - size_t dstOffset, size_t size, - uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferCopy( - hBufferSrc, hBufferDst, srcOffset, dstOffset, size, numEventsInWaitList, - phEventWaitList, phEvent); +ur_result_t urEnqueueMemBufferCopy(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, + size_t srcOffset, size_t dstOffset, + size_t size, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferCopy( + v2_cast(hBufferSrc), v2_cast(hBufferDst), srcOffset, dstOffset, size, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueMemBufferCopyRect( - ur_queue_handle_t hQueue, ur_mem_handle_t hBufferSrc, - ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferCopyRect( - hBufferSrc, hBufferDst, srcOrigin, dstOrigin, region, srcRowPitch, - srcSlicePitch, dstRowPitch, dstSlicePitch, numEventsInWaitList, - phEventWaitList, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferCopyRect( + v2_cast(hBufferSrc), v2_cast(hBufferDst), srcOrigin, dstOrigin, region, + srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemBufferFill(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, +ur_result_t urEnqueueMemBufferFill(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, const void *pPattern, size_t patternSize, size_t offset, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemBufferFill(hBuffer, pPattern, patternSize, - offset, size, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemBufferFill( + v2_cast(hBuffer), pPattern, patternSize, offset, size, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemImageRead( - ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingRead, - ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, - size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemImageRead( - hImage, blockingRead, origin, region, rowPitch, slicePitch, pDst, - numEventsInWaitList, phEventWaitList, phEvent); +ur_result_t urEnqueueMemImageRead(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hImage, bool blockingRead, + ur_rect_offset_t origin, + ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pDst, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemImageRead( + v2_cast(hImage), blockingRead, origin, region, rowPitch, slicePitch, pDst, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemImageWrite( - ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingWrite, - ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, - size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemImageWrite( - hImage, blockingWrite, origin, region, rowPitch, slicePitch, pSrc, - numEventsInWaitList, phEventWaitList, phEvent); +ur_result_t urEnqueueMemImageWrite(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hImage, bool blockingWrite, + ur_rect_offset_t origin, + ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pSrc, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemImageWrite( + v2_cast(hImage), blockingWrite, origin, region, rowPitch, slicePitch, + pSrc, numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t -urEnqueueMemImageCopy(ur_queue_handle_t hQueue, ur_mem_handle_t hImageSrc, - ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, +urEnqueueMemImageCopy(::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImageSrc, + ::ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemImageCopy( - hImageSrc, hImageDst, srcOrigin, dstOrigin, region, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemImageCopy( + v2_cast(hImageSrc), v2_cast(hImageDst), srcOrigin, dstOrigin, region, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemBufferMap(ur_queue_handle_t hQueue, - ur_mem_handle_t hBuffer, bool blockingMap, +ur_result_t urEnqueueMemBufferMap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingMap, ur_map_flags_t mapFlags, size_t offset, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent, void **ppRetMap) try { - return hQueue->get().enqueueMemBufferMap(hBuffer, blockingMap, mapFlags, - offset, size, numEventsInWaitList, - phEventWaitList, phEvent, ppRetMap); + return v2_cast(hQueue)->get().enqueueMemBufferMap( + v2_cast(hBuffer), blockingMap, mapFlags, offset, size, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent), + ppRetMap); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueMemUnmap(ur_queue_handle_t hQueue, ur_mem_handle_t hMem, - void *pMappedPtr, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueMemUnmap(hMem, pMappedPtr, numEventsInWaitList, - phEventWaitList, phEvent); +ur_result_t urEnqueueMemUnmap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hMem, void *pMappedPtr, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueMemUnmap( + v2_cast(hMem), pMappedPtr, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMFill(ur_queue_handle_t hQueue, void *pMem, +ur_result_t urEnqueueUSMFill(::ur_queue_handle_t hQueue, void *pMem, size_t patternSize, const void *pPattern, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMFill(pMem, patternSize, pPattern, size, - numEventsInWaitList, phEventWaitList, - phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMFill( + pMem, patternSize, pPattern, size, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMMemcpy(ur_queue_handle_t hQueue, bool blocking, +ur_result_t urEnqueueUSMMemcpy(::ur_queue_handle_t hQueue, bool blocking, void *pDst, const void *pSrc, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMMemcpy(blocking, pDst, pSrc, size, - numEventsInWaitList, phEventWaitList, - phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMMemcpy( + blocking, pDst, pSrc, size, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMPrefetch(ur_queue_handle_t hQueue, const void *pMem, +ur_result_t urEnqueueUSMPrefetch(::ur_queue_handle_t hQueue, const void *pMem, size_t size, ur_usm_migration_flags_t flags, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMPrefetch( - pMem, size, flags, numEventsInWaitList, phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMPrefetch( + pMem, size, flags, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMAdvise(ur_queue_handle_t hQueue, const void *pMem, +ur_result_t urEnqueueUSMAdvise(::ur_queue_handle_t hQueue, const void *pMem, size_t size, ur_usm_advice_flags_t advice, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMAdvise(pMem, size, advice, phEvent); + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMAdvise(pMem, size, advice, + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMFill2D(ur_queue_handle_t hQueue, void *pMem, +ur_result_t urEnqueueUSMFill2D(::ur_queue_handle_t hQueue, void *pMem, size_t pitch, size_t patternSize, const void *pPattern, size_t width, size_t height, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMFill2D(pMem, pitch, patternSize, pPattern, - width, height, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMFill2D( + pMem, pitch, patternSize, pPattern, width, height, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMMemcpy2D(ur_queue_handle_t hQueue, bool blocking, +ur_result_t urEnqueueUSMMemcpy2D(::ur_queue_handle_t hQueue, bool blocking, void *pDst, size_t dstPitch, const void *pSrc, size_t srcPitch, size_t width, size_t height, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMMemcpy2D( + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMMemcpy2D( blocking, pDst, dstPitch, pSrc, srcPitch, width, height, - numEventsInWaitList, phEventWaitList, phEvent); + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueDeviceGlobalVariableWrite( - ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, - bool blockingWrite, size_t count, size_t offset, const void *pSrc, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueDeviceGlobalVariableWrite( - hProgram, name, blockingWrite, count, offset, pSrc, numEventsInWaitList, - phEventWaitList, phEvent); + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingWrite, size_t count, size_t offset, + const void *pSrc, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueDeviceGlobalVariableWrite( + v2_cast(hProgram), name, blockingWrite, count, offset, pSrc, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueDeviceGlobalVariableRead( - ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, - bool blockingRead, size_t count, size_t offset, void *pDst, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueDeviceGlobalVariableRead( - hProgram, name, blockingRead, count, offset, pDst, numEventsInWaitList, - phEventWaitList, phEvent); + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingRead, size_t count, size_t offset, + void *pDst, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueDeviceGlobalVariableRead( + v2_cast(hProgram), name, blockingRead, count, offset, pDst, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueReadHostPipe(ur_queue_handle_t hQueue, - ur_program_handle_t hProgram, +ur_result_t urEnqueueReadHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, const char *pipe_symbol, bool blocking, void *pDst, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueReadHostPipe(hProgram, pipe_symbol, blocking, - pDst, size, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueReadHostPipe( + v2_cast(hProgram), pipe_symbol, blocking, pDst, size, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueWriteHostPipe(ur_queue_handle_t hQueue, - ur_program_handle_t hProgram, +ur_result_t urEnqueueWriteHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, const char *pipe_symbol, bool blocking, void *pSrc, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueWriteHostPipe(hProgram, pipe_symbol, blocking, - pSrc, size, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueWriteHostPipe( + v2_cast(hProgram), pipe_symbol, blocking, pSrc, size, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueKernelLaunchWithArgsExp( - ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, + ::ur_queue_handle_t hQueue, ::ur_kernel_handle_t hKernel, uint32_t workDim, const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, const ur_kernel_launch_ext_properties_t *launchPropList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueKernelLaunchWithArgsExp( - hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, - numArgs, pArgs, launchPropList, numEventsInWaitList, phEventWaitList, - phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueKernelLaunchWithArgsExp( + v2_cast(hKernel), workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numArgs, pArgs, launchPropList, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueUSMDeviceAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMDeviceAllocExp( - pPool, size, pProperties, numEventsInWaitList, phEventWaitList, ppMem, - phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMDeviceAllocExp( + v2_cast(pPool), size, pProperties, numEventsInWaitList, + v2_cast(phEventWaitList), ppMem, v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueUSMSharedAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMSharedAllocExp( - pPool, size, pProperties, numEventsInWaitList, phEventWaitList, ppMem, - phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMSharedAllocExp( + v2_cast(pPool), size, pProperties, numEventsInWaitList, + v2_cast(phEventWaitList), ppMem, v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueUSMHostAllocExp( - ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, const ur_exp_async_usm_alloc_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - void **ppMem, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMHostAllocExp(pPool, size, pProperties, - numEventsInWaitList, - phEventWaitList, ppMem, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMHostAllocExp( + v2_cast(pPool), size, pProperties, numEventsInWaitList, + v2_cast(phEventWaitList), ppMem, v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueUSMFreeExp(ur_queue_handle_t hQueue, - ur_usm_pool_handle_t pPool, void *pMem, +ur_result_t urEnqueueUSMFreeExp(::ur_queue_handle_t hQueue, + ::ur_usm_pool_handle_t pPool, void *pMem, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueUSMFreeExp(pPool, pMem, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueUSMFreeExp( + v2_cast(pPool), pMem, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urBindlessImagesImageCopyExp( - ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + ::ur_queue_handle_t hQueue, const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, const ur_image_format_t *pSrcImageFormat, const ur_image_format_t *pDstImageFormat, ur_exp_image_copy_region_t *pCopyRegion, ur_exp_image_copy_flags_t imageCopyFlags, ur_exp_image_copy_input_types_t imageCopyInputTypes, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().bindlessImagesImageCopyExp( + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().bindlessImagesImageCopyExp( pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, pDstImageFormat, pCopyRegion, imageCopyFlags, imageCopyInputTypes, - numEventsInWaitList, phEventWaitList, phEvent); + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urBindlessImagesWaitExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, bool hasWaitValue, uint64_t waitValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().bindlessImagesWaitExternalSemaphoreExp( - hSemaphore, hasWaitValue, waitValue, numEventsInWaitList, phEventWaitList, - phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().bindlessImagesWaitExternalSemaphoreExp( + hSemaphore, hasWaitValue, waitValue, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urBindlessImagesSignalExternalSemaphoreExp( - ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, bool hasSignalValue, uint64_t signalValue, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().bindlessImagesSignalExternalSemaphoreExp( + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().bindlessImagesSignalExternalSemaphoreExp( hSemaphore, hasSignalValue, signalValue, numEventsInWaitList, - phEventWaitList, phEvent); + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueTimestampRecordingExp( - ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueTimestampRecordingExp( - blocking, numEventsInWaitList, phEventWaitList, phEvent); +ur_result_t +urEnqueueTimestampRecordingExp(::ur_queue_handle_t hQueue, bool blocking, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueTimestampRecordingExp( + blocking, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueCommandBufferExp( - ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueCommandBufferExp( - hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); + ::ur_queue_handle_t hQueue, ::ur_exp_command_buffer_handle_t hCommandBuffer, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueCommandBufferExp( + v2_cast(hCommandBuffer), numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueHostTaskExp( - ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, + ::ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, void *data, const ur_exp_host_task_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueHostTaskExp(pfnHostTask, data, pProperties, - numEventsInWaitList, phEventWaitList, - phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueHostTaskExp( + pfnHostTask, data, pProperties, numEventsInWaitList, + v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueEventsWaitWithBarrierExt( - ur_queue_handle_t hQueue, + ::ur_queue_handle_t hQueue, const ur_exp_enqueue_ext_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueEventsWaitWithBarrierExt( - pProperties, numEventsInWaitList, phEventWaitList, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueEventsWaitWithBarrierExt( + pProperties, numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urEnqueueNativeCommandExp( - ur_queue_handle_t hQueue, + ::ur_queue_handle_t hQueue, ur_exp_enqueue_native_command_function_t pfnNativeEnqueue, void *data, - uint32_t numMemsInMemList, const ur_mem_handle_t *phMemList, + uint32_t numMemsInMemList, const ::ur_mem_handle_t *phMemList, const ur_exp_enqueue_native_command_properties_t *pProperties, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueNativeCommandExp( - pfnNativeEnqueue, data, numMemsInMemList, phMemList, pProperties, - numEventsInWaitList, phEventWaitList, phEvent); + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueNativeCommandExp( + pfnNativeEnqueue, data, numMemsInMemList, v2_cast(phMemList), pProperties, + numEventsInWaitList, v2_cast(phEventWaitList), v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueBeginGraphCaptureExp(ur_queue_handle_t hQueue) try { - return hQueue->get().queueBeginGraphCapteExp(); +ur_result_t urQueueBeginGraphCaptureExp(::ur_queue_handle_t hQueue) try { + return v2_cast(hQueue)->get().queueBeginGraphCapteExp(); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueBeginCaptureIntoGraphExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t hGraph) try { - return hQueue->get().queueBeginCapteIntoGraphExp(hGraph); +ur_result_t +urQueueBeginCaptureIntoGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t hGraph) try { + return v2_cast(hQueue)->get().queueBeginCapteIntoGraphExp(v2_cast(hGraph)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueEndGraphCaptureExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t *phGraph) try { - return hQueue->get().queueEndGraphCapteExp(phGraph); +ur_result_t urQueueEndGraphCaptureExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph) try { + return v2_cast(hQueue)->get().queueEndGraphCapteExp(v2_cast(phGraph)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueGraphExp(ur_queue_handle_t hQueue, - ur_exp_executable_graph_handle_t hGraph, +ur_result_t urEnqueueGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_executable_graph_handle_t hGraph, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueGraphExp(hGraph, numEventsInWaitList, - phEventWaitList, phEvent); + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent) try { + return v2_cast(hQueue)->get().enqueueGraphExp( + v2_cast(hGraph), numEventsInWaitList, v2_cast(phEventWaitList), + v2_cast(phEvent)); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueIsGraphCaptureEnabledExp(ur_queue_handle_t hQueue, +ur_result_t urQueueIsGraphCaptureEnabledExp(::ur_queue_handle_t hQueue, bool *pResult) try { - return hQueue->get().queueIsGraphCapteEnabledExp(pResult); + return v2_cast(hQueue)->get().queueIsGraphCapteEnabledExp(pResult); } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urQueueGetGraphExp(ur_queue_handle_t hQueue, - ur_exp_graph_handle_t *phGraph) try { - return hQueue->get().queueGetGraphExp(phGraph); +ur_result_t urQueueGetGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph) try { + return v2_cast(hQueue)->get().queueGetGraphExp(v2_cast(phGraph)); } catch (...) { return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp index 8088e1e025a29..c380c853368fe 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp @@ -15,9 +15,12 @@ #pragma once +#include "common.hpp" #include "queue_extensions.hpp" #include +namespace ur::level_zero::v2 { + struct ur_queue_t_ : ur_queue_extensions { virtual ~ur_queue_t_(); @@ -185,3 +188,5 @@ struct ur_queue_t_ : ur_queue_extensions { virtual ur_result_t queueIsGraphCapteEnabledExp(bool *) = 0; virtual ur_result_t queueGetGraphExp(ur_exp_graph_handle_t *) = 0; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_batched.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_batched.cpp index 09cfaf6a9d9cb..88bebe21f10f5 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_batched.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_batched.cpp @@ -18,20 +18,20 @@ #include "lockable.hpp" #include "memory.hpp" +#include "../common/helpers/kernel_helpers.hpp" +#include "../common/image_common.hpp" #include "../common/latency_tracker.hpp" -#include "../helpers/kernel_helpers.hpp" -#include "../image_common.hpp" #include "../program.hpp" -#include "../ur_interface_loader.hpp" #include "unified-runtime/ur_api.h" #include "ur.hpp" +#include "ur_interface_loader.hpp" #include "ze_api.h" #include #include #include -namespace v2 { +namespace ur::level_zero::v2 { ur_queue_batched_t::ur_queue_batched_t( ur_context_handle_t hContext, ur_device_handle_t hDevice, uint32_t ordinal, @@ -1102,4 +1102,4 @@ ur_result_t ur_queue_batched_t::enqueueHostTaskExp( this->getEvent(batchLocked, phEvent)); } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_batched.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_batched.hpp index 1d68103f6368d..df408a61e6675 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_batched.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_batched.hpp @@ -10,7 +10,7 @@ #pragma once #include "../common.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #include "command_list_cache.hpp" #include "common/ur_ref_count.hpp" @@ -44,7 +44,7 @@ // ur_queue_flags_t or globally, through the environment variable // UR_L0_V2_FORCE_BATCHED=1. -namespace v2 { +namespace ur::level_zero::v2 { // The limit of regular command lists stored for execution; if exceeded, the // vector is cleared as part of queueFinish and slots are renewed. @@ -159,7 +159,7 @@ struct batch_manager { void setGraphCapture(bool active) { graphCaptureActive = active; } }; -struct ur_queue_batched_t : ur_object, ur_queue_t_ { +struct ur_queue_batched_t : ur_object_t, ur_queue_t_ { private: ur_context_handle_t hContext; ur_device_handle_t hDevice; @@ -494,4 +494,4 @@ struct ur_queue_batched_t : ur_object, ur_queue_t_ { ur::RefCount RefCount; }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_create.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_create.cpp index ca20e41795261..b416f45917c70 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_create.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_create.cpp @@ -18,7 +18,7 @@ static const bool ForceBatched = getenv_tobool("UR_L0_V2_FORCE_BATCHED"); -namespace v2 { +namespace ur::level_zero::v2 { using queue_group_type = ur_device_handle_t_::queue_group_info_t::type; @@ -54,13 +54,17 @@ static event_flags_t eventFlagsFromQueueFlags(ur_queue_flags_t flags) { return eventFlags; } -} // namespace v2 +} // namespace ur::level_zero::v2 -namespace ur::level_zero { -ur_result_t urQueueCreate(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +namespace ur::level_zero::v2 { + +ur_result_t urQueueCreate(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, const ur_queue_properties_t *pProperties, - ur_queue_handle_t *phQueue) try { + ::ur_queue_handle_t *phQueueOpque) try { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + auto phQueue = v2_cast(phQueueOpque); if (!hContext->isValidDevice(hDevice)) { return UR_RESULT_ERROR_INVALID_DEVICE; } @@ -123,10 +127,14 @@ ur_result_t urQueueCreate(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -ur_result_t urQueueCreateWithNativeHandle( - ur_native_handle_t hNativeQueue, ur_context_handle_t hContext, - ur_device_handle_t hDevice, const ur_queue_native_properties_t *pProperties, - ur_queue_handle_t *phQueue) try { +ur_result_t +urQueueCreateWithNativeHandle(::ur_native_handle_t hNativeQueue, + ::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, + const ur_queue_native_properties_t *pProperties, + ::ur_queue_handle_t *phQueueOpque) try { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); // TODO: For now, always assume it's immediate, in-order bool ownNativeHandle = pProperties ? pProperties->isNativeHandleOwned : false; @@ -189,12 +197,13 @@ ur_result_t urQueueCreateWithNativeHandle( } }); - *phQueue = ur_queue_handle_t_::create( - hContext, hDevice, std::move(commandListHandle), - v2::eventFlagsFromQueueFlags(flags), flags); + *v2_cast(phQueueOpque) = + ur_queue_handle_t_::create( + hContext, hDevice, std::move(commandListHandle), + v2::eventFlagsFromQueueFlags(flags), flags); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_extensions.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_extensions.hpp index 6bff811a17fae..dd132a681227b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_extensions.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_extensions.hpp @@ -7,6 +7,8 @@ #include "unified-runtime/ur_api.h" +namespace ur::level_zero::v2 { + struct ur_queue_extensions { // Non-batched queues don't need to perform any action // @@ -21,3 +23,5 @@ struct ur_queue_extensions { virtual bool isInOrder() = 0; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index 6e2ca50accd2d..9fa4d564cc16c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -20,18 +20,19 @@ #include "queue_immediate_out_of_order.hpp" #include -struct ur_queue_handle_t_ : ur::handle_base { +namespace ur::level_zero::v2 { + +struct ur_queue_handle_t_ : v2::ur_object_t { using data_variant = std::variant; data_variant queue_data; - static constexpr uintptr_t queue_offset = - sizeof(ur::handle_base); + static constexpr uintptr_t queue_offset = sizeof(v2::ur_object_t); template ur_queue_handle_t_(std::in_place_type_t, Args &&...args) - : ur::handle_base(), + : v2::ur_object_t(), queue_data(std::in_place_type, std::forward(args)...) { assert(queue_offset == (std::visit([](auto &q) { return reinterpret_cast(&q); }, @@ -69,3 +70,5 @@ struct ur_queue_handle_t_ : ur::handle_base { queue_data); } }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index 5c409dcdb9a29..5f1190f9c232c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -13,14 +13,14 @@ #include "memory.hpp" #include "ur.hpp" +#include "../common/helpers/kernel_helpers.hpp" +#include "../common/image_common.hpp" #include "../common/latency_tracker.hpp" -#include "../helpers/kernel_helpers.hpp" -#include "../image_common.hpp" #include "../program.hpp" -#include "../ur_interface_loader.hpp" +#include "ur_interface_loader.hpp" -namespace v2 { +namespace ur::level_zero::v2 { ur_queue_immediate_in_order_t::ur_queue_immediate_in_order_t( ur_context_handle_t hContext, ur_device_handle_t hDevice, uint32_t ordinal, @@ -173,4 +173,4 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueEventsWaitWithBarrierExt( return cmdListMan->appendEventsWait(waitListView, event); } -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp index f96c7d2ca77b0..0b1bdeeb9d693 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp @@ -9,7 +9,7 @@ #pragma once #include "../common.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #include "common/ur_ref_count.hpp" #include "context.hpp" @@ -24,9 +24,9 @@ #include "command_list_manager.hpp" #include "lockable.hpp" -namespace v2 { +namespace ur::level_zero::v2 { -struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { +struct ur_queue_immediate_in_order_t : ur_object_t, ur_queue_t_ { private: ur_context_handle_t hContext; ur_device_handle_t hDevice; @@ -583,4 +583,4 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { ur::RefCount RefCount; }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index 296a54933d009..a63232829f79b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -12,7 +12,7 @@ #include "command_list_manager.hpp" #include "ur.hpp" -namespace v2 { +namespace ur::level_zero::v2 { template std::array createCommandListManagers( @@ -203,4 +203,5 @@ ur_result_t ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrierExt( return UR_RESULT_SUCCESS; } -} // namespace v2 + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 86a98164507d1..525a3b6802523 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -9,7 +9,7 @@ #pragma once #include "../common.hpp" -#include "../device.hpp" +#include "../common/device.hpp" #include "common/ur_ref_count.hpp" #include "context.hpp" @@ -21,9 +21,9 @@ #include "lockable.hpp" #include "ur/ur.hpp" -namespace v2 { +namespace ur::level_zero::v2 { -struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { +struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { private: // Number of command lists was chosen experimentally as a compromise // between number of allowed concurrent launches and overhead of @@ -654,4 +654,4 @@ struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { ur::RefCount RefCount; }; -} // namespace v2 +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.cpp new file mode 100644 index 0000000000000..84e62de65c325 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.cpp @@ -0,0 +1,807 @@ +//===--------- ur_interface_loader.cpp - Level Zero Adapter ------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include +#include +#include + +#include "ur_interface_loader.hpp" + +static ur_result_t validateProcInputs(ur_api_version_t version, + void *pDdiTable) { + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + // Pre 1.0 we enforce loader and adapter must have same version. + // Post 1.0 only major version match should be required. + if (version != UR_API_VERSION_CURRENT) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + return UR_RESULT_SUCCESS; +} + +#include "ur_interface_loader_common_forwarders.hpp" +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +namespace ur::level_zero::v2 { +#else +extern "C" { +#endif + +UR_APIEXPORT ur_result_t UR_APICALL urGetAdapterProcAddrTable( + ur_api_version_t version, ur_adapter_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGet = ur::level_zero::v2::urAdapterGet; + pDdiTable->pfnRelease = ur::level_zero::v2::urAdapterRelease; + pDdiTable->pfnRetain = ur::level_zero::v2::urAdapterRetain; + pDdiTable->pfnGetLastError = ur::level_zero::v2::urAdapterGetLastError; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urAdapterGetInfo; + pDdiTable->pfnSetLoggerCallback = + ur::level_zero::v2::urAdapterSetLoggerCallback; + pDdiTable->pfnSetLoggerCallbackLevel = + ur::level_zero::v2::urAdapterSetLoggerCallbackLevel; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( + ur_api_version_t version, ur_bindless_images_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnUnsampledImageHandleDestroyExp = + ur::level_zero::v2::urBindlessImagesUnsampledImageHandleDestroyExp; + pDdiTable->pfnSampledImageHandleDestroyExp = + ur::level_zero::v2::urBindlessImagesSampledImageHandleDestroyExp; + pDdiTable->pfnImageAllocateExp = + ur::level_zero::v2::urBindlessImagesImageAllocateExp; + pDdiTable->pfnImageFreeExp = ur::level_zero::v2::urBindlessImagesImageFreeExp; + pDdiTable->pfnUnsampledImageCreateExp = + ur::level_zero::v2::urBindlessImagesUnsampledImageCreateExp; + pDdiTable->pfnSampledImageCreateExp = + ur::level_zero::v2::urBindlessImagesSampledImageCreateExp; + pDdiTable->pfnImageCopyExp = ur::level_zero::v2::urBindlessImagesImageCopyExp; + pDdiTable->pfnImageGetInfoExp = + ur::level_zero::v2::urBindlessImagesImageGetInfoExp; + pDdiTable->pfnGetImageMemoryHandleTypeSupportExp = + ur::level_zero::v2::urBindlessImagesGetImageMemoryHandleTypeSupportExp; + pDdiTable->pfnGetImageUnsampledHandleSupportExp = + ur::level_zero::v2::urBindlessImagesGetImageUnsampledHandleSupportExp; + pDdiTable->pfnGetImageSampledHandleSupportExp = + ur::level_zero::v2::urBindlessImagesGetImageSampledHandleSupportExp; + pDdiTable->pfnMipmapGetLevelExp = + ur::level_zero::v2::urBindlessImagesMipmapGetLevelExp; + pDdiTable->pfnMipmapFreeExp = + ur::level_zero::v2::urBindlessImagesMipmapFreeExp; + pDdiTable->pfnImportExternalMemoryExp = + ur::level_zero::v2::urBindlessImagesImportExternalMemoryExp; + pDdiTable->pfnMapExternalArrayExp = + ur::level_zero::v2::urBindlessImagesMapExternalArrayExp; + pDdiTable->pfnMapExternalLinearMemoryExp = + ur::level_zero::v2::urBindlessImagesMapExternalLinearMemoryExp; + pDdiTable->pfnReleaseExternalMemoryExp = + ur::level_zero::v2::urBindlessImagesReleaseExternalMemoryExp; + pDdiTable->pfnFreeMappedLinearMemoryExp = + ur::level_zero::v2::urBindlessImagesFreeMappedLinearMemoryExp; + pDdiTable->pfnSupportsImportingHandleTypeExp = + ur::level_zero::v2::urBindlessImagesSupportsImportingHandleTypeExp; + pDdiTable->pfnImportExternalSemaphoreExp = + ur::level_zero::v2::urBindlessImagesImportExternalSemaphoreExp; + pDdiTable->pfnReleaseExternalSemaphoreExp = + ur::level_zero::v2::urBindlessImagesReleaseExternalSemaphoreExp; + pDdiTable->pfnWaitExternalSemaphoreExp = + ur::level_zero::v2::urBindlessImagesWaitExternalSemaphoreExp; + pDdiTable->pfnSignalExternalSemaphoreExp = + ur::level_zero::v2::urBindlessImagesSignalExternalSemaphoreExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetCommandBufferExpProcAddrTable( + ur_api_version_t version, ur_command_buffer_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreateExp = ur::level_zero::v2::urCommandBufferCreateExp; + pDdiTable->pfnRetainExp = ur::level_zero::v2::urCommandBufferRetainExp; + pDdiTable->pfnReleaseExp = ur::level_zero::v2::urCommandBufferReleaseExp; + pDdiTable->pfnFinalizeExp = ur::level_zero::v2::urCommandBufferFinalizeExp; + pDdiTable->pfnAppendKernelLaunchExp = + ur::level_zero::v2::urCommandBufferAppendKernelLaunchExp; + pDdiTable->pfnAppendKernelLaunchWithArgsExp = + ur::level_zero::v2::urCommandBufferAppendKernelLaunchWithArgsExp; + pDdiTable->pfnAppendUSMMemcpyExp = + ur::level_zero::v2::urCommandBufferAppendUSMMemcpyExp; + pDdiTable->pfnAppendUSMFillExp = + ur::level_zero::v2::urCommandBufferAppendUSMFillExp; + pDdiTable->pfnAppendMemBufferCopyExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferCopyExp; + pDdiTable->pfnAppendMemBufferWriteExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferWriteExp; + pDdiTable->pfnAppendMemBufferReadExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferReadExp; + pDdiTable->pfnAppendMemBufferCopyRectExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferCopyRectExp; + pDdiTable->pfnAppendMemBufferWriteRectExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferWriteRectExp; + pDdiTable->pfnAppendMemBufferReadRectExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferReadRectExp; + pDdiTable->pfnAppendMemBufferFillExp = + ur::level_zero::v2::urCommandBufferAppendMemBufferFillExp; + pDdiTable->pfnAppendUSMPrefetchExp = + ur::level_zero::v2::urCommandBufferAppendUSMPrefetchExp; + pDdiTable->pfnAppendUSMAdviseExp = + ur::level_zero::v2::urCommandBufferAppendUSMAdviseExp; + pDdiTable->pfnAppendNativeCommandExp = + ur::level_zero::v2::urCommandBufferAppendNativeCommandExp; + pDdiTable->pfnUpdateKernelLaunchExp = + ur::level_zero::v2::urCommandBufferUpdateKernelLaunchExp; + pDdiTable->pfnUpdateSignalEventExp = + ur::level_zero::v2::urCommandBufferUpdateSignalEventExp; + pDdiTable->pfnUpdateWaitEventsExp = + ur::level_zero::v2::urCommandBufferUpdateWaitEventsExp; + pDdiTable->pfnGetInfoExp = ur::level_zero::v2::urCommandBufferGetInfoExp; + pDdiTable->pfnGetNativeHandleExp = + ur::level_zero::v2::urCommandBufferGetNativeHandleExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetContextProcAddrTable( + ur_api_version_t version, ur_context_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreate = ur::level_zero::v2::urContextCreate; + pDdiTable->pfnRetain = ur::level_zero::v2::urContextRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urContextRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urContextGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urContextGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urContextCreateWithNativeHandle; + pDdiTable->pfnSetExtendedDeleter = + ur::level_zero::v2::urContextSetExtendedDeleter; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueProcAddrTable( + ur_api_version_t version, ur_enqueue_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnEventsWait = ur::level_zero::v2::urEnqueueEventsWait; + pDdiTable->pfnEventsWaitWithBarrier = + ur::level_zero::v2::urEnqueueEventsWaitWithBarrier; + pDdiTable->pfnMemBufferRead = ur::level_zero::v2::urEnqueueMemBufferRead; + pDdiTable->pfnMemBufferWrite = ur::level_zero::v2::urEnqueueMemBufferWrite; + pDdiTable->pfnMemBufferReadRect = + ur::level_zero::v2::urEnqueueMemBufferReadRect; + pDdiTable->pfnMemBufferWriteRect = + ur::level_zero::v2::urEnqueueMemBufferWriteRect; + pDdiTable->pfnMemBufferCopy = ur::level_zero::v2::urEnqueueMemBufferCopy; + pDdiTable->pfnMemBufferCopyRect = + ur::level_zero::v2::urEnqueueMemBufferCopyRect; + pDdiTable->pfnMemBufferFill = ur::level_zero::v2::urEnqueueMemBufferFill; + pDdiTable->pfnMemImageRead = ur::level_zero::v2::urEnqueueMemImageRead; + pDdiTable->pfnMemImageWrite = ur::level_zero::v2::urEnqueueMemImageWrite; + pDdiTable->pfnMemImageCopy = ur::level_zero::v2::urEnqueueMemImageCopy; + pDdiTable->pfnMemBufferMap = ur::level_zero::v2::urEnqueueMemBufferMap; + pDdiTable->pfnMemUnmap = ur::level_zero::v2::urEnqueueMemUnmap; + pDdiTable->pfnUSMFill = ur::level_zero::v2::urEnqueueUSMFill; + pDdiTable->pfnUSMMemcpy = ur::level_zero::v2::urEnqueueUSMMemcpy; + pDdiTable->pfnUSMPrefetch = ur::level_zero::v2::urEnqueueUSMPrefetch; + pDdiTable->pfnUSMAdvise = ur::level_zero::v2::urEnqueueUSMAdvise; + pDdiTable->pfnUSMFill2D = ur::level_zero::v2::urEnqueueUSMFill2D; + pDdiTable->pfnUSMMemcpy2D = ur::level_zero::v2::urEnqueueUSMMemcpy2D; + pDdiTable->pfnDeviceGlobalVariableWrite = + ur::level_zero::v2::urEnqueueDeviceGlobalVariableWrite; + pDdiTable->pfnDeviceGlobalVariableRead = + ur::level_zero::v2::urEnqueueDeviceGlobalVariableRead; + pDdiTable->pfnReadHostPipe = ur::level_zero::v2::urEnqueueReadHostPipe; + pDdiTable->pfnWriteHostPipe = ur::level_zero::v2::urEnqueueWriteHostPipe; + pDdiTable->pfnEventsWaitWithBarrierExt = + ur::level_zero::v2::urEnqueueEventsWaitWithBarrierExt; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( + ur_api_version_t version, ur_enqueue_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnKernelLaunchWithArgsExp = + ur::level_zero::v2::urEnqueueKernelLaunchWithArgsExp; + pDdiTable->pfnUSMDeviceAllocExp = + ur::level_zero::v2::urEnqueueUSMDeviceAllocExp; + pDdiTable->pfnUSMSharedAllocExp = + ur::level_zero::v2::urEnqueueUSMSharedAllocExp; + pDdiTable->pfnUSMHostAllocExp = ur::level_zero::v2::urEnqueueUSMHostAllocExp; + pDdiTable->pfnUSMFreeExp = ur::level_zero::v2::urEnqueueUSMFreeExp; + pDdiTable->pfnTimestampRecordingExp = + ur::level_zero::v2::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = + ur::level_zero::v2::urEnqueueCommandBufferExp; + pDdiTable->pfnHostTaskExp = ur::level_zero::v2::urEnqueueHostTaskExp; + pDdiTable->pfnNativeCommandExp = + ur::level_zero::v2::urEnqueueNativeCommandExp; + pDdiTable->pfnGraphExp = ur::level_zero::v2::urEnqueueGraphExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetEventProcAddrTable( + ur_api_version_t version, ur_event_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGetInfo = ur::level_zero::v2::urEventGetInfo; + pDdiTable->pfnGetProfilingInfo = ur::level_zero::v2::urEventGetProfilingInfo; + pDdiTable->pfnWait = ur::level_zero::v2::urEventWait; + pDdiTable->pfnRetain = ur::level_zero::v2::urEventRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urEventRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urEventGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urEventCreateWithNativeHandle; + pDdiTable->pfnSetCallback = ur::level_zero::v2::urEventSetCallback; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetEventExpProcAddrTable( + ur_api_version_t version, ur_event_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreateExp = ur::level_zero::v2::urEventCreateExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetGraphExpProcAddrTable( + ur_api_version_t version, ur_graph_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreateExp = ur::level_zero::v2::urGraphCreateExp; + pDdiTable->pfnInstantiateGraphExp = + ur::level_zero::v2::urGraphInstantiateGraphExp; + pDdiTable->pfnDestroyExp = ur::level_zero::v2::urGraphDestroyExp; + pDdiTable->pfnExecutableGraphDestroyExp = + ur::level_zero::v2::urGraphExecutableGraphDestroyExp; + pDdiTable->pfnIsEmptyExp = ur::level_zero::v2::urGraphIsEmptyExp; + pDdiTable->pfnGetIdExp = ur::level_zero::v2::urGraphGetIdExp; + pDdiTable->pfnSetDestructionCallbackExp = + ur::level_zero::v2::urGraphSetDestructionCallbackExp; + pDdiTable->pfnDumpContentsExp = ur::level_zero::v2::urGraphDumpContentsExp; + pDdiTable->pfnGetNativeHandleExp = + ur::level_zero::v2::urGraphGetNativeHandleExp; + pDdiTable->pfnExecutableGraphGetNativeHandleExp = + ur::level_zero::v2::urGraphExecutableGraphGetNativeHandleExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( + ur_api_version_t version, ur_ipc_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGetMemHandleExp = ur::level_zero::v2::urIPCGetMemHandleExp; + pDdiTable->pfnPutMemHandleExp = ur::level_zero::v2::urIPCPutMemHandleExp; + pDdiTable->pfnOpenMemHandleExp = ur::level_zero::v2::urIPCOpenMemHandleExp; + pDdiTable->pfnCloseMemHandleExp = ur::level_zero::v2::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur::level_zero::v2::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur::level_zero::v2::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur::level_zero::v2::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur::level_zero::v2::urIPCClosePhysMemHandleExp; + pDdiTable->pfnGetEventHandleExp = ur::level_zero::v2::urIPCGetEventHandleExp; + pDdiTable->pfnPutEventHandleExp = ur::level_zero::v2::urIPCPutEventHandleExp; + pDdiTable->pfnOpenEventHandleExp = + ur::level_zero::v2::urIPCOpenEventHandleExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetKernelProcAddrTable( + ur_api_version_t version, ur_kernel_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreate = ur::level_zero::v2::urKernelCreate; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urKernelGetInfo; + pDdiTable->pfnGetGroupInfo = ur::level_zero::v2::urKernelGetGroupInfo; + pDdiTable->pfnGetSubGroupInfo = ur::level_zero::v2::urKernelGetSubGroupInfo; + pDdiTable->pfnRetain = ur::level_zero::v2::urKernelRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urKernelRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urKernelGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urKernelCreateWithNativeHandle; + pDdiTable->pfnGetSuggestedLocalWorkSize = + ur::level_zero::v2::urKernelGetSuggestedLocalWorkSize; + pDdiTable->pfnGetSuggestedLocalWorkSizeWithArgs = + ur::level_zero::v2::urKernelGetSuggestedLocalWorkSizeWithArgs; + pDdiTable->pfnSetExecInfo = ur::level_zero::v2::urKernelSetExecInfo; + pDdiTable->pfnSetSpecializationConstants = + ur::level_zero::v2::urKernelSetSpecializationConstants; + pDdiTable->pfnSuggestMaxCooperativeGroupCount = + ur::level_zero::v2::urKernelSuggestMaxCooperativeGroupCount; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urGetMemProcAddrTable(ur_api_version_t version, ur_mem_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnImageCreate = ur::level_zero::v2::urMemImageCreate; + pDdiTable->pfnBufferCreate = ur::level_zero::v2::urMemBufferCreate; + pDdiTable->pfnRetain = ur::level_zero::v2::urMemRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urMemRelease; + pDdiTable->pfnBufferPartition = ur::level_zero::v2::urMemBufferPartition; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urMemGetNativeHandle; + pDdiTable->pfnBufferCreateWithNativeHandle = + ur::level_zero::v2::urMemBufferCreateWithNativeHandle; + pDdiTable->pfnImageCreateWithNativeHandle = + ur::level_zero::v2::urMemImageCreateWithNativeHandle; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urMemGetInfo; + pDdiTable->pfnImageGetInfo = ur::level_zero::v2::urMemImageGetInfo; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnAllocExportableMemoryExp = + ur::level_zero::v2::urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur::level_zero::v2::urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = + ur::level_zero::v2::urMemoryExportExportMemoryHandleExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( + ur_api_version_t version, ur_physical_mem_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreate = ur::level_zero::v2::urPhysicalMemCreate; + pDdiTable->pfnRetain = ur::level_zero::v2::urPhysicalMemRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urPhysicalMemRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urPhysicalMemGetInfo; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetPlatformProcAddrTable( + ur_api_version_t version, ur_platform_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGet = ur::level_zero::v2::urPlatformGet; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urPlatformGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urPlatformGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urPlatformCreateWithNativeHandle; + pDdiTable->pfnGetApiVersion = ur::level_zero::v2::urPlatformGetApiVersion; + pDdiTable->pfnGetBackendOption = + ur::level_zero::v2::urPlatformGetBackendOption; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetProgramProcAddrTable( + ur_api_version_t version, ur_program_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreateWithIL = ur::level_zero::v2::urProgramCreateWithIL; + pDdiTable->pfnCreateWithBinary = + ur::level_zero::v2::urProgramCreateWithBinary; + pDdiTable->pfnBuild = ur::level_zero::v2::urProgramBuild; + pDdiTable->pfnCompile = ur::level_zero::v2::urProgramCompile; + pDdiTable->pfnLink = ur::level_zero::v2::urProgramLink; + pDdiTable->pfnRetain = ur::level_zero::v2::urProgramRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urProgramRelease; + pDdiTable->pfnGetFunctionPointer = + ur::level_zero::v2::urProgramGetFunctionPointer; + pDdiTable->pfnGetGlobalVariablePointer = + ur::level_zero::v2::urProgramGetGlobalVariablePointer; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urProgramGetInfo; + pDdiTable->pfnGetBuildInfo = ur::level_zero::v2::urProgramGetBuildInfo; + pDdiTable->pfnSetSpecializationConstants = + ur::level_zero::v2::urProgramSetSpecializationConstants; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urProgramGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urProgramCreateWithNativeHandle; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetProgramExpProcAddrTable( + ur_api_version_t version, ur_program_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnDynamicLinkExp = ur::level_zero::v2::urProgramDynamicLinkExp; + pDdiTable->pfnBuildExp = ur::level_zero::v2::urProgramBuildExp; + pDdiTable->pfnCompileExp = ur::level_zero::v2::urProgramCompileExp; + pDdiTable->pfnLinkExp = ur::level_zero::v2::urProgramLinkExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetQueueProcAddrTable( + ur_api_version_t version, ur_queue_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGetInfo = ur::level_zero::v2::urQueueGetInfo; + pDdiTable->pfnCreate = ur::level_zero::v2::urQueueCreate; + pDdiTable->pfnRetain = ur::level_zero::v2::urQueueRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urQueueRelease; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urQueueGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urQueueCreateWithNativeHandle; + pDdiTable->pfnFinish = ur::level_zero::v2::urQueueFinish; + pDdiTable->pfnFlush = ur::level_zero::v2::urQueueFlush; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetQueueExpProcAddrTable( + ur_api_version_t version, ur_queue_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnBeginGraphCaptureExp = + ur::level_zero::v2::urQueueBeginGraphCaptureExp; + pDdiTable->pfnBeginCaptureIntoGraphExp = + ur::level_zero::v2::urQueueBeginCaptureIntoGraphExp; + pDdiTable->pfnEndGraphCaptureExp = + ur::level_zero::v2::urQueueEndGraphCaptureExp; + pDdiTable->pfnIsGraphCaptureEnabledExp = + ur::level_zero::v2::urQueueIsGraphCaptureEnabledExp; + pDdiTable->pfnGetGraphExp = ur::level_zero::v2::urQueueGetGraphExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetSamplerProcAddrTable( + ur_api_version_t version, ur_sampler_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnCreate = ur::level_zero::v2::urSamplerCreate; + pDdiTable->pfnRetain = ur::level_zero::v2::urSamplerRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urSamplerRelease; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urSamplerGetInfo; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urSamplerGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urSamplerCreateWithNativeHandle; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urGetUSMProcAddrTable(ur_api_version_t version, ur_usm_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnHostAlloc = ur::level_zero::v2::urUSMHostAlloc; + pDdiTable->pfnDeviceAlloc = ur::level_zero::v2::urUSMDeviceAlloc; + pDdiTable->pfnSharedAlloc = ur::level_zero::v2::urUSMSharedAlloc; + pDdiTable->pfnFree = ur::level_zero::v2::urUSMFree; + pDdiTable->pfnGetMemAllocInfo = ur::level_zero::v2::urUSMGetMemAllocInfo; + pDdiTable->pfnPoolCreate = ur::level_zero::v2::urUSMPoolCreate; + pDdiTable->pfnPoolRetain = ur::level_zero::v2::urUSMPoolRetain; + pDdiTable->pfnPoolRelease = ur::level_zero::v2::urUSMPoolRelease; + pDdiTable->pfnPoolGetInfo = ur::level_zero::v2::urUSMPoolGetInfo; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( + ur_api_version_t version, ur_usm_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnPoolCreateExp = ur::level_zero::v2::urUSMPoolCreateExp; + pDdiTable->pfnPoolDestroyExp = ur::level_zero::v2::urUSMPoolDestroyExp; + pDdiTable->pfnPoolGetDefaultDevicePoolExp = + ur::level_zero::v2::urUSMPoolGetDefaultDevicePoolExp; + pDdiTable->pfnPoolGetInfoExp = ur::level_zero::v2::urUSMPoolGetInfoExp; + pDdiTable->pfnPoolSetInfoExp = ur::level_zero::v2::urUSMPoolSetInfoExp; + pDdiTable->pfnPoolSetDevicePoolExp = + ur::level_zero::v2::urUSMPoolSetDevicePoolExp; + pDdiTable->pfnPoolGetDevicePoolExp = + ur::level_zero::v2::urUSMPoolGetDevicePoolExp; + pDdiTable->pfnPoolTrimToExp = ur::level_zero::v2::urUSMPoolTrimToExp; + pDdiTable->pfnPitchedAllocExp = ur::level_zero::v2::urUSMPitchedAllocExp; + pDdiTable->pfnContextMemcpyExp = ur::level_zero::v2::urUSMContextMemcpyExp; + pDdiTable->pfnHostAllocUnregisterExp = + ur::level_zero::v2::urUSMHostAllocUnregisterExp; + pDdiTable->pfnHostAllocRegisterExp = + ur::level_zero::v2::urUSMHostAllocRegisterExp; + pDdiTable->pfnImportExp = ur::level_zero::v2::urUSMImportExp; + pDdiTable->pfnReleaseExp = ur::level_zero::v2::urUSMReleaseExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetUsmP2PExpProcAddrTable( + ur_api_version_t version, ur_usm_p2p_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnEnablePeerAccessExp = + ur::level_zero::v2::urUsmP2PEnablePeerAccessExp; + pDdiTable->pfnDisablePeerAccessExp = + ur::level_zero::v2::urUsmP2PDisablePeerAccessExp; + pDdiTable->pfnPeerAccessGetInfoExp = + ur::level_zero::v2::urUsmP2PPeerAccessGetInfoExp; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( + ur_api_version_t version, ur_virtual_mem_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGranularityGetInfo = + ur::level_zero::v2::urVirtualMemGranularityGetInfo; + pDdiTable->pfnReserve = ur::level_zero::v2::urVirtualMemReserve; + pDdiTable->pfnFree = ur::level_zero::v2::urVirtualMemFree; + pDdiTable->pfnMap = ur::level_zero::v2::urVirtualMemMap; + pDdiTable->pfnUnmap = ur::level_zero::v2::urVirtualMemUnmap; + pDdiTable->pfnSetAccess = ur::level_zero::v2::urVirtualMemSetAccess; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urVirtualMemGetInfo; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceProcAddrTable( + ur_api_version_t version, ur_device_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnGet = ur::level_zero::v2::urDeviceGet; + pDdiTable->pfnGetInfo = ur::level_zero::v2::urDeviceGetInfo; + pDdiTable->pfnRetain = ur::level_zero::v2::urDeviceRetain; + pDdiTable->pfnRelease = ur::level_zero::v2::urDeviceRelease; + pDdiTable->pfnPartition = ur::level_zero::v2::urDevicePartition; + pDdiTable->pfnSelectBinary = ur::level_zero::v2::urDeviceSelectBinary; + pDdiTable->pfnGetNativeHandle = ur::level_zero::v2::urDeviceGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::level_zero::v2::urDeviceCreateWithNativeHandle; + pDdiTable->pfnGetGlobalTimestamps = + ur::level_zero::v2::urDeviceGetGlobalTimestamps; + + return result; +} + +UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceExpProcAddrTable( + ur_api_version_t version, ur_device_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnWaitExp = ur::level_zero::v2::urDeviceWaitExp; + + return result; +} + +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +} // namespace ur::level_zero::v2 +#else +} // extern "C" +#endif + +namespace { +ur_result_t populateDdiTable(ur_dditable_t *ddi) { + if (ddi == nullptr) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + ur_result_t result; + +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +#define NAMESPACE_ ::ur::level_zero::v2 +#else +#define NAMESPACE_ +#endif + + result = NAMESPACE_::urGetAdapterProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Adapter); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetBindlessImagesExpProcAddrTable( + UR_API_VERSION_CURRENT, &ddi->BindlessImagesExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetCommandBufferExpProcAddrTable( + UR_API_VERSION_CURRENT, &ddi->CommandBufferExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetContextProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Context); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEnqueueProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Enqueue); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEnqueueExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->EnqueueExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = + NAMESPACE_::urGetEventProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Event); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEventExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->EventExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetGraphExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->GraphExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetIPCExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->IPCExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetKernelProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Kernel); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetMemProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Mem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetMemoryExportExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->MemoryExportExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetPhysicalMemProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->PhysicalMem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetPlatformProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Platform); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetProgramProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Program); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetProgramExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->ProgramExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = + NAMESPACE_::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Queue); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetQueueExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->QueueExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetSamplerProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Sampler); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, &ddi->USM); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUSMExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->USMExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUsmP2PExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->UsmP2PExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetVirtualMemProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->VirtualMem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetDeviceProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Device); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetDeviceExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->DeviceExp); + if (result != UR_RESULT_SUCCESS) + return result; + +#undef NAMESPACE_ + + return result; +} +} // namespace + +namespace ur::level_zero::v2 { +const ur_dditable_t *ddi_getter::value() { + static std::once_flag flag; + static ur_dditable_t table; + + std::call_once(flag, []() { populateDdiTable(&table); }); + return &table; +} + +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) { + return populateDdiTable(ddi); +} +#endif +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.hpp new file mode 100644 index 0000000000000..8d941d067c653 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader.hpp @@ -0,0 +1,945 @@ +//===--------- ur_interface_loader.hpp - Level Zero Adapter ------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include + +namespace ur::level_zero::v2 { +ur_result_t urAdapterGet(uint32_t NumEntries, ::ur_adapter_handle_t *phAdapters, + uint32_t *pNumAdapters); +ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapter); +ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapter); +ur_result_t urAdapterGetLastError(::ur_adapter_handle_t hAdapter, + const char **ppMessage, int32_t *pError); +ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapter, + ur_adapter_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urAdapterSetLoggerCallback(::ur_adapter_handle_t hAdapter, + ur_logger_callback_t pfnLoggerCallback, + void *pUserData, + ur_logger_level_t level); +ur_result_t urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapter, + ur_logger_level_t level); +ur_result_t urPlatformGet(::ur_adapter_handle_t hAdapter, uint32_t NumEntries, + ::ur_platform_handle_t *phPlatforms, + uint32_t *pNumPlatforms); +ur_result_t urPlatformGetInfo(::ur_platform_handle_t hPlatform, + ur_platform_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urPlatformGetApiVersion(::ur_platform_handle_t hPlatform, + ur_api_version_t *pVersion); +ur_result_t urPlatformGetNativeHandle(::ur_platform_handle_t hPlatform, + ::ur_native_handle_t *phNativePlatform); +ur_result_t urPlatformCreateWithNativeHandle( + ::ur_native_handle_t hNativePlatform, ::ur_adapter_handle_t hAdapter, + const ur_platform_native_properties_t *pProperties, + ::ur_platform_handle_t *phPlatform); +ur_result_t urPlatformGetBackendOption(::ur_platform_handle_t hPlatform, + const char *pFrontendOption, + const char **ppPlatformOption); +ur_result_t urDeviceGet(::ur_platform_handle_t hPlatform, + ur_device_type_t DeviceType, uint32_t NumEntries, + ::ur_device_handle_t *phDevices, uint32_t *pNumDevices); +ur_result_t urDeviceGetInfo(::ur_device_handle_t hDevice, + ur_device_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urDeviceRetain(::ur_device_handle_t hDevice); +ur_result_t urDeviceRelease(::ur_device_handle_t hDevice); +ur_result_t +urDevicePartition(::ur_device_handle_t hDevice, + const ur_device_partition_properties_t *pProperties, + uint32_t NumDevices, ::ur_device_handle_t *phSubDevices, + uint32_t *pNumDevicesRet); +ur_result_t urDeviceSelectBinary(::ur_device_handle_t hDevice, + const ur_device_binary_t *pBinaries, + uint32_t NumBinaries, + uint32_t *pSelectedBinary); +ur_result_t urDeviceGetNativeHandle(::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeDevice); +ur_result_t +urDeviceCreateWithNativeHandle(::ur_native_handle_t hNativeDevice, + ::ur_adapter_handle_t hAdapter, + const ur_device_native_properties_t *pProperties, + ::ur_device_handle_t *phDevice); +ur_result_t urDeviceGetGlobalTimestamps(::ur_device_handle_t hDevice, + uint64_t *pDeviceTimestamp, + uint64_t *pHostTimestamp); +ur_result_t urContextCreate(uint32_t DeviceCount, + const ::ur_device_handle_t *phDevices, + const ur_context_properties_t *pProperties, + ::ur_context_handle_t *phContext); +ur_result_t urContextRetain(::ur_context_handle_t hContext); +ur_result_t urContextRelease(::ur_context_handle_t hContext); +ur_result_t urContextGetInfo(::ur_context_handle_t hContext, + ur_context_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urContextGetNativeHandle(::ur_context_handle_t hContext, + ::ur_native_handle_t *phNativeContext); +ur_result_t urContextCreateWithNativeHandle( + ::ur_native_handle_t hNativeContext, ::ur_adapter_handle_t hAdapter, + uint32_t numDevices, const ::ur_device_handle_t *phDevices, + const ur_context_native_properties_t *pProperties, + ::ur_context_handle_t *phContext); +ur_result_t +urContextSetExtendedDeleter(::ur_context_handle_t hContext, + ur_context_extended_deleter_t pfnDeleter, + void *pUserData); +ur_result_t urMemImageCreate(::ur_context_handle_t hContext, + ur_mem_flags_t flags, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, void *pHost, + ::ur_mem_handle_t *phMem); +ur_result_t urMemBufferCreate(::ur_context_handle_t hContext, + ur_mem_flags_t flags, size_t size, + const ur_buffer_properties_t *pProperties, + ::ur_mem_handle_t *phBuffer); +ur_result_t urMemRetain(::ur_mem_handle_t hMem); +ur_result_t urMemRelease(::ur_mem_handle_t hMem); +ur_result_t urMemBufferPartition(::ur_mem_handle_t hBuffer, + ur_mem_flags_t flags, + ur_buffer_create_type_t bufferCreateType, + const ur_buffer_region_t *pRegion, + ::ur_mem_handle_t *phMem); +ur_result_t urMemGetNativeHandle(::ur_mem_handle_t hMem, + ::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeMem); +ur_result_t urMemBufferCreateWithNativeHandle( + ::ur_native_handle_t hNativeMem, ::ur_context_handle_t hContext, + const ur_mem_native_properties_t *pProperties, ::ur_mem_handle_t *phMem); +ur_result_t urMemImageCreateWithNativeHandle( + ::ur_native_handle_t hNativeMem, ::ur_context_handle_t hContext, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_mem_native_properties_t *pProperties, ::ur_mem_handle_t *phMem); +ur_result_t urMemGetInfo(::ur_mem_handle_t hMemory, ur_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urMemImageGetInfo(::ur_mem_handle_t hMemory, + ur_image_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urSamplerCreate(::ur_context_handle_t hContext, + const ur_sampler_desc_t *pDesc, + ::ur_sampler_handle_t *phSampler); +ur_result_t urSamplerRetain(::ur_sampler_handle_t hSampler); +ur_result_t urSamplerRelease(::ur_sampler_handle_t hSampler); +ur_result_t urSamplerGetInfo(::ur_sampler_handle_t hSampler, + ur_sampler_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urSamplerGetNativeHandle(::ur_sampler_handle_t hSampler, + ::ur_native_handle_t *phNativeSampler); +ur_result_t urSamplerCreateWithNativeHandle( + ::ur_native_handle_t hNativeSampler, ::ur_context_handle_t hContext, + const ur_sampler_native_properties_t *pProperties, + ::ur_sampler_handle_t *phSampler); +ur_result_t urUSMHostAlloc(::ur_context_handle_t hContext, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMDeviceAlloc(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMSharedAlloc(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMFree(::ur_context_handle_t hContext, void *pMem); +ur_result_t urUSMGetMemAllocInfo(::ur_context_handle_t hContext, + const void *pMem, ur_usm_alloc_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urUSMPoolCreate(::ur_context_handle_t hContext, + ur_usm_pool_desc_t *pPoolDesc, + ::ur_usm_pool_handle_t *ppPool); +ur_result_t urUSMPoolRetain(::ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolRelease(::ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolGetInfo(::ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urVirtualMemGranularityGetInfo( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t allocationSize, ur_virtual_mem_granularity_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet); +ur_result_t urVirtualMemReserve(::ur_context_handle_t hContext, + const void *pStart, size_t size, + void **ppStart); +ur_result_t urVirtualMemFree(::ur_context_handle_t hContext, const void *pStart, + size_t size); +ur_result_t urVirtualMemMap(::ur_context_handle_t hContext, const void *pStart, + size_t size, + ::ur_physical_mem_handle_t hPhysicalMem, + size_t offset, ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContext, + const void *pStart, size_t size); +ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urPhysicalMemCreate(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, size_t size, + const ur_physical_mem_properties_t *pProperties, + ::ur_physical_mem_handle_t *phPhysicalMem); +ur_result_t urPhysicalMemRetain(::ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemRelease(::ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemGetInfo(::ur_physical_mem_handle_t hPhysicalMem, + ur_physical_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urProgramCreateWithIL(::ur_context_handle_t hContext, + const void *pIL, size_t length, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram); +ur_result_t +urProgramCreateWithBinary(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, size_t *pLengths, + const uint8_t **ppBinaries, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram); +ur_result_t urProgramBuild(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions); +ur_result_t urProgramCompile(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions); +ur_result_t urProgramLink(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms, + const char *pOptions, + ::ur_program_handle_t *phProgram); +ur_result_t urProgramRetain(::ur_program_handle_t hProgram); +ur_result_t urProgramRelease(::ur_program_handle_t hProgram); +ur_result_t urProgramGetFunctionPointer(::ur_device_handle_t hDevice, + ::ur_program_handle_t hProgram, + const char *pFunctionName, + void **ppFunctionPointer); +ur_result_t urProgramGetGlobalVariablePointer( + ::ur_device_handle_t hDevice, ::ur_program_handle_t hProgram, + const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, + void **ppGlobalVariablePointerRet); +ur_result_t urProgramGetInfo(::ur_program_handle_t hProgram, + ur_program_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urProgramGetBuildInfo(::ur_program_handle_t hProgram, + ::ur_device_handle_t hDevice, + ur_program_build_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urProgramSetSpecializationConstants( + ::ur_program_handle_t hProgram, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants); +ur_result_t urProgramGetNativeHandle(::ur_program_handle_t hProgram, + ::ur_native_handle_t *phNativeProgram); +ur_result_t urProgramCreateWithNativeHandle( + ::ur_native_handle_t hNativeProgram, ::ur_context_handle_t hContext, + const ur_program_native_properties_t *pProperties, + ::ur_program_handle_t *phProgram); +ur_result_t urQueueGetInfo(::ur_queue_handle_t hQueue, ur_queue_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urQueueCreate(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_queue_properties_t *pProperties, + ::ur_queue_handle_t *phQueue); +ur_result_t urQueueRetain(::ur_queue_handle_t hQueue); +ur_result_t urQueueRelease(::ur_queue_handle_t hQueue); +ur_result_t urQueueGetNativeHandle(::ur_queue_handle_t hQueue, + ur_queue_native_desc_t *pDesc, + ::ur_native_handle_t *phNativeQueue); +ur_result_t +urQueueCreateWithNativeHandle(::ur_native_handle_t hNativeQueue, + ::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_queue_native_properties_t *pProperties, + ::ur_queue_handle_t *phQueue); +ur_result_t urQueueFinish(::ur_queue_handle_t hQueue); +ur_result_t urQueueFlush(::ur_queue_handle_t hQueue); +ur_result_t urEventGetInfo(::ur_event_handle_t hEvent, ur_event_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urEventGetProfilingInfo(::ur_event_handle_t hEvent, + ur_profiling_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urEventWait(uint32_t numEvents, + const ::ur_event_handle_t *phEventWaitList); +ur_result_t urEventRetain(::ur_event_handle_t hEvent); +ur_result_t urEventRelease(::ur_event_handle_t hEvent); +ur_result_t urEventGetNativeHandle(::ur_event_handle_t hEvent, + ::ur_native_handle_t *phNativeEvent); +ur_result_t +urEventCreateWithNativeHandle(::ur_native_handle_t hNativeEvent, + ::ur_context_handle_t hContext, + const ur_event_native_properties_t *pProperties, + ::ur_event_handle_t *phEvent); +ur_result_t urEventSetCallback(::ur_event_handle_t hEvent, + ur_execution_info_t execStatus, + ur_event_callback_t pfnNotify, void *pUserData); +ur_result_t urEnqueueEventsWait(::ur_queue_handle_t hQueue, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueEventsWaitWithBarrier( + ::ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferRead(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingRead, + size_t offset, size_t size, void *pDst, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferWrite( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingWrite, + size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferReadRect( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingRead, + ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pDst, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferWriteRect( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBuffer, bool blockingWrite, + ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferCopy(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, + size_t srcOffset, size_t dstOffset, + size_t size, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferCopyRect( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hBufferSrc, + ::ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, + size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferFill(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, + const void *pPattern, size_t patternSize, + size_t offset, size_t size, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemImageRead( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImage, bool blockingRead, + ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemImageWrite( + ::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImage, bool blockingWrite, + ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t +urEnqueueMemImageCopy(::ur_queue_handle_t hQueue, ::ur_mem_handle_t hImageSrc, + ::ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferMap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hBuffer, bool blockingMap, + ur_map_flags_t mapFlags, size_t offset, + size_t size, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent, + void **ppRetMap); +ur_result_t urEnqueueMemUnmap(::ur_queue_handle_t hQueue, + ::ur_mem_handle_t hMem, void *pMappedPtr, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill(::ur_queue_handle_t hQueue, void *pMem, + size_t patternSize, const void *pPattern, + size_t size, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy(::ur_queue_handle_t hQueue, bool blocking, + void *pDst, const void *pSrc, size_t size, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMPrefetch(::ur_queue_handle_t hQueue, const void *pMem, + size_t size, ur_usm_migration_flags_t flags, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMAdvise(::ur_queue_handle_t hQueue, const void *pMem, + size_t size, ur_usm_advice_flags_t advice, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill2D(::ur_queue_handle_t hQueue, void *pMem, + size_t pitch, size_t patternSize, + const void *pPattern, size_t width, + size_t height, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy2D(::ur_queue_handle_t hQueue, bool blocking, + void *pDst, size_t dstPitch, const void *pSrc, + size_t srcPitch, size_t width, size_t height, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueDeviceGlobalVariableWrite( + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingWrite, size_t count, size_t offset, + const void *pSrc, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueDeviceGlobalVariableRead( + ::ur_queue_handle_t hQueue, ::ur_program_handle_t hProgram, + const char *name, bool blockingRead, size_t count, size_t offset, + void *pDst, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueReadHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, + const char *pipe_symbol, bool blocking, + void *pDst, size_t size, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueWriteHostPipe(::ur_queue_handle_t hQueue, + ::ur_program_handle_t hProgram, + const char *pipe_symbol, bool blocking, + void *pSrc, size_t size, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueKernelLaunchWithArgsExp( + ::ur_queue_handle_t hQueue, ::ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, + const ur_kernel_launch_ext_properties_t *launchPropList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urKernelCreate(::ur_program_handle_t hProgram, + const char *pKernelName, + ::ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetInfo(::ur_kernel_handle_t hKernel, + ur_kernel_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urKernelGetGroupInfo(::ur_kernel_handle_t hKernel, + ::ur_device_handle_t hDevice, + ur_kernel_group_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urKernelGetSubGroupInfo(::ur_kernel_handle_t hKernel, + ::ur_device_handle_t hDevice, + ur_kernel_sub_group_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urKernelRetain(::ur_kernel_handle_t hKernel); +ur_result_t urKernelRelease(::ur_kernel_handle_t hKernel); +ur_result_t +urKernelSetExecInfo(::ur_kernel_handle_t hKernel, + ur_kernel_exec_info_t propName, size_t propSize, + const ur_kernel_exec_info_properties_t *pProperties, + const void *pPropValue); +ur_result_t urKernelSetSpecializationConstants( + ::ur_kernel_handle_t hKernel, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants); +ur_result_t urKernelGetNativeHandle(::ur_kernel_handle_t hKernel, + ::ur_native_handle_t *phNativeKernel); +ur_result_t +urKernelCreateWithNativeHandle(::ur_native_handle_t hNativeKernel, + ::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const ur_kernel_native_properties_t *pProperties, + ::ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetSuggestedLocalWorkSize(::ur_kernel_handle_t hKernel, + ::ur_queue_handle_t hQueue, + uint32_t numWorkDim, + const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, + size_t *pSuggestedLocalWorkSize); +ur_result_t urKernelGetSuggestedLocalWorkSizeWithArgs( + ::ur_kernel_handle_t hKernel, ::ur_queue_handle_t hQueue, + uint32_t numWorkDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, + size_t *pSuggestedLocalWorkSize); +ur_result_t urKernelSuggestMaxCooperativeGroupCount( + ::ur_kernel_handle_t hKernel, ::ur_device_handle_t hDevice, + uint32_t workDim, const size_t *pLocalWorkSize, + size_t dynamicSharedMemorySize, uint32_t *pGroupCountRet); +ur_result_t urEnqueueUSMDeviceAllocExp( + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMSharedAllocExp( + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMHostAllocExp( + ::ur_queue_handle_t hQueue, ::ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + void **ppMem, ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFreeExp(::ur_queue_handle_t hQueue, + ::ur_usm_pool_handle_t pPool, void *pMem, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urUSMPoolCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_usm_pool_desc_t *pPoolDesc, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolDestroyExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDefaultDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolGetInfoExp(::ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urUSMPoolSetInfoExp(::ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, void *pPropValue, + size_t propSize); +ur_result_t urUSMPoolSetDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDevicePoolExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolTrimToExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_usm_pool_handle_t hPool, + size_t minBytesToKeep); +ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch); +ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesImageAllocateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesImageFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem); +ur_result_t urBindlessImagesUnsampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_image_native_handle_t *phImage); +ur_result_t urBindlessImagesSampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_sampler_desc_t *pSamplerDesc, + ::ur_exp_image_native_handle_t *phImage); +ur_result_t urBindlessImagesImageCopyExp( + ::ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, + ur_exp_image_copy_input_types_t imageCopyInputTypes, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t +urBindlessImagesImageGetInfoExp(::ur_context_handle_t hContext, + ::ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesMipmapGetLevelExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ::ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ::ur_exp_image_mem_native_handle_t hMem); +ur_result_t urBindlessImagesImportExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, size_t size, + ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ::ur_exp_external_mem_handle_t *phExternalMem); +ur_result_t urBindlessImagesMapExternalArrayExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_external_mem_handle_t hExternalMem, + ::ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + uint64_t offset, uint64_t size, ::ur_exp_external_mem_handle_t hExternalMem, + void **ppRetMem); +ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_mem_handle_t hExternalMem); +ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem); +ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( + ::ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, + ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ::ur_exp_external_semaphore_handle_t *phExternalSemaphore); +ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_semaphore_handle_t hExternalSemaphore); +ur_result_t urBindlessImagesWaitExternalSemaphoreExp( + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, + bool hasWaitValue, uint64_t waitValue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urBindlessImagesSignalExternalSemaphoreExp( + ::ur_queue_handle_t hQueue, ::ur_exp_external_semaphore_handle_t hSemaphore, + bool hasSignalValue, uint64_t signalValue, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urDeviceWaitExp(::ur_device_handle_t hDevice); +ur_result_t urProgramDynamicLinkExp(::ur_context_handle_t hContext, + uint32_t count, + const ::ur_program_handle_t *phPrograms); +ur_result_t urEnqueueTimestampRecordingExp( + ::ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, ::ur_event_handle_t *phEvent); +ur_result_t urIPCGetMemHandleExp(::ur_context_handle_t hContext, void *pMem, + void **ppIPCMemHandleData, + size_t *pIPCMemHandleDataSizeRet); +ur_result_t urIPCPutMemHandleExp(::ur_context_handle_t hContext, + void *pIPCMemHandleData); +ur_result_t urIPCOpenMemHandleExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + void *pIPCMemHandleData, + size_t ipcMemHandleDataSize, void **ppMem); +ur_result_t urIPCCloseMemHandleExp(::ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet); +ur_result_t urIPCPutPhysMemHandleExp(::ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData); +ur_result_t urIPCOpenPhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ::ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(::ur_context_handle_t hContext, + ::ur_physical_mem_handle_t hPhysMem); +ur_result_t urIPCGetEventHandleExp(::ur_event_handle_t hEvent, + void **ppIPCEventHandleData, + size_t *pIPCEventHandleDataSizeRet); +ur_result_t urIPCPutEventHandleExp(::ur_context_handle_t hContext, + void *pIPCEventHandleData); +ur_result_t urIPCOpenEventHandleExp(::ur_context_handle_t hContext, + const void *pIPCEventHandleData, + size_t ipcEventHandleDataSize, + ::ur_event_handle_t *phEvent); +ur_result_t urMemoryExportAllocExportableMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t +urMemoryExportFreeExportableMemoryExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, void *pMem); +ur_result_t urMemoryExportExportMemoryHandleExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet); +ur_result_t urProgramBuildExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramCompileExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t +urProgramLinkExp(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, + uint32_t count, const ::ur_program_handle_t *phPrograms, + const char *pOptions, ::ur_program_handle_t *phProgram); +ur_result_t urUSMContextMemcpyExp(::ur_context_handle_t hContext, void *pDst, + const void *pSrc, size_t size); +ur_result_t urUSMHostAllocRegisterExp( + ::ur_context_handle_t hContext, void *pHostMem, size_t size, + const ur_exp_usm_host_alloc_register_properties_t *pProperties); +ur_result_t urUSMHostAllocUnregisterExp(::ur_context_handle_t hContext, + void *pHostMem); +ur_result_t urUSMImportExp(::ur_context_handle_t hContext, void *pMem, + size_t size); +ur_result_t urUSMReleaseExp(::ur_context_handle_t hContext, void *pMem); +ur_result_t urUsmP2PEnablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice); +ur_result_t urUsmP2PDisablePeerAccessExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice); +ur_result_t urUsmP2PPeerAccessGetInfoExp(::ur_device_handle_t commandDevice, + ::ur_device_handle_t peerDevice, + ur_exp_peer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t +urCommandBufferCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + ::ur_exp_command_buffer_handle_t *phCommandBuffer); +ur_result_t +urCommandBufferRetainExp(::ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t +urCommandBufferReleaseExp(::ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t +urCommandBufferFinalizeExp(::ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t urCommandBufferAppendKernelLaunchExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternatives, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, uint32_t numKernelAlternatives, + ::ur_kernel_handle_t *phKernelAlternatives, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMMemcpyExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, + const void *pSrc, size_t size, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMFillExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory, + const void *pPattern, size_t patternSize, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferCopyExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hSrcMem, + ::ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferWriteExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, + size_t offset, size_t size, const void *pSrc, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferReadExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, + size_t offset, size_t size, void *pDst, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferCopyRectExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hSrcMem, + ::ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, + size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferWriteRectExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, + ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferReadRectExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, + ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pDst, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferFillExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, ::ur_mem_handle_t hBuffer, + const void *pPattern, size_t patternSize, size_t offset, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMPrefetchExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + size_t size, ur_usm_migration_flags_t flags, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMAdviseExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + size_t size, ur_usm_advice_flags_t advice, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, + ::ur_event_handle_t *phEvent, + ::ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendNativeCommandExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + void *pData, ::ur_exp_command_buffer_handle_t hChildCommandBuffer, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint); +ur_result_t urEnqueueCommandBufferExp( + ::ur_queue_handle_t hQueue, ::ur_exp_command_buffer_handle_t hCommandBuffer, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urCommandBufferUpdateKernelLaunchExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates, + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch); +ur_result_t urCommandBufferUpdateSignalEventExp( + ::ur_exp_command_buffer_command_handle_t hCommand, + ::ur_event_handle_t *phSignalEvent); +ur_result_t urCommandBufferUpdateWaitEventsExp( + ::ur_exp_command_buffer_command_handle_t hCommand, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList); +ur_result_t +urCommandBufferGetInfoExp(::ur_exp_command_buffer_handle_t hCommandBuffer, + ur_exp_command_buffer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urCommandBufferGetNativeHandleExp( + ::ur_exp_command_buffer_handle_t hCommandBuffer, + ::ur_native_handle_t *phNativeCommandBuffer); +ur_result_t urEnqueueHostTaskExp( + ::ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, + void *data, const ur_exp_host_task_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueEventsWaitWithBarrierExt( + ::ur_queue_handle_t hQueue, + const ur_exp_enqueue_ext_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEnqueueNativeCommandExp( + ::ur_queue_handle_t hQueue, + ur_exp_enqueue_native_command_function_t pfnNativeEnqueue, void *data, + uint32_t numMemsInMemList, const ::ur_mem_handle_t *phMemList, + const ur_exp_enqueue_native_command_properties_t *pProperties, + uint32_t numEventsInWaitList, const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urEventCreateExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_exp_event_desc_t *pEventDesc, + ::ur_event_handle_t *phEvent); +ur_result_t urGraphCreateExp(::ur_context_handle_t hContext, + ::ur_exp_graph_handle_t *phGraph); +ur_result_t urQueueBeginGraphCaptureExp(::ur_queue_handle_t hQueue); +ur_result_t urQueueBeginCaptureIntoGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t hGraph); +ur_result_t urQueueEndGraphCaptureExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph); +ur_result_t +urGraphInstantiateGraphExp(::ur_exp_graph_handle_t hGraph, + ::ur_exp_executable_graph_handle_t *phExecGraph); +ur_result_t urEnqueueGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_executable_graph_handle_t hGraph, + uint32_t numEventsInWaitList, + const ::ur_event_handle_t *phEventWaitList, + ::ur_event_handle_t *phEvent); +ur_result_t urGraphDestroyExp(::ur_exp_graph_handle_t hGraph); +ur_result_t urGraphExecutableGraphDestroyExp( + ::ur_exp_executable_graph_handle_t hExecutableGraph); +ur_result_t urQueueIsGraphCaptureEnabledExp(::ur_queue_handle_t hQueue, + bool *pResult); +ur_result_t urQueueGetGraphExp(::ur_queue_handle_t hQueue, + ::ur_exp_graph_handle_t *phGraph); +ur_result_t urGraphIsEmptyExp(::ur_exp_graph_handle_t hGraph, bool *pResult); +ur_result_t urGraphGetIdExp(::ur_exp_graph_handle_t hGraph, uint64_t *pGraphId); +ur_result_t urGraphSetDestructionCallbackExp( + ::ur_exp_graph_handle_t hGraph, + ur_exp_graph_destruction_callback_t pfnCallback, void *pUserData); +ur_result_t urGraphDumpContentsExp(::ur_exp_graph_handle_t hGraph, + const char *filePath); +ur_result_t urGraphGetNativeHandleExp(::ur_exp_graph_handle_t hGraph, + ::ur_native_handle_t *phNativeGraph); +ur_result_t urGraphExecutableGraphGetNativeHandleExp( + ::ur_exp_executable_graph_handle_t hExecutableGraph, + ::ur_native_handle_t *phNativeExecutableGraph); +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); +#endif + +struct ddi_getter { + const static ur_dditable_t *value(); +}; +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader_common_forwarders.hpp b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader_common_forwarders.hpp new file mode 100644 index 0000000000000..0b8bbf345d018 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/ur_interface_loader_common_forwarders.hpp @@ -0,0 +1,496 @@ +//===--------- ur_interface_loader_common_forwarders.hpp - L0 Adapter ----===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Several official UR entry points are implemented once, in the common +// `ur::level_zero` namespace (see common/api.hpp). The generated interface +// loader declares every entry point inside `ur::level_zero::v2`. For the common +// entry points those `ur::level_zero::v2` symbols would otherwise be undefined. + +#pragma once + +#include + +#include "../common/api.hpp" +#include "ur_interface_loader.hpp" + +namespace ur::level_zero::v2 { + +inline ur_result_t urDeviceGet(::ur_platform_handle_t hPlatform, + ::ur_device_type_t DeviceType, + uint32_t NumEntries, + ::ur_device_handle_t *phDevices, + uint32_t *pNumDevices) { + return ::ur::level_zero::urDeviceGet(hPlatform, DeviceType, NumEntries, + phDevices, pNumDevices); +} +inline ur_result_t urDeviceRetain(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceRetain(hDevice); +} +inline ur_result_t urDeviceRelease(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceRelease(hDevice); +} +inline ur_result_t +urDevicePartition(::ur_device_handle_t hDevice, + const ::ur_device_partition_properties_t *pProperties, + uint32_t NumDevices, ::ur_device_handle_t *phSubDevices, + uint32_t *pNumDevicesRet) { + return ::ur::level_zero::urDevicePartition(hDevice, pProperties, NumDevices, + phSubDevices, pNumDevicesRet); +} +inline ur_result_t urDeviceSelectBinary(::ur_device_handle_t hDevice, + const ::ur_device_binary_t *pBinaries, + uint32_t NumBinaries, + uint32_t *pSelectedBinary) { + return ::ur::level_zero::urDeviceSelectBinary(hDevice, pBinaries, NumBinaries, + pSelectedBinary); +} +inline ur_result_t +urDeviceGetNativeHandle(::ur_device_handle_t hDevice, + ::ur_native_handle_t *phNativeDevice) { + return ::ur::level_zero::urDeviceGetNativeHandle(hDevice, phNativeDevice); +} +inline ur_result_t urDeviceCreateWithNativeHandle( + ::ur_native_handle_t hNativeDevice, ::ur_adapter_handle_t hAdapter, + const ::ur_device_native_properties_t *pProperties, + ::ur_device_handle_t *phDevice) { + return ::ur::level_zero::urDeviceCreateWithNativeHandle( + hNativeDevice, hAdapter, pProperties, phDevice); +} +inline ur_result_t urDeviceGetGlobalTimestamps(::ur_device_handle_t hDevice, + uint64_t *pDeviceTimestamp, + uint64_t *pHostTimestamp) { + return ::ur::level_zero::urDeviceGetGlobalTimestamps( + hDevice, pDeviceTimestamp, pHostTimestamp); +} +inline ur_result_t urDeviceWaitExp(::ur_device_handle_t hDevice) { + return ::ur::level_zero::urDeviceWaitExp(hDevice); +} +inline ur_result_t urPlatformGet(::ur_adapter_handle_t hAdapter, + uint32_t NumEntries, + ::ur_platform_handle_t *phPlatforms, + uint32_t *pNumPlatforms) { + return ::ur::level_zero::urPlatformGet(hAdapter, NumEntries, phPlatforms, + pNumPlatforms); +} +inline ur_result_t urPlatformGetApiVersion(::ur_platform_handle_t hPlatform, + ur_api_version_t *pVersion) { + return ::ur::level_zero::urPlatformGetApiVersion(hPlatform, pVersion); +} +inline ur_result_t +urPlatformGetNativeHandle(::ur_platform_handle_t hPlatform, + ::ur_native_handle_t *phNativePlatform) { + return ::ur::level_zero::urPlatformGetNativeHandle(hPlatform, + phNativePlatform); +} +inline ur_result_t urPlatformCreateWithNativeHandle( + ::ur_native_handle_t hNativePlatform, ::ur_adapter_handle_t hAdapter, + const ur_platform_native_properties_t *pProperties, + ::ur_platform_handle_t *phPlatform) { + return ::ur::level_zero::urPlatformCreateWithNativeHandle( + hNativePlatform, hAdapter, pProperties, phPlatform); +} +inline ur_result_t urPlatformGetBackendOption(::ur_platform_handle_t hPlatform, + const char *pFrontendOption, + const char **ppPlatformOption) { + return ::ur::level_zero::urPlatformGetBackendOption( + hPlatform, pFrontendOption, ppPlatformOption); +} +inline ur_result_t urAdapterRelease(::ur_adapter_handle_t hAdapter) { + return ::ur::level_zero::urAdapterRelease(hAdapter); +} +inline ur_result_t urAdapterRetain(::ur_adapter_handle_t hAdapter) { + return ::ur::level_zero::urAdapterRetain(hAdapter); +} +inline ur_result_t urAdapterGetLastError(::ur_adapter_handle_t hAdapter, + const char **ppMessage, + int32_t *pError) { + return ::ur::level_zero::urAdapterGetLastError(hAdapter, ppMessage, pError); +} +inline ur_result_t urAdapterGetInfo(::ur_adapter_handle_t hAdapter, + ur_adapter_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urAdapterGetInfo(hAdapter, propName, propSize, + pPropValue, pPropSizeRet); +} +inline ur_result_t +urAdapterSetLoggerCallback(::ur_adapter_handle_t hAdapter, + ur_logger_callback_t pfnLoggerCallback, + void *pUserData, ur_logger_level_t level) { + return ::ur::level_zero::urAdapterSetLoggerCallback( + hAdapter, pfnLoggerCallback, pUserData, level); +} +inline ur_result_t +urAdapterSetLoggerCallbackLevel(::ur_adapter_handle_t hAdapter, + ur_logger_level_t level) { + return ::ur::level_zero::urAdapterSetLoggerCallbackLevel(hAdapter, level); +} +inline ur_result_t urSamplerCreate(::ur_context_handle_t Context, + const ur_sampler_desc_t *Props, + ::ur_sampler_handle_t *Sampler) { + return ::ur::level_zero::urSamplerCreate(Context, Props, Sampler); +} +inline ur_result_t urSamplerRetain(::ur_sampler_handle_t Sampler) { + return ::ur::level_zero::urSamplerRetain(Sampler); +} +inline ur_result_t urSamplerRelease(::ur_sampler_handle_t Sampler) { + return ::ur::level_zero::urSamplerRelease(Sampler); +} +inline ur_result_t urSamplerGetInfo(::ur_sampler_handle_t Sampler, + ur_sampler_info_t PropName, + size_t PropValueSize, void *PropValue, + size_t *PropSizeRet) { + return ::ur::level_zero::urSamplerGetInfo(Sampler, PropName, PropValueSize, + PropValue, PropSizeRet); +} +inline ur_result_t +urSamplerGetNativeHandle(::ur_sampler_handle_t Sampler, + ::ur_native_handle_t *NativeSampler) { + return ::ur::level_zero::urSamplerGetNativeHandle(Sampler, NativeSampler); +} +inline ur_result_t urSamplerCreateWithNativeHandle( + ::ur_native_handle_t NativeSampler, ::ur_context_handle_t Context, + const ur_sampler_native_properties_t *Properties, + ::ur_sampler_handle_t *Sampler) { + return ::ur::level_zero::urSamplerCreateWithNativeHandle( + NativeSampler, Context, Properties, Sampler); +} +inline ur_result_t +urProgramCreateWithIL(::ur_context_handle_t hContext, const void *pIL, + size_t length, const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithIL(hContext, pIL, length, + pProperties, phProgram); +} +inline ur_result_t +urProgramCreateWithBinary(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, size_t *pLengths, + const uint8_t **ppBinaries, + const ur_program_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithBinary( + hContext, numDevices, phDevices, pLengths, ppBinaries, pProperties, + phProgram); +} +inline ur_result_t urProgramBuild(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions) { + return ::ur::level_zero::urProgramBuild(hContext, hProgram, pOptions); +} +inline ur_result_t urProgramCompile(::ur_context_handle_t hContext, + ::ur_program_handle_t hProgram, + const char *pOptions) { + return ::ur::level_zero::urProgramCompile(hContext, hProgram, pOptions); +} +inline ur_result_t urProgramLink(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms, + const char *pOptions, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramLink(hContext, count, phPrograms, pOptions, + phProgram); +} +inline ur_result_t urProgramRetain(::ur_program_handle_t hProgram) { + return ::ur::level_zero::urProgramRetain(hProgram); +} +inline ur_result_t urProgramRelease(::ur_program_handle_t hProgram) { + return ::ur::level_zero::urProgramRelease(hProgram); +} +inline ur_result_t urProgramGetFunctionPointer(::ur_device_handle_t hDevice, + ::ur_program_handle_t hProgram, + const char *pFunctionName, + void **ppFunctionPointer) { + return ::ur::level_zero::urProgramGetFunctionPointer( + hDevice, hProgram, pFunctionName, ppFunctionPointer); +} +inline ur_result_t urProgramGetGlobalVariablePointer( + ::ur_device_handle_t hDevice, ::ur_program_handle_t hProgram, + const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, + void **ppGlobalVariablePointerRet) { + return ::ur::level_zero::urProgramGetGlobalVariablePointer( + hDevice, hProgram, pGlobalVariableName, pGlobalVariableSizeRet, + ppGlobalVariablePointerRet); +} +inline ur_result_t urProgramGetInfo(::ur_program_handle_t hProgram, + ur_program_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urProgramGetInfo(hProgram, propName, propSize, + pPropValue, pPropSizeRet); +} +inline ur_result_t urProgramGetBuildInfo(::ur_program_handle_t hProgram, + ::ur_device_handle_t hDevice, + ur_program_build_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + return ::ur::level_zero::urProgramGetBuildInfo( + hProgram, hDevice, propName, propSize, pPropValue, pPropSizeRet); +} +inline ur_result_t urProgramSetSpecializationConstants( + ::ur_program_handle_t hProgram, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants) { + return ::ur::level_zero::urProgramSetSpecializationConstants(hProgram, count, + pSpecConstants); +} +inline ur_result_t +urProgramGetNativeHandle(::ur_program_handle_t hProgram, + ::ur_native_handle_t *phNativeProgram) { + return ::ur::level_zero::urProgramGetNativeHandle(hProgram, phNativeProgram); +} +inline ur_result_t urProgramCreateWithNativeHandle( + ::ur_native_handle_t hNativeProgram, ::ur_context_handle_t hContext, + const ur_program_native_properties_t *pProperties, + ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramCreateWithNativeHandle( + hNativeProgram, hContext, pProperties, phProgram); +} +inline ur_result_t +urProgramDynamicLinkExp(::ur_context_handle_t hContext, uint32_t count, + const ::ur_program_handle_t *phPrograms) { + return ::ur::level_zero::urProgramDynamicLinkExp(hContext, count, phPrograms); +} +inline ur_result_t urProgramBuildExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions) { + return ::ur::level_zero::urProgramBuildExp(hProgram, numDevices, phDevices, + flags, pOptions); +} +inline ur_result_t urProgramCompileExp(::ur_program_handle_t hProgram, + uint32_t numDevices, + ::ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions) { + return ::ur::level_zero::urProgramCompileExp(hProgram, numDevices, phDevices, + flags, pOptions); +} +inline ur_result_t +urProgramLinkExp(::ur_context_handle_t hContext, uint32_t numDevices, + ::ur_device_handle_t *phDevices, ur_exp_program_flags_t flags, + uint32_t count, const ::ur_program_handle_t *phPrograms, + const char *pOptions, ::ur_program_handle_t *phProgram) { + return ::ur::level_zero::urProgramLinkExp(hContext, numDevices, phDevices, + flags, count, phPrograms, pOptions, + phProgram); +} +inline ur_result_t urVirtualMemGranularityGetInfo( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t allocationSize, ur_virtual_mem_granularity_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urVirtualMemGranularityGetInfo( + hContext, hDevice, allocationSize, propName, propSize, pPropValue, + pPropSizeRet); +} +inline ur_result_t urVirtualMemReserve(::ur_context_handle_t hContext, + const void *pStart, size_t size, + void **ppStart) { + return ::ur::level_zero::urVirtualMemReserve(hContext, pStart, size, ppStart); +} +inline ur_result_t urVirtualMemFree(::ur_context_handle_t hContext, + const void *pStart, size_t size) { + return ::ur::level_zero::urVirtualMemFree(hContext, pStart, size); +} +inline ur_result_t urVirtualMemMap(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ::ur_physical_mem_handle_t hPhysicalMem, + size_t offset, + ur_virtual_mem_access_flags_t flags) { + return ::ur::level_zero::urVirtualMemMap(hContext, pStart, size, hPhysicalMem, + offset, flags); +} +inline ur_result_t urVirtualMemUnmap(::ur_context_handle_t hContext, + const void *pStart, size_t size) { + return ::ur::level_zero::urVirtualMemUnmap(hContext, pStart, size); +} +inline ur_result_t urVirtualMemSetAccess(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_access_flags_t flags) { + return ::ur::level_zero::urVirtualMemSetAccess(hContext, pStart, size, flags); +} +inline ur_result_t urVirtualMemGetInfo(::ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + return ::ur::level_zero::urVirtualMemGetInfo( + hContext, pStart, size, propName, propSize, pPropValue, pPropSizeRet); +} +inline ur_result_t urMemoryExportAllocExportableMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + size_t alignment, size_t size, + ur_exp_external_mem_type_t handleTypeToExport, void **ppMem) { + return ::ur::level_zero::urMemoryExportAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); +} +inline ur_result_t urMemoryExportFreeExportableMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem) { + return ::ur::level_zero::urMemoryExportFreeExportableMemoryExp(hContext, + hDevice, pMem); +} +inline ur_result_t urMemoryExportExportMemoryHandleExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet) { + return ::ur::level_zero::urMemoryExportExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); +} +inline ur_result_t urUSMPitchedAllocExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ::ur_usm_pool_handle_t pool, + size_t widthInBytes, size_t height, + size_t elementSizeBytes, void **ppMem, + size_t *pResultPitch) { + return ::ur::level_zero::urUSMPitchedAllocExp( + hContext, hDevice, pUSMDesc, pool, widthInBytes, height, elementSizeBytes, + ppMem, pResultPitch); +} +inline ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + return ::ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp( + hContext, hDevice, hImage); +} +inline ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + return ::ur::level_zero::urBindlessImagesSampledImageHandleDestroyExp( + hContext, hDevice, hImage); +} +inline ur_result_t urBindlessImagesImageAllocateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesImageAllocateExp( + hContext, hDevice, pImageFormat, pImageDesc, phImageMem); +} +inline ur_result_t +urBindlessImagesImageFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem) { + return ::ur::level_zero::urBindlessImagesImageFreeExp(hContext, hDevice, + hImageMem); +} +inline ur_result_t urBindlessImagesUnsampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_native_handle_t *phImage) { + return ::ur::level_zero::urBindlessImagesUnsampledImageCreateExp( + hContext, hDevice, hImageMem, pImageFormat, pImageDesc, phImage); +} +inline ur_result_t urBindlessImagesSampledImageCreateExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_sampler_desc_t *pSamplerDesc, + ur_exp_image_native_handle_t *phImage) { + return ::ur::level_zero::urBindlessImagesSampledImageCreateExp( + hContext, hDevice, hImageMem, pImageFormat, pImageDesc, pSamplerDesc, + phImage); +} +inline ur_result_t urBindlessImagesImageGetInfoExp( + ::ur_context_handle_t hContext, ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { + return ::ur::level_zero::urBindlessImagesImageGetInfoExp( + hContext, hImageMem, propName, pPropValue, pPropSizeRet); +} +inline ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageMemoryHandleTypeSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageUnsampledHandleSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesGetImageSampledHandleSupportExp( + hContext, hDevice, pImageDesc, pImageFormat, imageMemHandleType, + pSupportedRet); +} +inline ur_result_t urBindlessImagesMipmapGetLevelExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesMipmapGetLevelExp( + hContext, hDevice, hImageMem, mipmapLevel, phImageMem); +} +inline ur_result_t urBindlessImagesImportExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, size_t size, + ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ::ur_exp_external_mem_handle_t *phExternalMem) { + return ::ur::level_zero::urBindlessImagesImportExternalMemoryExp( + hContext, hDevice, size, memHandleType, pExternalMemDesc, phExternalMem); +} +inline ur_result_t urBindlessImagesMapExternalArrayExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ::ur_exp_external_mem_handle_t hExternalMem, + ur_exp_image_mem_native_handle_t *phImageMem) { + return ::ur::level_zero::urBindlessImagesMapExternalArrayExp( + hContext, hDevice, pImageFormat, pImageDesc, hExternalMem, phImageMem); +} +inline ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + uint64_t offset, uint64_t size, ::ur_exp_external_mem_handle_t hExternalMem, + void **ppRetMem) { + return ::ur::level_zero::urBindlessImagesMapExternalLinearMemoryExp( + hContext, hDevice, offset, size, hExternalMem, ppRetMem); +} +inline ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_mem_handle_t hExternalMem) { + return ::ur::level_zero::urBindlessImagesReleaseExternalMemoryExp( + hContext, hDevice, hExternalMem); +} +inline ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, void *pMem) { + return ::ur::level_zero::urBindlessImagesFreeMappedLinearMemoryExp( + hContext, hDevice, pMem); +} +inline ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( + ::ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, + ur_bool_t *pSupportedRet) { + return ::ur::level_zero::urBindlessImagesSupportsImportingHandleTypeExp( + hDevice, memHandleType, pSupportedRet); +} +inline ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ::ur_exp_external_semaphore_handle_t *phExternalSemaphore) { + return ::ur::level_zero::urBindlessImagesImportExternalSemaphoreExp( + hContext, hDevice, semHandleType, pExternalSemaphoreDesc, + phExternalSemaphore); +} +inline ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ::ur_context_handle_t hContext, ::ur_device_handle_t hDevice, + ::ur_exp_external_semaphore_handle_t hExternalSemaphore) { + return ::ur::level_zero::urBindlessImagesReleaseExternalSemaphoreExp( + hContext, hDevice, hExternalSemaphore); +} +inline ur_result_t +urBindlessImagesMipmapFreeExp(::ur_context_handle_t hContext, + ::ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hMem) { + return ::ur::level_zero::urBindlessImagesMipmapFreeExp(hContext, hDevice, + hMem); +} + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index ead2ca4a95260..cd0147218d829 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -9,7 +9,8 @@ #include "unified-runtime/ur_api.h" -#include "../device.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" #include "context.hpp" #include "umf_pools/disjoint_pool_config_parser.hpp" #include "usm.hpp" @@ -17,6 +18,8 @@ #include #include +namespace ur::level_zero::v2 { + static inline void UMF_CALL_THROWS(umf_result_t res) { if (res != UMF_RESULT_SUCCESS) { UR_DFAILURE("some umf call in v2 L0 adapter returned " @@ -25,21 +28,6 @@ static inline void UMF_CALL_THROWS(umf_result_t res) { } } -namespace umf { -ur_result_t getProviderNativeError(const char *providerName, - int32_t nativeError) { - if (strcmp(providerName, "LEVEL_ZERO") == 0) { - auto zeResult = static_cast(nativeError); - if (zeResult == ZE_RESULT_ERROR_UNSUPPORTED_SIZE) { - return UR_RESULT_ERROR_INVALID_USM_SIZE; - } - return ze2urResult(zeResult); - } - - return UR_RESULT_ERROR_UNKNOWN; -} -} // namespace umf - static std::optional initializeDisjointPoolConfig() { if (getenv_tobool("UR_L0_DISABLE_USM_ALLOCATOR") || @@ -100,11 +88,13 @@ makeProvider(usm::pool_descriptor poolDescriptor) { decltype(&umfLevelZeroMemoryProviderParamsDestroy)> params(hParams, &umfLevelZeroMemoryProviderParamsDestroy); + auto hContext = v2_cast(poolDescriptor.hContext); UMF_CALL_THROWS(umfLevelZeroMemoryProviderParamsSetContext( - hParams, poolDescriptor.hContext->getZeHandle())); + hParams, hContext->getZeHandle())); + auto hDevice = common_cast(poolDescriptor.hDevice); ze_device_handle_t level_zero_device_handle = - poolDescriptor.hDevice ? poolDescriptor.hDevice->ZeDevice : nullptr; + poolDescriptor.hDevice ? hDevice->ZeDevice : nullptr; UMF_CALL_THROWS(umfLevelZeroMemoryProviderParamsSetDevice( hParams, level_zero_device_handle)); @@ -121,9 +111,9 @@ makeProvider(usm::pool_descriptor poolDescriptor) { // Make memory resident on the source device itself plus all peer devices // that have explicitly enabled peer access to it. zeDeviceHandles.push_back(level_zero_device_handle); - for (auto dev : - poolDescriptor.hContext->getDevicesWhichCanAccessAllocationsPresentOn( - poolDescriptor.hDevice)) { + for (auto dev : v2_cast(poolDescriptor.hContext) + ->getDevicesWhichCanAccessAllocationsPresentOn( + common_cast(poolDescriptor.hDevice))) { zeDeviceHandles.push_back(dev->ZeDevice); } @@ -174,7 +164,7 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t hContext, auto devicesAndSubDevices = CollectDevicesForUsmPoolCreation(hContext->getDevices()); auto descriptors = usm::pool_descriptor::createFromDevices( - this, hContext, devicesAndSubDevices); + v2_cast(this), v2_cast(hContext), common_cast(devicesAndSubDevices)); for (auto &desc : descriptors) { std::unique_ptr usmPool; if (disjointPoolConfigs.has_value()) { @@ -212,27 +202,30 @@ ur_usm_pool_handle_t_::ur_usm_pool_handle_t_(ur_context_handle_t hContext, } // Create pool descriptor for single device provided + auto selfOpque = v2_cast(this); + auto hContextOpque = v2_cast(hContext); + auto hDeviceOpque = common_cast(hDevice); std::vector descriptors; { auto &desc = descriptors.emplace_back(); - desc.poolHandle = this; - desc.hContext = hContext; - desc.hDevice = hDevice; + desc.poolHandle = selfOpque; + desc.hContext = hContextOpque; + desc.hDevice = hDeviceOpque; desc.type = UR_USM_TYPE_DEVICE; } { auto &desc = descriptors.emplace_back(); - desc.poolHandle = this; - desc.hContext = hContext; - desc.hDevice = hDevice; + desc.poolHandle = selfOpque; + desc.hContext = hContextOpque; + desc.hDevice = hDeviceOpque; desc.type = UR_USM_TYPE_SHARED; desc.deviceReadOnly = false; } { auto &desc = descriptors.emplace_back(); - desc.poolHandle = this; - desc.hContext = hContext; - desc.hDevice = hDevice; + desc.poolHandle = selfOpque; + desc.hContext = hContextOpque; + desc.hDevice = hDeviceOpque; desc.type = UR_USM_TYPE_SHARED; desc.deviceReadOnly = true; } @@ -287,7 +280,7 @@ ur_result_t ur_usm_pool_handle_t_::allocate( auto deviceFlags = getDeviceFlags(pUSMDesc); auto pool = getPool(usm::pool_descriptor{ - this, hContext, hDevice, type, + v2_cast(this), v2_cast(hContext), common_cast(hDevice), type, bool(deviceFlags & UR_USM_DEVICE_MEM_FLAG_DEVICE_READ_ONLY)}); if (!pool) { return UR_RESULT_ERROR_INVALID_ARGUMENT; @@ -357,8 +350,8 @@ ur_usm_pool_handle_t_::allocateEnqueued(ur_context_handle_t hContext, void *hQueue, bool isInOrderQueue, ur_device_handle_t hDevice, ur_usm_type_t type, size_t size) { - auto umfPool = - getPool(usm::pool_descriptor{this, hContext, hDevice, type, false}); + auto umfPool = getPool(usm::pool_descriptor{ + v2_cast(this), v2_cast(hContext), common_cast(hDevice), type, false}); if (!umfPool) { return std::nullopt; } @@ -368,12 +361,13 @@ ur_usm_pool_handle_t_::allocateEnqueued(ur_context_handle_t hContext, return std::nullopt; } + auto hEvent = v2_cast(allocation->Event); if (allocation->Queue == hQueue && isInOrderQueue) { - if (allocation->Event) - allocation->Event->release(); + if (hEvent) + hEvent->release(); return std::make_pair(allocation->Ptr, nullptr); } else { - return std::make_pair(allocation->Ptr, allocation->Event); + return std::make_pair(allocation->Ptr, hEvent); } } @@ -450,10 +444,11 @@ void ur_usm_pool_handle_t_::changeResidentDevice(ur_device_handle_t hDevice, bool isAdding) { poolManager.forEachPoolWithDesc([=](const auto &desc, auto pool) { if (desc.supportsResidentDevices() && desc.hDevice && - desc.hDevice->ZeDevice == hDevice->ZeDevice) { + common_cast(desc.hDevice)->ZeDevice == hDevice->ZeDevice) { UR_LOG(INFO, "found {} of srcDevice:{} valid to {} peerDevice:{}", - logger::makeStringFromStreamable(desc), desc.hDevice->Id.value(), - isAdding ? "add" : "remove", peerDevice->Id.value()); + logger::makeStringFromStreamable(desc), + common_cast(desc.hDevice)->Id.value(), isAdding ? "add" : "remove", + peerDevice->Id.value()); umf_memory_provider_handle_t hProvider; umf_result_t getProviderResult = umfPoolGetMemoryProvider(pool->umfPool.get(), &hProvider); @@ -489,17 +484,18 @@ void ur_usm_pool_handle_t_::changeResidentDevice(ur_device_handle_t hDevice, }); } -namespace ur::level_zero { ur_result_t urUSMPoolCreate( /// [in] handle of the context object - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] pointer to USM pool descriptor. Can be chained with /// ::ur_usm_pool_limits_desc_t ur_usm_pool_desc_t *pPoolDesc, /// [out] pointer to USM memory pool - ur_usm_pool_handle_t *hPool) try { - *hPool = new ur_usm_pool_handle_t_(hContext, pPoolDesc); - hContext->addUsmPool(*hPool); + ::ur_usm_pool_handle_t *hPoolOpque) try { + auto hContext = v2_cast(hContextOpque); + auto hPool = new ur_usm_pool_handle_t_(hContext, pPoolDesc); + *hPoolOpque = v2_cast(hPool); + hContext->addUsmPool(hPool); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -509,8 +505,8 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool -urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { - hPool->RefCount.retain(); +urUSMPoolRetain(::ur_usm_pool_handle_t hPoolOpque) try { + v2_cast(hPoolOpque)->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -520,7 +516,8 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool -urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { +urUSMPoolRelease(::ur_usm_pool_handle_t hPoolOpque) try { + auto hPool = v2_cast(hPoolOpque); if (hPool->RefCount.release()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; @@ -534,7 +531,7 @@ urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { ur_result_t urUSMPoolGetInfo( /// [in] handle of the USM memory pool - ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t hPoolOpque, /// [in] name of the pool property to query ur_usm_pool_info_t propName, /// [in] size in bytes of the pool property value provided @@ -543,6 +540,7 @@ ur_result_t urUSMPoolGetInfo( void *pPropValue, /// [out] size in bytes returned in pool property value size_t *pPropSizeRet) try { + auto hPool = v2_cast(hPoolOpque); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { @@ -562,11 +560,14 @@ ur_result_t urUSMPoolGetInfo( return exceptionToResult(std::current_exception()); } -ur_result_t urUSMPoolCreateExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, +ur_result_t urUSMPoolCreateExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t hDeviceOpque, ur_usm_pool_desc_t *pPoolDesc, - ur_usm_pool_handle_t *pPool) try { - *pPool = new ur_usm_pool_handle_t_(hContext, hDevice, pPoolDesc); + ::ur_usm_pool_handle_t *pPoolOpque) try { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + auto pPool = v2_cast(pPoolOpque); + *pPool = new v2::ur_usm_pool_handle_t_(hContext, hDevice, pPoolDesc); hContext->addUsmPool(*pPool); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { @@ -575,8 +576,9 @@ ur_result_t urUSMPoolCreateExp(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -ur_result_t urUSMPoolDestroyExp(ur_context_handle_t, ur_device_handle_t, - ur_usm_pool_handle_t hPool) try { +ur_result_t urUSMPoolDestroyExp(::ur_context_handle_t, ::ur_device_handle_t, + ::ur_usm_pool_handle_t hPoolOpque) try { + auto hPool = v2_cast(hPoolOpque); if (hPool->RefCount.release()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; @@ -588,9 +590,10 @@ ur_result_t urUSMPoolDestroyExp(ur_context_handle_t, ur_device_handle_t, return exceptionToResult(std::current_exception()); } -ur_result_t urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, +ur_result_t urUSMPoolGetInfoExp(::ur_usm_pool_handle_t hPoolOpque, ur_usm_pool_info_t propName, void *pPropValue, size_t *pPropSizeRet) { + auto hPool = v2_cast(hPoolOpque); size_t value = 0; switch (propName) { case UR_USM_POOL_INFO_RELEASE_THRESHOLD_EXP: @@ -625,7 +628,7 @@ ur_result_t urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, return UR_RESULT_SUCCESS; } -ur_result_t urUSMPoolSetInfoExp(ur_usm_pool_handle_t /*hPool*/, +ur_result_t urUSMPoolSetInfoExp(::ur_usm_pool_handle_t /*hPoolOpque*/, ur_usm_pool_info_t propName, void * /*pPropValue*/, size_t propSize) { if (propSize < sizeof(size_t)) { @@ -647,29 +650,34 @@ ur_result_t urUSMPoolSetInfoExp(ur_usm_pool_handle_t /*hPool*/, return UR_RESULT_SUCCESS; } -ur_result_t urUSMPoolGetDefaultDevicePoolExp(ur_context_handle_t hContext, - ur_device_handle_t, - ur_usm_pool_handle_t *pPool) { +ur_result_t +urUSMPoolGetDefaultDevicePoolExp(::ur_context_handle_t hContextOpque, + ::ur_device_handle_t, + ::ur_usm_pool_handle_t *pPoolOpque) { + auto hContext = v2_cast(hContextOpque); // Default async pool should contain an internal pool for all detected // devices. - *pPool = hContext->getAsyncPool(); + *pPoolOpque = v2_cast(hContext->getAsyncPool()); return UR_RESULT_SUCCESS; } ur_result_t urUSMDeviceAlloc( /// [in] handle of the context object - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] handle of the device object - ur_device_handle_t hDevice, + ::ur_device_handle_t hDeviceOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *pUSMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t hPoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t size, /// [out] pointer to USM device memory object void **ppRetMem) try { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + auto hPool = v2_cast(hPoolOpque); if (!hPool) { hPool = hContext->getDefaultUSMPool(); } @@ -684,17 +692,20 @@ ur_result_t urUSMDeviceAlloc( ur_result_t urUSMSharedAlloc( /// [in] handle of the context object - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] handle of the device object - ur_device_handle_t hDevice, + ::ur_device_handle_t hDeviceOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *pUSMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t hPoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t size, /// [out] pointer to USM shared memory object void **ppRetMem) try { + auto hContext = v2_cast(hContextOpque); + auto hDevice = common_cast(hDeviceOpque); + auto hPool = v2_cast(hPoolOpque); if (!hPool) { hPool = hContext->getDefaultUSMPool(); } @@ -709,15 +720,17 @@ ur_result_t urUSMSharedAlloc( ur_result_t urUSMHostAlloc( /// [in] handle of the context object - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in][optional] USM memory allocation descriptor const ur_usm_desc_t *pUSMDesc, /// [in][optional] Pointer to a pool created using urUSMPoolCreate - ur_usm_pool_handle_t hPool, + ::ur_usm_pool_handle_t hPoolOpque, /// [in] size in bytes of the USM memory object to be allocated size_t size, /// [out] pointer to USM host memory object void **ppRetMem) try { + auto hContext = v2_cast(hContextOpque); + auto hPool = v2_cast(hPoolOpque); if (!hPool) { hPool = hContext->getDefaultUSMPool(); } @@ -732,7 +745,7 @@ ur_result_t urUSMHostAlloc( ur_result_t urUSMFree( /// [in] handle of the context object - ur_context_handle_t /*hContext*/, + ::ur_context_handle_t /*hContext*/, /// [in] pointer to USM memory object void *pMem) try { umf_memory_pool_handle_t umfPool = nullptr; @@ -741,7 +754,7 @@ ur_result_t urUSMFree( return UR_RESULT_ERROR_INVALID_MEM_OBJECT; } - UsmPool *usmPool = nullptr; + v2::UsmPool *usmPool = nullptr; umfRet = umfPoolGetTag(umfPool, (void **)&usmPool); if (umfRet != UMF_RESULT_SUCCESS || !usmPool) { // This should never happen @@ -757,7 +770,7 @@ ur_result_t urUSMFree( ur_result_t urUSMGetMemAllocInfo( /// [in] handle of the context object - ur_context_handle_t hContext, + ::ur_context_handle_t hContextOpque, /// [in] pointer to USM memory object const void *ptr, /// [in] the name of the USM allocation property to query @@ -775,8 +788,9 @@ ur_result_t urUSMGetMemAllocInfo( // https://github.com/oneapi-src/unified-memory-framework/issues/686 // https://github.com/oneapi-src/unified-memory-framework/issues/687 // are implemented + auto ctx = v2_cast(hContextOpque); ZE2UR_CALL(zeMemGetAllocProperties, - (hContext->getZeHandle(), ptr, &zeMemoryAllocationProperties, + (ctx->getZeHandle(), ptr, &zeMemoryAllocationProperties, &zeDeviceHandle)); UrReturnHelper ReturnValue(propValueSize, pPropValue, pPropValueSizeRet); @@ -804,7 +818,7 @@ ur_result_t urUSMGetMemAllocInfo( } case UR_USM_ALLOC_INFO_DEVICE: if (zeDeviceHandle) { - auto Platform = hContext->getPlatform(); + auto Platform = ctx->getPlatform(); auto Device = Platform->getDeviceFromNativeHandle(zeDeviceHandle); return Device ? ReturnValue(Device) : UR_RESULT_ERROR_INVALID_VALUE; } else { @@ -812,14 +826,12 @@ ur_result_t urUSMGetMemAllocInfo( } case UR_USM_ALLOC_INFO_BASE_PTR: { void *base; - ZE2UR_CALL(zeMemGetAddressRange, - (hContext->getZeHandle(), ptr, &base, nullptr)); + ZE2UR_CALL(zeMemGetAddressRange, (ctx->getZeHandle(), ptr, &base, nullptr)); return ReturnValue(base); } case UR_USM_ALLOC_INFO_SIZE: { size_t size; - ZE2UR_CALL(zeMemGetAddressRange, - (hContext->getZeHandle(), ptr, nullptr, &size)); + ZE2UR_CALL(zeMemGetAddressRange, (ctx->getZeHandle(), ptr, nullptr, &size)); return ReturnValue(size); } case UR_USM_ALLOC_INFO_POOL: { @@ -830,7 +842,7 @@ ur_result_t urUSMGetMemAllocInfo( } ur_result_t ret = UR_RESULT_ERROR_INVALID_VALUE; - hContext->forEachUsmPool([&](ur_usm_pool_handle_t hPool) { + ctx->forEachUsmPool([&](ur_usm_pool_handle_t hPool) { if (hPool->hasPool(umfPool)) { ret = ReturnValue(hPool); return false; // break; @@ -850,8 +862,9 @@ ur_result_t urUSMGetMemAllocInfo( return exceptionToResult(std::current_exception()); } -ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *hostPtr, +ur_result_t urUSMImportExp(::ur_context_handle_t hContextOpque, void *hostPtr, size_t size) { + auto hContext = v2_cast(hContextOpque); UR_ASSERT(hContext, UR_RESULT_ERROR_INVALID_CONTEXT); // Promote the host ptr to USM host memory. @@ -874,7 +887,9 @@ ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *hostPtr, return UR_RESULT_SUCCESS; } -ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *hostPtr) { +ur_result_t urUSMReleaseExp(::ur_context_handle_t hContextOpque, + void *hostPtr) { + auto hContext = v2_cast(hContextOpque); UR_ASSERT(hContext, UR_RESULT_ERROR_INVALID_CONTEXT); // Release the imported memory. @@ -884,15 +899,16 @@ ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *hostPtr) { return UR_RESULT_SUCCESS; } -ur_result_t UR_APICALL urUSMContextMemcpyExp(ur_context_handle_t hContext, - void *pDst, const void *pSrc, - size_t size) { - ur_device_handle_t hDevice = hContext->getDevices()[0]; +ur_result_t UR_APICALL +urUSMContextMemcpyExp(::ur_context_handle_t hContextOpque, void *pDst, + const void *pSrc, size_t size) { + auto ctx = v2_cast(hContextOpque); + ur_device_handle_t hDevice = ctx->getDevices()[0]; auto Ordinal = static_cast( hDevice ->QueueGroup[ur_device_handle_t_::queue_group_info_t::type::Compute] .ZeOrdinal); - auto commandList = hContext->getCommandListCache().getImmediateCommandList( + auto commandList = ctx->getCommandListCache().getImmediateCommandList( hDevice->ZeDevice, {true, Ordinal, true}, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL, std::nullopt); @@ -902,8 +918,9 @@ ur_result_t UR_APICALL urUSMContextMemcpyExp(ur_context_handle_t hContext, } ur_result_t urUSMHostAllocRegisterExp( - ur_context_handle_t hContext, void *pHostMem, size_t size, + ::ur_context_handle_t hContextOpque, void *pHostMem, size_t size, const ur_exp_usm_host_alloc_register_properties_t *pProperties) { + auto hContext = v2_cast(hContextOpque); if (!hContext->getPlatform()->ZeExternalMemoryMappingExtensionSupported) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -946,8 +963,9 @@ ur_result_t urUSMHostAllocRegisterExp( return UR_RESULT_SUCCESS; } -ur_result_t urUSMHostAllocUnregisterExp(ur_context_handle_t hContext, +ur_result_t urUSMHostAllocUnregisterExp(::ur_context_handle_t hContextOpque, void *pHostMem) { + auto hContext = v2_cast(hContextOpque); if (!hContext->getPlatform()->ZeExternalMemoryMappingExtensionSupported) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -958,4 +976,4 @@ ur_result_t urUSMHostAllocUnregisterExp(ur_context_handle_t hContext, return UR_RESULT_SUCCESS; } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.hpp b/unified-runtime/source/adapters/level_zero/v2/usm.hpp index 5ea29d7e3220b..2fb53dbbfafb1 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.hpp @@ -11,12 +11,15 @@ #include "unified-runtime/ur_api.h" -#include "../enqueued_pool.hpp" +#include "../common/enqueued_pool.hpp" +#include "../common/ur_ref_count.hpp" #include "common.hpp" -#include "common/ur_ref_count.hpp" #include "event.hpp" +#include "ur_interface_loader.hpp" #include "ur_pool_manager.hpp" +namespace ur::level_zero::v2 { + struct UsmPool; struct AllocationStats { @@ -51,7 +54,7 @@ struct AllocationStats { std::atomic_size_t PeakAllocatedMemorySize{0}; }; -struct ur_usm_pool_handle_t_ : ur_object { +struct ur_usm_pool_handle_t_ : v2::ur_object_t { ur_usm_pool_handle_t_(ur_context_handle_t hContext, ur_usm_pool_desc_t *pPoolDes); ur_usm_pool_handle_t_(ur_context_handle_t hContext, @@ -94,13 +97,18 @@ struct ur_usm_pool_handle_t_ : ur_object { struct UsmPool { UsmPool(ur_usm_pool_handle_t urPool, umf::pool_unique_handle_t umfPool) : urPool(urPool), umfPool(std::move(umfPool)), - asyncPool([](ur_event_handle_t hEvent) { return hEvent->release(); }, - [context = urPool->getContextHandle()](void *ptr) { - return ur::level_zero::urUSMFree(context, ptr); - }) {} + asyncPool( + [](::ur_event_handle_t hEvent) { + return v2_cast(hEvent)->release(); + }, + [hContextOpque = v2_cast(urPool->getContextHandle())](void *ptr) { + return ur::level_zero::v2::urUSMFree(hContextOpque, ptr); + }) {} ur_usm_pool_handle_t urPool; umf::pool_unique_handle_t umfPool; // 'asyncPool' needs to be declared after 'umfPool' so its destructor is // invoked first. EnqueuedPool asyncPool; }; + +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp b/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp index 0f424ebcdef3a..ef91691ee00b0 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp @@ -7,11 +7,12 @@ // //===----------------------------------------------------------------------===// -#include "../device.hpp" +#include "../common/device.hpp" +#include "../common/platform.hpp" #include "context.hpp" #include "logger/ur_logger.hpp" -namespace ur::level_zero { +namespace ur::level_zero::v2 { // Validates that two devices are compatible for P2P operations: both must have // an assigned Id, must belong to the same platform (i.e. share the same device @@ -69,34 +70,41 @@ static ur_result_t urUsmP2PChangePeerAccessExp(ur_device_handle_t commandDevice, // Copy the context list under the mutex and iterate outside the critical // section to avoid holding ContextsMutex during potentially heavy // changeResidentDevice calls and to reduce deadlock risk. - std::list Contexts; + std::list<::ur_context_handle_t> Contexts; { std::scoped_lock Lock(Platform->ContextsMutex); Contexts = Platform->Contexts; } UR_LOG(INFO, "changing peers in {} contexts", Contexts.size()); for (auto Context : Contexts) { - Context->changeResidentDevice(peerDevice, commandDevice, isAdding); + v2_cast(Context)->changeResidentDevice(peerDevice, commandDevice, isAdding); } return UR_RESULT_SUCCESS; } -ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice) { +ur_result_t urUsmP2PEnablePeerAccessExp(::ur_device_handle_t commandDeviceOpque, + ::ur_device_handle_t peerDeviceOpque) { + auto commandDevice = common_cast(commandDeviceOpque); + auto peerDevice = common_cast(peerDeviceOpque); return urUsmP2PChangePeerAccessExp(commandDevice, peerDevice, true); } -ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice) { +ur_result_t +urUsmP2PDisablePeerAccessExp(::ur_device_handle_t commandDeviceOpque, + ::ur_device_handle_t peerDeviceOpque) { + auto commandDevice = common_cast(commandDeviceOpque); + auto peerDevice = common_cast(peerDeviceOpque); return urUsmP2PChangePeerAccessExp(commandDevice, peerDevice, false); } -ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice, - ur_exp_peer_info_t propName, - size_t propSize, void *pPropValue, - size_t *pPropSizeRet) { +ur_result_t +urUsmP2PPeerAccessGetInfoExp(::ur_device_handle_t commandDeviceOpque, + ::ur_device_handle_t peerDeviceOpque, + ur_exp_peer_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + auto commandDevice = common_cast(commandDeviceOpque); + auto peerDevice = common_cast(peerDeviceOpque); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); @@ -125,4 +133,4 @@ ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, return ReturnValue(propertyValue); } -} // namespace ur::level_zero +} // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/native_cpu/usm.cpp b/unified-runtime/source/adapters/native_cpu/usm.cpp index 36e7d4791376b..b4984d276ff64 100644 --- a/unified-runtime/source/adapters/native_cpu/usm.cpp +++ b/unified-runtime/source/adapters/native_cpu/usm.cpp @@ -14,12 +14,6 @@ #include "context.hpp" #include -namespace umf { -ur_result_t getProviderNativeError(const char *, int32_t) { - return UR_RESULT_ERROR_UNKNOWN; -} -} // namespace umf - static ur_result_t alloc_helper(ur_context_handle_t hContext, const ur_usm_desc_t *pUSMDesc, size_t size, void **ppMem, ur_usm_type_t type) { diff --git a/unified-runtime/source/adapters/opencl/usm.cpp b/unified-runtime/source/adapters/opencl/usm.cpp index 540ffa502cc88..e1bc4fe76caf0 100644 --- a/unified-runtime/source/adapters/opencl/usm.cpp +++ b/unified-runtime/source/adapters/opencl/usm.cpp @@ -24,13 +24,6 @@ void AllocDeleterCallback(cl_event event, cl_int, void *pUserData) { delete Info; } -namespace umf { -ur_result_t getProviderNativeError(const char *, int32_t) { - // TODO: implement when UMF supports OpenCL - return UR_RESULT_ERROR_UNKNOWN; -} -} // namespace umf - inline cl_mem_alloc_flags_intel hostDescToClFlags(const ur_usm_host_desc_t &desc) { cl_mem_alloc_flags_intel allocFlags = 0; diff --git a/unified-runtime/source/common/CMakeLists.txt b/unified-runtime/source/common/CMakeLists.txt index e0c6d17a3d4dd..27d6ef9c0dcad 100644 --- a/unified-runtime/source/common/CMakeLists.txt +++ b/unified-runtime/source/common/CMakeLists.txt @@ -61,10 +61,19 @@ target_include_directories(ur_common PUBLIC ) -if (UR_STATIC_ADAPTER_L0) +# Only a statically-embedded Level Zero adapter that is actually built pulls UMF +# in statically. UR_STATIC_ADAPTER_L0 defaults ON even when L0 is not built, so +# gate on the adapter being built too; otherwise non-L0 builds (e.g. macOS) +# would needlessly force static UMF and its hwloc dependency. +if ((UR_STATIC_ADAPTER_L0 AND UR_BUILD_ADAPTER_L0) OR + (UR_STATIC_ADAPTER_L0_V2 AND UR_BUILD_ADAPTER_L0_V2)) if (UMF_BUILD_SHARED_LIBRARY) message(STATUS "Static adapter is not compatible with shared UMF, switching to fully statically linked UMF") set(UMF_BUILD_SHARED_LIBRARY OFF) + # A statically-linked UMF must also link hwloc statically; otherwise it + # requires a system hwloc that isn't present on every platform (e.g. + # macOS), breaking the link. + set(UMF_LINK_HWLOC_STATICALLY ON) endif() endif() @@ -99,6 +108,9 @@ else() set(UMF_BUILD_TESTS OFF CACHE INTERNAL "Build UMF tests") set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "Build UMF examples") set(UMF_BUILD_SHARED_LIBRARY ${UMF_BUILD_SHARED_LIBRARY} CACHE INTERNAL "Build UMF shared library") + if(UMF_LINK_HWLOC_STATICALLY) + set(UMF_LINK_HWLOC_STATICALLY ON CACHE INTERNAL "static HWLOC") + endif() FetchContent_MakeAvailable(unified-memory-framework) FetchContent_GetProperties(unified-memory-framework) @@ -162,7 +174,7 @@ target_link_libraries(ur_umf INTERFACE # Install ur_umf when any static adapter is enabled, so install(EXPORT ...) # can resolve it for static consumers. -if(UR_STATIC_ADAPTER_L0 OR UR_STATIC_ADAPTER_OPENCL) +if(UR_STATIC_ADAPTER_L0 OR UR_STATIC_ADAPTER_L0_V2 OR UR_STATIC_ADAPTER_OPENCL) install(TARGETS ur_umf EXPORT ${PROJECT_NAME}-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/unified-runtime/source/common/umf_helpers.hpp b/unified-runtime/source/common/umf_helpers.hpp index f200ad5a1026e..fb90efa4e4b2a 100644 --- a/unified-runtime/source/common/umf_helpers.hpp +++ b/unified-runtime/source/common/umf_helpers.hpp @@ -282,13 +282,25 @@ template umf_result_t &getPoolLastStatusRef() { return last_status; } -ur_result_t getProviderNativeError(const char *providerName, - int32_t nativeError); +namespace detail { +inline ur_result_t defaultGetProviderNativeError(const char * /*providerName*/, + int32_t /*nativeError*/) { + return UR_RESULT_ERROR_UNKNOWN; +} +} // namespace detail + +// Customization point for translating a memory provider's backend-native error +// to a ur_result_t. Each adapter #defines this to its own internal-linkage +// translator before including this header. +#ifndef UMF_GET_PROVIDER_NATIVE_ERROR +#define UMF_GET_PROVIDER_NATIVE_ERROR \ + ::umf::detail::defaultGetProviderNativeError +#endif /// @brief translates UMF return values to UR. /// This function assumes that the native error of /// the last failed memory provider is ur_result_t. -inline ur_result_t umf2urResult(umf_result_t umfResult) { +static inline ur_result_t umf2urResult(umf_result_t umfResult) { switch (umfResult) { case UMF_RESULT_SUCCESS: return UR_RESULT_SUCCESS; @@ -315,7 +327,7 @@ inline ur_result_t umf2urResult(umf_result_t umfResult) { return UR_RESULT_ERROR_UNKNOWN; } - return getProviderNativeError(Name, Err); + return UMF_GET_PROVIDER_NATIVE_ERROR(Name, Err); } case UMF_RESULT_ERROR_INVALID_ARGUMENT: return UR_RESULT_ERROR_INVALID_ARGUMENT; diff --git a/unified-runtime/source/loader/CMakeLists.txt b/unified-runtime/source/loader/CMakeLists.txt index 5eea34acc46d7..96021c316d8d2 100644 --- a/unified-runtime/source/loader/CMakeLists.txt +++ b/unified-runtime/source/loader/CMakeLists.txt @@ -79,7 +79,7 @@ target_link_libraries(ur_loader PRIVATE ${PROJECT_NAME}::headers ) -if(UR_STATIC_ADAPTER_L0) +if(UR_STATIC_ADAPTER_L0 AND (UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)) target_link_libraries(ur_loader PRIVATE ur_adapter_level_zero ) @@ -91,6 +91,21 @@ if(UR_STATIC_ADAPTER_OPENCL AND (UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL target_compile_definitions(ur_loader PRIVATE UR_STATIC_ADAPTER_OPENCL) endif() +if(UR_STATIC_ADAPTER_L0_V2 AND (UR_BUILD_ADAPTER_L0_V2 OR UR_BUILD_ADAPTER_ALL)) + target_link_libraries(ur_loader PRIVATE + ur_adapter_level_zero_v2 + ) + target_compile_definitions(ur_loader PRIVATE UR_STATIC_ADAPTER_LEVEL_ZERO_V2) +endif() + +# A static Level Zero adapter links UMF statically; let the loader hold a UMF +# reference across teardown (see ur_lib.cpp). +if((UR_STATIC_ADAPTER_L0 AND UR_BUILD_ADAPTER_L0) OR + (UR_STATIC_ADAPTER_L0_V2 AND UR_BUILD_ADAPTER_L0_V2)) + target_compile_definitions(ur_loader PRIVATE UR_STATIC_UMF) + target_link_libraries(ur_loader PRIVATE umf::umf umf::headers) +endif() + if(UR_ENABLE_TRACING) target_link_libraries(ur_loader PRIVATE ${TARGET_XPTI}) target_include_directories(ur_loader PRIVATE ${xpti_SOURCE_DIR}/include) diff --git a/unified-runtime/source/loader/ur_lib.cpp b/unified-runtime/source/loader/ur_lib.cpp index 2eebe84194483..9030e1c7c7574 100644 --- a/unified-runtime/source/loader/ur_lib.cpp +++ b/unified-runtime/source/loader/ur_lib.cpp @@ -26,6 +26,13 @@ #include #include +// With static UMF, its library destructor finalizes UMF before the loader +// tears down. Hold a umfInit reference so UMF outlives the layers' frees in +// urLoaderTearDown. +#if defined(UR_STATIC_UMF) +#include +#endif + namespace ur_lib { /////////////////////////////////////////////////////////////////////////////// context_t::context_t() { parseEnvEnabledLayers(); } @@ -85,6 +92,12 @@ __urdlllocal ur_result_t context_t::Init( initLayers(); } +#if defined(UR_STATIC_UMF) + if (UR_RESULT_SUCCESS == result) { + umfInit(); + } +#endif + return result; } @@ -205,6 +218,13 @@ ur_result_t urLoaderTearDown() { delete context; }); +#if defined(UR_STATIC_UMF) + // Release the Init reference only after the layers are torn down above. + if (ret == 0) { + umfTearDown(); + } +#endif + ur_result_t result = ret == 0 ? UR_RESULT_SUCCESS : UR_RESULT_ERROR_UNINITIALIZED; UR_LOG(INFO, "---> urLoaderTearDown() -> {}", result); diff --git a/unified-runtime/source/loader/ur_loader.cpp b/unified-runtime/source/loader/ur_loader.cpp index f216fdb43ea69..bb49623d80b81 100644 --- a/unified-runtime/source/loader/ur_loader.cpp +++ b/unified-runtime/source/loader/ur_loader.cpp @@ -14,6 +14,9 @@ #ifdef UR_STATIC_ADAPTER_OPENCL #include "adapters/opencl/ur_interface_loader.hpp" #endif +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 +#include "adapters/level_zero/v2/ur_interface_loader.hpp" +#endif namespace ur_loader { /////////////////////////////////////////////////////////////////////////////// @@ -40,7 +43,7 @@ ur_result_t context_t::init() { if (!adapter_registry.adaptersForceLoaded()) { #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO auto &level_zero = platforms.emplace_back(nullptr); - ur::level_zero::urAdapterGetDdiTables(&level_zero.dditable); + ur::level_zero::v1::urAdapterGetDdiTables(&level_zero.dditable); adapter_registry.markAdapterAsStaticallyLoaded("ur_adapter_level_zero"); #endif #ifdef UR_STATIC_ADAPTER_OPENCL @@ -51,6 +54,16 @@ ur_result_t context_t::init() { } #endif +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO_V2 + // If the adapters were force loaded, it means the user wants to use + // a specific adapter library. Don't load any static adapters. + if (!adapter_registry.adaptersForceLoaded()) { + auto &level_zero_v2 = platforms.emplace_back(nullptr); + ur::level_zero::v2::urAdapterGetDdiTables(&level_zero_v2.dditable); + adapter_registry.markAdapterAsStaticallyLoaded("ur_adapter_level_zero_v2"); + } +#endif + for (const auto &adapterPaths : adapter_registry) { // Skip dynamic adapters that have already been statically registered // to avoid double-registration of the same backend. diff --git a/unified-runtime/test/adapters/level_zero/v2/CMakeLists.txt b/unified-runtime/test/adapters/level_zero/v2/CMakeLists.txt index 2e9397c7c5741..b216d7fce289d 100644 --- a/unified-runtime/test/adapters/level_zero/v2/CMakeLists.txt +++ b/unified-runtime/test/adapters/level_zero/v2/CMakeLists.txt @@ -2,8 +2,13 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -function(add_l0_v2_devices_test test) - add_conformance_devices_test(${test} ${ARGN}) +function(link_l0_v2_adapter_sources test) + if(NOT UR_STATIC_ADAPTER_L0_V2) + target_sources(${test}-test PRIVATE + $ + ) + target_link_libraries(${test}-test PRIVATE ur_adapter_level_zero_common) + endif() target_link_libraries(${test}-test PRIVATE ${PROJECT_NAME}::umf @@ -13,73 +18,36 @@ function(add_l0_v2_devices_test test) ) target_include_directories(${test}-test PRIVATE - ${PROJECT_SOURCE_DIR}/source + $ ${PROJECT_SOURCE_DIR}/source/adapters ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2 - LevelZeroLoader-Headers ) endfunction() +function(add_l0_v2_devices_test test) + add_conformance_devices_test(${test} ${ARGN}) + link_l0_v2_adapter_sources(${test}) +endfunction() + function(add_l0_v2_kernels_test test) add_conformance_kernels_test(${test} ${ARGN}) - - target_link_libraries(${test}-test PRIVATE - ${PROJECT_NAME}::umf - LevelZeroLoader - LevelZeroLoader-Headers - ComputeRuntimeLevelZero-Headers - ) - - target_include_directories(${test}-test PRIVATE - ${PROJECT_SOURCE_DIR}/source - ${PROJECT_SOURCE_DIR}/source/adapters - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2 - LevelZeroLoader-Headers - ) + link_l0_v2_adapter_sources(${test}) endfunction() add_l0_v2_devices_test(command_list_cache command_list_cache_test.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/command_list_cache.cpp ) add_l0_v2_devices_test(event_pool event_pool_test.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/adapter.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/device.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/platform.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_pool_cache.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_pool.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_provider_counter.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_provider_normal.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event.cpp - ${PROJECT_SOURCE_DIR}/source/ur/ur.cpp ) add_l0_v2_devices_test(memory_residency memory_residency.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp ) add_l0_v2_devices_test(batched_queue batched_queue_test.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/adapter.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/device.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/platform.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_pool_cache.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_pool.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_provider_counter.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event_provider_normal.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/event.cpp - ${PROJECT_SOURCE_DIR}/source/ur/ur.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/command_list_cache.cpp ) if(NOT UR_FOUND_DPCXX) @@ -90,7 +58,5 @@ if(NOT UR_FOUND_DPCXX) else() add_l0_v2_kernels_test(deferred_kernel deferred_kernel.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp - ${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp ) endif() diff --git a/unified-runtime/test/adapters/level_zero/v2/batched_queue_test.cpp b/unified-runtime/test/adapters/level_zero/v2/batched_queue_test.cpp index 9ebb6c701bcd3..e7ef0e254e8b9 100644 --- a/unified-runtime/test/adapters/level_zero/v2/batched_queue_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/batched_queue_test.cpp @@ -8,7 +8,7 @@ #include "command_list_cache.hpp" #include "level_zero/common.hpp" -#include "level_zero/device.hpp" +#include "level_zero/common/device.hpp" #include "../ze_helpers.hpp" #include "context.hpp" @@ -35,19 +35,14 @@ #include #include -const ur_dditable_t *ur::level_zero::ddi_getter::value() { - // Return a blank dditable - static ur_dditable_t table{}; - return &table; -}; +// Note: ddi_getter::value() and ur_context_handle_t_::getDevices() are now +// provided by ur_adapter_level_zero_common and ur_adapter_level_zero_v2, +// which this test links against. No local mocks are needed. -// mock necessary functions from context, we can't pull in entire context -// implementation due to a lot of other dependencies -std::vector mockVec{}; -const std::vector & -ur_context_handle_t_::getDevices() const { - return mockVec; -} +// The adapter types this test exercises live in `ur::level_zero::v2`; alias it +// so the unqualified `v2::` references resolve without pulling the whole +// `ur::level_zero` namespace (which would clash with the public `urX` C API). +namespace v2 = ur::level_zero::v2; struct urBatchedQueueTest : uur::urContextTest { void SetUp() override { @@ -83,6 +78,7 @@ struct urBatchedQueueTest : uur::urContextTest { } void vectorOfSubmittedBatchesIsClearedHelper() { + auto ctx = v2::v2_cast(context); std::vector data(buffer_size, 42); // Initially, the vector of batches submitted for execution is empty. After // every iteration, each queueFlush results in submitting the current batch @@ -99,7 +95,7 @@ struct urBatchedQueueTest : uur::urContextTest { // The maximum arbitrarily set capacity is reached, but the vector is not // cleared - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 0); + ASSERT_EQ(ctx->getCommandListCache().getNumRegularCommandLists(), 0); std::vector output(buffer_size, 0); ASSERT_SUCCESS(urEnqueueMemBufferRead(queue1, buffer, false, 0, buffer_size, @@ -112,7 +108,7 @@ struct urBatchedQueueTest : uur::urContextTest { // reaching its arbitrarily set capacity, queueFinish is called and the // vector is cleared. Submitted batches (regular command lists) are // returned to the command list cache in their destructors. - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), + ASSERT_EQ(ctx->getCommandListCache().getNumRegularCommandLists(), v2::initialSlotsForBatches); for (size_t index = 0; index < buffer_size; index++) { @@ -191,12 +187,12 @@ event1 = enqueueSth(q1) event2 = enqueueSth(q1) enqueueSth(q2, event2) // submit the current batch from q1 enqueueSth(q1, ..., getEvent) // access to the current batchNr from q1 -getEvent->getBatch > event2->getBatch +v2::v2_cast(getEvent)->getBatch > v2::v2_cast(event2)->getBatch enqueueSth(q2, event1) // already run in q1 enqueueSth(q1, ..., getEvent2) // access to batchNr - check if the batch has // been submitted for execution for the second time -getEvent2->getBatch == getEvent->getBatch +v2::v2_cast(getEvent2)->getBatch == v2::v2_cast(getEvent)->getBatch event statuses in L0v2 are only UR_EVENT_STATUS_SUBMITTED and UR_EVENT_STATUS_COMPLETE @@ -207,15 +203,15 @@ TEST_P(urBatchedQueueTest, RunBatchOnlyWhenNeededSimple) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* blocking */ false, 0, buffer_size, data.data(), 0, nullptr, &event1)); - ASSERT_NE(event1->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event1)->getBatch(), std::nullopt); ur_event_handle_t event2 = nullptr; ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* blocking */ false, 0, buffer_size, data.data(), 0, nullptr, &event2)); - ASSERT_NE(event2->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event2)->getBatch(), std::nullopt); // Events from the same batch - ASSERT_EQ(event1->getBatch(), event2->getBatch()); + ASSERT_EQ(v2::v2_cast(event1)->getBatch(), v2::v2_cast(event2)->getBatch()); // Submit the current batch in queue1 for execution std::vector output(buffer_size, 0); @@ -227,9 +223,10 @@ TEST_P(urBatchedQueueTest, RunBatchOnlyWhenNeededSimple) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* blocking */ false, 0, buffer_size, data.data(), 0, nullptr, &getEvent1)); - ASSERT_NE(getEvent1->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(getEvent1)->getBatch(), std::nullopt); - ASSERT_EQ(getEvent1->getBatch().value(), event2->getBatch().value() + 1); + ASSERT_EQ(v2::v2_cast(getEvent1)->getBatch().value(), + v2::v2_cast(event2)->getBatch().value() + 1); // Event1 is from the batch from q1, which has been already submitted for // execution @@ -242,10 +239,11 @@ TEST_P(urBatchedQueueTest, RunBatchOnlyWhenNeededSimple) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* blocking */ false, 0, buffer_size, data.data(), 0, nullptr, &getEvent2)); - ASSERT_NE(getEvent2->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(getEvent2)->getBatch(), std::nullopt); // Events should be assigned to the same batch - ASSERT_EQ(getEvent1->getBatch(), getEvent2->getBatch()); + ASSERT_EQ(v2::v2_cast(getEvent1)->getBatch(), + v2::v2_cast(getEvent2)->getBatch()); ASSERT_SUCCESS(urQueueFinish(queue1)); ASSERT_SUCCESS(urQueueFinish(queue2)); @@ -262,25 +260,26 @@ TEST_P(urBatchedQueueTest, IncreaseGenerationNumberAfterQueueFinish) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &event1)); - ASSERT_NE(event1->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event1)->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(event1)); ur_event_handle_t event2 = nullptr; ASSERT_SUCCESS(urEnqueueMemBufferWrite( queue1, buffer, /* blocking involves queueFinish */ true, 0, buffer_size, data.data(), 0, nullptr, &event2)); - ASSERT_NE(event2->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event2)->getBatch(), std::nullopt); ur_event_handle_t event3 = nullptr; ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &event3)); - ASSERT_NE(event3->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event3)->getBatch(), std::nullopt); // Events from the same batch - ASSERT_EQ(event1->getBatch(), event2->getBatch()); + ASSERT_EQ(v2::v2_cast(event1)->getBatch(), v2::v2_cast(event2)->getBatch()); - ASSERT_EQ(event3->getBatch().value(), event2->getBatch().value() + 1); + ASSERT_EQ(v2::v2_cast(event3)->getBatch().value(), + v2::v2_cast(event2)->getBatch().value() + 1); ASSERT_SUCCESS(urQueueFinish(queue1)); @@ -353,16 +352,17 @@ TEST_P(urBatchedQueueTest, RunBatchIfNeededCommandBuffer) { // command list. Therefore, for events generated by submitting command // buffers on batched queues, the generation number of the current batch is // not tracked. - ASSERT_EQ(eventOnImmediate->getBatch(), std::nullopt); + ASSERT_EQ(v2::v2_cast(eventOnImmediate)->getBatch(), std::nullopt); ur_event_handle_t eventAfterEnqueueCmdBuff = nullptr; std::vector data(buffer_size, 42); ASSERT_SUCCESS(urEnqueueMemBufferWrite( queue1, buffer, /* blocking write involves queueFinish */ false, 0, buffer_size, data.data(), 0, nullptr, &eventAfterEnqueueCmdBuff)); - ASSERT_NE(eventAfterEnqueueCmdBuff->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(eventAfterEnqueueCmdBuff)->getBatch(), std::nullopt); - ASSERT_EQ(eventAfterEnqueueCmdBuff->getBatch(), v2::initialGenerationNumber); + ASSERT_EQ(v2::v2_cast(eventAfterEnqueueCmdBuff)->getBatch(), + v2::initialGenerationNumber); // Enqueue command buffer when the current batch is not empty if (cmd_buf_handle) { @@ -386,7 +386,8 @@ TEST_P(urBatchedQueueTest, RunBatchIfNeededCommandBuffer) { ASSERT_SUCCESS(urEnqueueMemBufferRead(queue1, output, false, 0, buffer_size, output2.data(), 0, nullptr, &eventNonemptyBatch)); - ASSERT_EQ(eventNonemptyBatch->getBatch(), v2::initialGenerationNumber + 1); + ASSERT_EQ(v2::v2_cast(eventNonemptyBatch)->getBatch(), + v2::initialGenerationNumber + 1); urQueueFinish(queue1); @@ -409,7 +410,7 @@ TEST_P(urBatchedQueueTest, RunBatchWhenNeededSameQueue) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &event1)); - ASSERT_NE(event1->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event1)->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(event1)); ur_event_handle_t event2 = nullptr; @@ -418,7 +419,7 @@ TEST_P(urBatchedQueueTest, RunBatchWhenNeededSameQueue) { 0, buffer_size, data2.data(), 0, nullptr, &event2)); ASSERT_NE(event2, nullptr); - ASSERT_NE(event2->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(event2)->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(event2)); // wait_list_view is constructed before passing arguments to command list // manager functions. Therefore, if the batch from the current queue might @@ -427,7 +428,7 @@ TEST_P(urBatchedQueueTest, RunBatchWhenNeededSameQueue) { // increased. However, there is no need to submit batches assigned to events // from the same queue, since the operations are executed in-order: either // from different consecutive batches or as part of the same batch. - ASSERT_EQ(event1->getBatch(), event2->getBatch()); + ASSERT_EQ(v2::v2_cast(event1)->getBatch(), v2::v2_cast(event2)->getBatch()); ASSERT_SUCCESS(urQueueFinish(queue1)); @@ -445,11 +446,12 @@ TEST_P(urBatchedQueueTest, RunBatchWhenNeededQueueFlush) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &eventEmpty)); - ASSERT_NE(eventEmpty->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(eventEmpty)->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(eventEmpty)); // The batch should have not been run when empty - ASSERT_EQ(eventEmpty->getBatch().value(), v2::initialGenerationNumber); + ASSERT_EQ(v2::v2_cast(eventEmpty)->getBatch().value(), + v2::initialGenerationNumber); // A non-empty batch should have been sumitted for execution and renewed // The generation number is increased @@ -461,11 +463,12 @@ TEST_P(urBatchedQueueTest, RunBatchWhenNeededQueueFlush) { output.data(), 0, nullptr, &eventNonEmpty)); - ASSERT_NE(eventNonEmpty->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(eventNonEmpty)->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(eventNonEmpty)); // The batch should not have been run when empty - ASSERT_EQ(eventNonEmpty->getBatch().value(), v2::initialGenerationNumber + 1); + ASSERT_EQ(v2::v2_cast(eventNonEmpty)->getBatch().value(), + v2::initialGenerationNumber + 1); ASSERT_SUCCESS(urQueueFinish(queue1)); @@ -484,7 +487,8 @@ TEST_P(urBatchedQueueTest, VectorOfSubmittedBatchesIsClearedQueueFlush) { } TEST_P(urBatchedQueueTest, VectorOfSubmittedBatchesIsClearedQueueFinish) { - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 0); + auto ctx = v2::v2_cast(context); + ASSERT_EQ(ctx->getCommandListCache().getNumRegularCommandLists(), 0); std::vector data(buffer_size, 42); ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, @@ -492,7 +496,7 @@ TEST_P(urBatchedQueueTest, VectorOfSubmittedBatchesIsClearedQueueFinish) { nullptr, nullptr)); // A non-empty batch should be submitted for execution and renewed ASSERT_SUCCESS(urQueueFlush(queue1)); - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 0); + ASSERT_EQ(ctx->getCommandListCache().getNumRegularCommandLists(), 0); // The vector of current batches is cleared ASSERT_SUCCESS(urQueueFinish(queue1)); @@ -502,7 +506,7 @@ TEST_P(urBatchedQueueTest, VectorOfSubmittedBatchesIsClearedQueueFinish) { // the command list cache in their destructors. The current batch is reset // during queueFinish, therefore only one command list is returned to the // command list cache - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 1); + ASSERT_EQ(ctx->getCommandListCache().getNumRegularCommandLists(), 1); } TEST_P(urBatchedQueueTest, ReuseCommandLists) { @@ -526,14 +530,14 @@ TEST_P(urBatchedQueueTest, FlushBatchAfterEnqueuedOperationsLimitIsReached) { ASSERT_SUCCESS(urEnqueueMemBufferWrite( queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &events[lastIdx])); - ASSERT_NE(events[lastIdx]->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(events[lastIdx])->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(events[lastIdx])); lastIdx++; } int64_t idxFirstGeneration = lastIdx - 1; - ASSERT_EQ(events[idxFirstGeneration]->getBatch(), + ASSERT_EQ(v2::v2_cast(events[idxFirstGeneration])->getBatch(), v2::initialGenerationNumber); // The next operation should exceed the allowed number of operations enqueued @@ -543,11 +547,11 @@ TEST_P(urBatchedQueueTest, FlushBatchAfterEnqueuedOperationsLimitIsReached) { ASSERT_SUCCESS(urEnqueueMemBufferWrite(queue1, buffer, /* isBlocking */ false, 0, buffer_size, data.data(), 0, nullptr, &events[lastIdx])); - ASSERT_NE(events[lastIdx]->getBatch(), std::nullopt); + ASSERT_NE(v2::v2_cast(events[lastIdx])->getBatch(), std::nullopt); ASSERT_NO_FATAL_FAILURE(assertEventIsSubmitted(events[lastIdx])); int64_t idxNextGeneration = lastIdx; - ASSERT_EQ(events[idxNextGeneration]->getBatch(), + ASSERT_EQ(v2::v2_cast(events[idxNextGeneration])->getBatch(), v2::initialGenerationNumber + 1); ASSERT_SUCCESS(urQueueFinish(queue1)); @@ -642,7 +646,7 @@ TEST_P(urBatchedQueueTest, EnqueueDuringGraphCaptureUsesImmediateList) { // The capture operation should not be associated with the batched queue's // regular batch generation. - ASSERT_EQ(event->getBatch(), std::nullopt); + ASSERT_EQ(v2::v2_cast(event)->getBatch(), std::nullopt); ur_exp_graph_handle_t graph = nullptr; ASSERT_SUCCESS(urQueueEndGraphCaptureExp(captureQueue, &graph)); diff --git a/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp b/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp index ac076abecf03c..f17e739f2bb34 100644 --- a/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp @@ -10,7 +10,7 @@ #include "context.hpp" #include "level_zero/common.hpp" -#include "level_zero/device.hpp" +#include "level_zero/common/device.hpp" #include "uur/fixtures.h" #include "uur/raii.h" @@ -22,6 +22,10 @@ #include #include +// The adapter types this test exercises live in `ur::level_zero::v2`; bring the +// enclosing namespace into scope so the unqualified `v2::` references resolve. +using namespace ur::level_zero; + struct CommandListCacheTest : public uur::urContextTest { void SetUp() override { // Initialize Level Zero driver is required if this test is linked @@ -36,7 +40,9 @@ UUR_INSTANTIATE_DEVICE_TEST_SUITE(CommandListCacheTest); TEST_P(CommandListCacheTest, CanStoreAndRetriveImmediateAndRegularCmdLists) { v2::supported_extensions_descriptor_t supportedExtensions(false, false, false, false); - v2::command_list_cache_t cache(context->getZeHandle(), supportedExtensions); + v2::command_list_cache_t cache(v2::v2_cast(context)->getZeHandle(), + supportedExtensions); + auto zeDevice = common_cast(device)->ZeDevice; bool IsInOrder = false; uint32_t Ordinal = 0; @@ -57,13 +63,12 @@ TEST_P(CommandListCacheTest, CanStoreAndRetriveImmediateAndRegularCmdLists) { desc.Mutable = false; // get command lists from the cache for (int i = 0; i < numListsPerType; ++i) { - regCmdListOwners.emplace_back( - cache.getRegularCommandList(device->ZeDevice, desc)); + regCmdListOwners.emplace_back(cache.getRegularCommandList(zeDevice, desc)); auto [it, _] = regCmdLists.emplace(regCmdListOwners.back().get()); ASSERT_TRUE(*it != nullptr); immCmdListOwners.emplace_back( - cache.getImmediateCommandList(device->ZeDevice, desc, Mode, Priority)); + cache.getImmediateCommandList(zeDevice, desc, Mode, Priority)); std::tie(it, _) = immCmdLists.emplace(immCmdListOwners.back().get()); ASSERT_TRUE(*it != nullptr); } @@ -74,11 +79,11 @@ TEST_P(CommandListCacheTest, CanStoreAndRetriveImmediateAndRegularCmdLists) { // verify we get back the same command lists for (int i = 0; i < numListsPerType; ++i) { - auto regCmdList = cache.getRegularCommandList(device->ZeDevice, desc); + auto regCmdList = cache.getRegularCommandList(zeDevice, desc); ASSERT_TRUE(regCmdList != nullptr); auto immCmdList = - cache.getImmediateCommandList(device->ZeDevice, desc, Mode, Priority); + cache.getImmediateCommandList(zeDevice, desc, Mode, Priority); ASSERT_TRUE(immCmdList != nullptr); ASSERT_EQ(regCmdLists.erase(regCmdList.get()), 1); @@ -93,11 +98,13 @@ TEST_P(CommandListCacheTest, CanStoreAndRetriveImmediateAndRegularCmdLists) { TEST_P(CommandListCacheTest, ImmediateCommandListsHaveProperAttributes) { v2::supported_extensions_descriptor_t supportedExtensions(false, false, false, false); - v2::command_list_cache_t cache(context->getZeHandle(), supportedExtensions); + v2::command_list_cache_t cache(v2::v2_cast(context)->getZeHandle(), + supportedExtensions); + auto zeDevice = common_cast(device)->ZeDevice; uint32_t numQueueGroups = 0; - ASSERT_EQ(zeDeviceGetCommandQueueGroupProperties(device->ZeDevice, - &numQueueGroups, nullptr), + ASSERT_EQ(zeDeviceGetCommandQueueGroupProperties(zeDevice, &numQueueGroups, + nullptr), ZE_RESULT_SUCCESS); if (numQueueGroups == 0) { @@ -106,8 +113,8 @@ TEST_P(CommandListCacheTest, ImmediateCommandListsHaveProperAttributes) { std::vector QueueGroupProperties( numQueueGroups); - ASSERT_EQ(zeDeviceGetCommandQueueGroupProperties( - device->ZeDevice, &numQueueGroups, QueueGroupProperties.data()), + ASSERT_EQ(zeDeviceGetCommandQueueGroupProperties(zeDevice, &numQueueGroups, + QueueGroupProperties.data()), ZE_RESULT_SUCCESS); bool IsInOrder = false; @@ -123,13 +130,13 @@ TEST_P(CommandListCacheTest, ImmediateCommandListsHaveProperAttributes) { desc.IsInOrder = IsInOrder; desc.Ordinal = Ordinal; desc.CopyOffloadEnable = true; - auto CommandList = cache.getImmediateCommandList(device->ZeDevice, desc, - Mode, Priority, Index); + auto CommandList = + cache.getImmediateCommandList(zeDevice, desc, Mode, Priority, Index); ze_device_handle_t ZeDevice; auto Ret = zeCommandListGetDeviceHandle(CommandList.get(), &ZeDevice); if (Ret == ZE_RESULT_SUCCESS) { - ASSERT_EQ(ZeDevice, device->ZeDevice); + ASSERT_EQ(ZeDevice, zeDevice); } else { ASSERT_EQ(Ret, ZE_RESULT_ERROR_UNSUPPORTED_FEATURE); } @@ -156,13 +163,13 @@ TEST_P(CommandListCacheTest, ImmediateCommandListsHaveProperAttributes) { desc.Ordinal = Ordinal; desc.CopyOffloadEnable = true; // verify list creation without an index - auto CommandList = cache.getImmediateCommandList( - device->ZeDevice, desc, Mode, Priority, std::nullopt); + auto CommandList = cache.getImmediateCommandList(zeDevice, desc, Mode, + Priority, std::nullopt); ze_device_handle_t ZeDevice; auto Ret = zeCommandListGetDeviceHandle(CommandList.get(), &ZeDevice); if (Ret == ZE_RESULT_SUCCESS) { - ASSERT_EQ(ZeDevice, device->ZeDevice); + ASSERT_EQ(ZeDevice, zeDevice); } else { ASSERT_EQ(Ret, ZE_RESULT_ERROR_UNSUPPORTED_FEATURE); } @@ -189,6 +196,7 @@ TEST_P(CommandListCacheTest, CommandListsAreReusedByQueues) { static constexpr int NumQueuesPerType = 5; size_t NumUniqueQueueTypes = 0; bool isBatched = false; + auto &clCache = v2::v2_cast(context)->getCommandListCache(); for (int I = 0; I < NumQueuesPerType; I++) { NumUniqueQueueTypes = 0; @@ -225,19 +233,16 @@ TEST_P(CommandListCacheTest, CommandListsAreReusedByQueues) { } } - ASSERT_EQ(context->getCommandListCache().getNumImmediateCommandLists(), - 0); - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 0); + ASSERT_EQ(clCache.getNumImmediateCommandLists(), 0); + ASSERT_EQ(clCache.getNumRegularCommandLists(), 0); } // Queues scope - ASSERT_EQ(context->getCommandListCache().getNumImmediateCommandLists(), - NumUniqueQueueTypes); + ASSERT_EQ(clCache.getNumImmediateCommandLists(), NumUniqueQueueTypes); if (isBatched) { - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), - NumUniqueQueueTypes); + ASSERT_EQ(clCache.getNumRegularCommandLists(), NumUniqueQueueTypes); } else { - ASSERT_EQ(context->getCommandListCache().getNumRegularCommandLists(), 0); + ASSERT_EQ(clCache.getNumRegularCommandLists(), 0); } } } @@ -246,9 +251,10 @@ TEST_P(CommandListCacheTest, CommandListsCacheIsThreadSafe) { static constexpr int NumThreads = 10; static constexpr int NumIters = 10; + auto &clCache = v2::v2_cast(context)->getCommandListCache(); std::vector Threads; for (int I = 0; I < NumThreads; I++) { - Threads.emplace_back([I, this]() { + Threads.emplace_back([I, this, &clCache]() { for (int J = 0; J < NumIters; J++) { ur_queue_properties_t QueueProps{UR_STRUCTURE_TYPE_QUEUE_PROPERTIES, nullptr, 0}; @@ -263,8 +269,7 @@ TEST_P(CommandListCacheTest, CommandListsCacheIsThreadSafe) { ASSERT_EQ(urQueueCreate(context, device, &QueueProps, Queue.ptr()), UR_RESULT_SUCCESS); - ASSERT_LE(context->getCommandListCache().getNumImmediateCommandLists(), - NumThreads); + ASSERT_LE(clCache.getNumImmediateCommandLists(), NumThreads); } }); } @@ -273,6 +278,5 @@ TEST_P(CommandListCacheTest, CommandListsCacheIsThreadSafe) { Thread.join(); } - ASSERT_LE(context->getCommandListCache().getNumImmediateCommandLists(), - NumThreads); + ASSERT_LE(clCache.getNumImmediateCommandLists(), NumThreads); } diff --git a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp index d559a517898e6..b70873ddabd08 100644 --- a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp @@ -13,7 +13,7 @@ #include "command_list_cache.hpp" #include "level_zero/common.hpp" -#include "level_zero/device.hpp" +#include "level_zero/common/device.hpp" #include "level_zero/ur_interface_loader.hpp" #include "../ze_helpers.hpp" @@ -33,24 +33,10 @@ #include #include -using namespace v2; +namespace v2 = ur::level_zero::v2; static constexpr size_t MAX_DEVICES = 10; -const ur_dditable_t *ur::level_zero::ddi_getter::value() { - static ur_dditable_t table{}; - table.Event.pfnRelease = ur::level_zero::urEventRelease; - return &table; -}; - -// mock necessary functions from context, we can't pull in entire context -// implementation due to a lot of other dependencies -std::vector mockVec{}; -const std::vector & -ur_context_handle_t_::getDevices() const { - return mockVec; -} - enum ProviderType { TEST_PROVIDER_NORMAL, TEST_PROVIDER_COUNTER, @@ -67,15 +53,15 @@ static const char *provider_to_str(ProviderType p) { } } -static std::string flags_to_str(event_flags_t flags) { +static std::string flags_to_str(v2::event_flags_t flags) { std::string str; - if (flags & EVENT_FLAGS_COUNTER) { + if (flags & v2::EVENT_FLAGS_COUNTER) { str += "provider_counter"; } else { str += "provider_normal"; } - if (flags & EVENT_FLAGS_PROFILING_ENABLED) { + if (flags & v2::EVENT_FLAGS_PROFILING_ENABLED) { str += "_profiling"; } else { str += "_no_profiling"; @@ -84,11 +70,11 @@ static std::string flags_to_str(event_flags_t flags) { return str; } -static const char *queue_to_str(queue_type e) { +static const char *queue_to_str(v2::queue_type e) { switch (e) { - case QUEUE_REGULAR: + case v2::QUEUE_REGULAR: return "QUEUE_REGULAR"; - case QUEUE_IMMEDIATE: + case v2::QUEUE_IMMEDIATE: return "QUEUE_IMMEDIATE"; default: return nullptr; @@ -97,7 +83,7 @@ static const char *queue_to_str(queue_type e) { struct ProviderParams { ProviderType provider; - event_flags_t flags; + v2::event_flags_t flags; v2::queue_type queue; }; @@ -127,40 +113,40 @@ struct EventPoolTest : public uur::urQueueTestWithParam { // statically with Level Zero loader, the driver will not be init otherwise. zeInit(ZE_INIT_FLAG_GPU_ONLY); - mockVec.push_back(device); - - cache = std::unique_ptr(new event_pool_cache( + cache = std::unique_ptr(new v2::event_pool_cache( nullptr, MAX_DEVICES, - [this, params](DeviceId, - event_flags_t flags) -> std::unique_ptr { + [this, params](ur::level_zero::DeviceId, v2::event_flags_t flags) + -> std::unique_ptr { // normally id would be used to find the appropriate device to create // the provider switch (params.provider) { case TEST_PROVIDER_COUNTER: - return std::make_unique( - platform, context, params.queue, device, params.flags); + return std::make_unique( + ur::level_zero::common_cast(platform), v2::v2_cast(context), + params.queue, ur::level_zero::common_cast(device), + params.flags); case TEST_PROVIDER_NORMAL: - return std::make_unique(context, params.queue, - flags); + return std::make_unique(v2::v2_cast(context), + params.queue, flags); } return nullptr; })); } void TearDown() override { cache.reset(); - mockVec.clear(); UUR_RETURN_ON_FATAL_FAILURE(urQueueTestWithParam::TearDown()); } - std::unique_ptr cache; + std::unique_ptr cache; }; static ProviderParams test_cases[] = { - {TEST_PROVIDER_NORMAL, 0, QUEUE_REGULAR}, - {TEST_PROVIDER_NORMAL, EVENT_FLAGS_COUNTER, QUEUE_REGULAR}, - {TEST_PROVIDER_NORMAL, EVENT_FLAGS_COUNTER, QUEUE_IMMEDIATE}, - {TEST_PROVIDER_NORMAL, EVENT_FLAGS_COUNTER | EVENT_FLAGS_PROFILING_ENABLED, - QUEUE_IMMEDIATE}, + {TEST_PROVIDER_NORMAL, 0, v2::QUEUE_REGULAR}, + {TEST_PROVIDER_NORMAL, v2::EVENT_FLAGS_COUNTER, v2::QUEUE_REGULAR}, + {TEST_PROVIDER_NORMAL, v2::EVENT_FLAGS_COUNTER, v2::QUEUE_IMMEDIATE}, + {TEST_PROVIDER_NORMAL, + v2::EVENT_FLAGS_COUNTER | v2::EVENT_FLAGS_PROFILING_ENABLED, + v2::QUEUE_IMMEDIATE}, // TODO: counter provided is not fully unimplemented // counter-based provider ignores event and queue type //{TEST_PROVIDER_COUNTER, EVENT_COUNTER, QUEUE_IMMEDIATE}, @@ -177,30 +163,31 @@ TEST_P(EventPoolTest, InvalidDevice) { } TEST_P(EventPoolTest, Basic) { + auto deviceId = ur::level_zero::common_cast(device)->Id.value(); { - ur_event_handle_t first; + v2::ur_event_handle_t first; ze_event_handle_t zeFirst; { - auto pool = cache->borrow(device->Id.value(), getParam().flags); + auto pool = cache->borrow(deviceId, getParam().flags); first = pool->allocate(); first->setQueue(nullptr); first->setCommandType(UR_COMMAND_KERNEL_LAUNCH); zeFirst = first->getZeEvent(); - urEventRelease(first); + urEventRelease(v2::v2_cast(first)); } - ur_event_handle_t second; + v2::ur_event_handle_t second; ze_event_handle_t zeSecond; { - auto pool = cache->borrow(device->Id.value(), getParam().flags); + auto pool = cache->borrow(deviceId, getParam().flags); second = pool->allocate(); second->setQueue(nullptr); second->setCommandType(UR_COMMAND_KERNEL_LAUNCH); zeSecond = second->getZeEvent(); - urEventRelease(second); + urEventRelease(v2::v2_cast(second)); } ASSERT_EQ(first, second); ASSERT_EQ(zeFirst, zeSecond); @@ -209,19 +196,20 @@ TEST_P(EventPoolTest, Basic) { TEST_P(EventPoolTest, Threaded) { std::vector threads; + auto deviceId = ur::level_zero::common_cast(device)->Id.value(); for (int iters = 0; iters < 3; ++iters) { for (int th = 0; th < 10; ++th) { threads.emplace_back([&] { - auto pool = cache->borrow(device->Id.value(), getParam().flags); - std::vector events; + auto pool = cache->borrow(deviceId, getParam().flags); + std::vector events; for (int i = 0; i < 100; ++i) { events.push_back(pool->allocate()); events.back()->setQueue(nullptr); events.back()->setCommandType(UR_COMMAND_KERNEL_LAUNCH); } for (int i = 0; i < 100; ++i) { - urEventRelease(events[i]); + urEventRelease(v2::v2_cast(events[i])); } }); } @@ -233,8 +221,9 @@ TEST_P(EventPoolTest, Threaded) { } TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) { - auto pool = cache->borrow(device->Id.value(), getParam().flags); - std::list events; + auto deviceId = ur::level_zero::common_cast(device)->Id.value(); + auto pool = cache->borrow(deviceId, getParam().flags); + std::list events; for (int i = 0; i < 128; ++i) { auto event = pool->allocate(); event->setQueue(nullptr); @@ -243,7 +232,7 @@ TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) { } auto frontZeHandle = events.front()->getZeEvent(); for (int i = 0; i < 8; ++i) { - urEventRelease(events.front()); + urEventRelease(v2::v2_cast(events.front())); events.pop_front(); } for (int i = 0; i < 8; ++i) { @@ -257,7 +246,7 @@ TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) { ASSERT_EQ(frontZeHandle, events.back()->getZeEvent()); for (auto e : events) { - urEventRelease(e); + urEventRelease(v2::v2_cast(e)); } } @@ -275,7 +264,7 @@ TEST_P(EventPoolTestWithQueue, WithTimestamp) { // Skip due to driver bug causing a sigbus SKIP_IF_DRIVER_TOO_OLD("Level-Zero", minL0DriverVersion, platform, device); - if (!(getParam().flags & EVENT_FLAGS_PROFILING_ENABLED)) { + if (!(getParam().flags & v2::EVENT_FLAGS_PROFILING_ENABLED)) { GTEST_SKIP() << "Profiling needs to be enabled"; } @@ -301,7 +290,7 @@ TEST_P(EventPoolTestWithQueue, WithTimestamp) { ur_event_handle_t second; ASSERT_SUCCESS(urEnqueueEventsWaitWithBarrier(queue, 0, nullptr, &second)); // even if the event is reused, it should not be timestamped anymore - ASSERT_FALSE(second->isTimestamped()); + ASSERT_FALSE(v2::v2_cast(second)->isTimestamped()); ASSERT_SUCCESS(urEventRelease(second)); ASSERT_EQ(zeEventHostSignal(zeEvent.get()), ZE_RESULT_SUCCESS);