diff --git a/sycl/cmake/modules/BuildUnifiedRuntime.cmake b/sycl/cmake/modules/BuildUnifiedRuntime.cmake index 2dd021ee418ac..a71857ac17f91 100644 --- a/sycl/cmake/modules/BuildUnifiedRuntime.cmake +++ b/sycl/cmake/modules/BuildUnifiedRuntime.cmake @@ -182,6 +182,24 @@ if("offload" IN_LIST SYCL_ENABLE_BACKENDS) add_sycl_ur_adapter(offload) endif() +# Determine whether the given adapter is statically linked into the loader by +# querying the corresponding UR_STATIC_ADAPTER_* variable (if it exists). +# Sets OUT_VAR in the caller's scope to TRUE if the adapter is static, +# FALSE otherwise. +function(ur_adapter_is_static adapter out_var) + set(static_var "UR_STATIC_ADAPTER_${adapter}") + string(TOUPPER "${static_var}" static_var) + # Special case: level_zero uses UR_STATIC_ADAPTER_L0 + if(adapter STREQUAL "level_zero") + set(static_var "UR_STATIC_ADAPTER_L0") + endif() + if(DEFINED ${static_var} AND ${static_var}) + set(${out_var} TRUE PARENT_SCOPE) + else() + set(${out_var} FALSE PARENT_SCOPE) + endif() +endfunction() + if(CMAKE_SYSTEM_NAME STREQUAL Windows) # On Windows, also build/install debug libraries with the d suffix that are # compiled with /MDd so users can link against these in debug builds. @@ -220,6 +238,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) -DUR_BUILD_ADAPTER_HIP:BOOL=${UR_BUILD_ADAPTER_HIP} -DUR_BUILD_ADAPTER_NATIVE_CPU:BOOL=${UR_BUILD_ADAPTER_NATIVE_CPU} -DUR_STATIC_LOADER:BOOL=${UR_STATIC_LOADER} + -DUR_STATIC_ADAPTER_OPENCL:BOOL=${UR_STATIC_ADAPTER_OPENCL} -DUMF_BUILD_EXAMPLES:BOOL=${UMF_BUILD_EXAMPLES} -DUMF_BUILD_SHARED_LIBRARY:BOOL=${UMF_BUILD_SHARED_LIBRARY} -DUMF_LINK_HWLOC_STATICALLY:BOOL=${UMF_LINK_HWLOC_STATICALLY} @@ -268,14 +287,25 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) ${LLVM_BINARY_DIR}/lib/ur_loaderd.lib ${LLVM_BINARY_DIR}/lib/ur_commond.lib dbghelp) + # Link static adapters into the loader + foreach(adapter ${SYCL_ENABLE_BACKENDS}) + ur_adapter_is_static(${adapter} adapter_is_static) + set(build_var "UR_BUILD_ADAPTER_${adapter}") + string(TOUPPER "${build_var}" build_var) + if(adapter_is_static AND DEFINED ${build_var} AND ${build_var}) + target_link_libraries(UnifiedRuntimeLoaderDebug INTERFACE + ${LLVM_BINARY_DIR}/lib/ur_adapter_${adapter}d.lib) + endif() + endforeach() endif() - foreach(adatper ${SYCL_ENABLE_BACKENDS}) - if(adapter MATCHES "level_zero") - set(shared "NOT;${UR_STATIC_ADAPTER_L0}") + foreach(adapter ${SYCL_ENABLE_BACKENDS}) + ur_adapter_is_static(${adapter} adapter_is_static) + if(adapter_is_static) + set(shared FALSE) else() set(shared TRUE) endif() - urd_copy_library_to_build(ur_adapter_${adatper}d "${shared}") + urd_copy_library_to_build(ur_adapter_${adapter}d "${shared}") endforeach() # Also copy umfd.dll/umfd.lib urd_copy_library_to_build(umfd ${UMF_BUILD_SHARED_LIBRARY}) @@ -290,9 +320,12 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) DESTINATION "bin" COMPONENT unified-runtime-loader) endif() foreach(adapter ${SYCL_ENABLE_BACKENDS}) - install( - FILES ${URD_INSTALL_DIR}/bin/ur_adapter_${adapter}d.dll - DESTINATION "bin" COMPONENT ur_adapter_${adapter}) + ur_adapter_is_static(${adapter} adapter_is_static) + if(NOT adapter_is_static) + install( + FILES ${URD_INSTALL_DIR}/bin/ur_adapter_${adapter}d.dll + DESTINATION "bin" COMPONENT ur_adapter_${adapter}) + endif() endforeach() if(UMF_BUILD_SHARED_LIBRARY) # Also install umfd.dll diff --git a/sycl/include/sycl/detail/os_util.hpp b/sycl/include/sycl/detail/os_util.hpp index 5fa8e02f1bb7d..3159840948760 100644 --- a/sycl/include/sycl/detail/os_util.hpp +++ b/sycl/include/sycl/detail/os_util.hpp @@ -110,14 +110,18 @@ fn *dynLookupFunction(const char *const *LibNames, size_t LibNameSize, return reinterpret_cast(dynLookup(LibNames, LibNameSize, FunName)); } -// On Linux, first try to load from libur_adapter_opencl.so, then -// libur_adapter_opencl.so.0 if the first is not found. libur_adapter_opencl.so -// and libur_adapter_opencl.so.0 might be different libraries if they are not -// symlinked, which is the case with PyPi compiler distribution package. -// We can't load libur_adapter_opencl.so.0 always as the first choice because -// that would break SYCL unittests, which rely on mocking libur_adapter_opencl. +// On Linux, probe libur_adapter_opencl.so first, then libur_adapter_opencl.so.0 +// (these may differ in PyPi distributions where they are not symlinked). +// +// When the OpenCL adapter is statically linked into the loader, there is no +// libur_adapter_opencl.so to look up symbols in. The static adapter dlopens +// libOpenCL.so.1 (or libOpenCL.so) at initialization, so probe those names +// instead to find clRetain* and friends. #ifdef __SYCL_RT_OS_WINDOWS constexpr std::array OCLLibNames = {"OpenCL"}; +#elif defined(SYCL_UR_STATIC_ADAPTER_OPENCL) +constexpr std::array OCLLibNames = {"libOpenCL.so.1", + "libOpenCL.so"}; #else constexpr std::array OCLLibNames = { "libur_adapter_opencl.so", "libur_adapter_opencl.so.0"}; diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt index 0cc5b2cac2b8a..b7c119e177296 100644 --- a/sycl/source/CMakeLists.txt +++ b/sycl/source/CMakeLists.txt @@ -68,6 +68,7 @@ function(add_sycl_rt_library LIB_NAME LIB_OBJ_NAME) __SYCL_DISABLE_DEPRECATION_WARNINGS $<$:__SYCL_BUILD_SYCL_DLL> $<$:SYCL_UR_STATIC_LOADER> + $<$:SYCL_UR_STATIC_ADAPTER_OPENCL> ) target_include_directories( diff --git a/sycl/unittests/mock_opencl/CMakeLists.txt b/sycl/unittests/mock_opencl/CMakeLists.txt index d9508d80a045e..b90323a9a32fc 100644 --- a/sycl/unittests/mock_opencl/CMakeLists.txt +++ b/sycl/unittests/mock_opencl/CMakeLists.txt @@ -1,11 +1,14 @@ get_target_property(SYCL_BINARY_DIR sycl-toolchain BINARY_DIR) -# Linux looks up ur_adapter_opencl rather than libOpenCL. -# On Windows, this is copied into libOpenCL.dll. +# On Windows: mock provides libOpenCL.dll (always). +# On Linux: mock is always OpenCL (the ICD), with VERSION 1 so CMake produces +# both libOpenCL.so.1 and libOpenCL.so symlinks. The static adapter dlopens +# libOpenCL.so.1, and the dynamic adapter links to it at build time. if(WIN32) set(LIBNAME OpenCL) else() - set(LIBNAME ur_adapter_opencl) + set(LIBNAME OpenCL) + set(MOCK_OCL_VERSION 1) endif() add_library(mockOpenCL SHARED EXCLUDE_FROM_ALL mock_opencl.cpp) @@ -15,3 +18,6 @@ set_target_properties(mockOpenCL PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${SYCL_BINARY_DIR}/unittests/lib RUNTIME_OUTPUT_NAME ${LIBNAME} ) +if(DEFINED MOCK_OCL_VERSION) + set_target_properties(mockOpenCL PROPERTIES VERSION ${MOCK_OCL_VERSION}) +endif() diff --git a/unified-runtime/CMakeLists.txt b/unified-runtime/CMakeLists.txt index d2cb05fc3f5ee..28256344244a4 100644 --- a/unified-runtime/CMakeLists.txt +++ b/unified-runtime/CMakeLists.txt @@ -48,6 +48,7 @@ option(UR_BUILD_ADAPTER_ALL "Build all currently supported adapters" OFF) option(UR_BUILD_ADAPTER_L0_V2 "Build the (experimental) Level-Zero v2 adapter" OFF) option(UR_BUILD_ADAPTER_OFFLOAD "Build the experimental Offload adapter" OFF) option(UR_STATIC_ADAPTER_L0 "Build the Level-Zero adapter as static and embed in the loader" OFF) +option(UR_STATIC_ADAPTER_OPENCL "Build the OpenCL adapter as static library with dynamic OpenCL loading" ON) option(UR_BUILD_EXAMPLE_CODEGEN "Build the codegen example." OFF) option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF) option(UR_ENABLE_ASSERTIONS "Enable assertions for all build types" OFF) diff --git a/unified-runtime/scripts/generate_code.py b/unified-runtime/scripts/generate_code.py index eed7b7a170d53..08568eab81447 100644 --- a/unified-runtime/scripts/generate_code.py +++ b/unified-runtime/scripts/generate_code.py @@ -468,6 +468,12 @@ def generate_adapters(path, section, namespace, tags, version, specs, meta): loc += _mako_interface_loader_api( dstpath, "level_zero", "hpp", namespace, tags, version, specs, meta ) + loc += _mako_interface_loader_api( + dstpath, "opencl", "cpp", namespace, tags, version, specs, meta + ) + loc += _mako_interface_loader_api( + dstpath, "opencl", "hpp", namespace, tags, version, specs, meta + ) print("Generated %s lines of code.\n" % loc) diff --git a/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako b/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako index b8ff7c1174efb..aa27674f2c923 100644 --- a/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako +++ b/unified-runtime/scripts/templates/ur_interface_loader.hpp.mako @@ -40,7 +40,7 @@ ${x}_result_t ${th.make_func_name(n, tags, obj)}( %endif %endfor %endfor -#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO +#ifdef UR_STATIC_ADAPTER_${Adapter} ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); #endif diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index feb8cb0a018ee..908bb021f0a8c 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -64,7 +64,7 @@ if(UR_BUILD_ADAPTER_L0) target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_LEVEL_ZERO) if(UR_STATIC_ADAPTER_L0) target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO) - set(ADAPTER_L0_TARGETS_TO_INSTALL ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers) + set(ADAPTER_L0_TARGETS_TO_INSTALL LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers) # 'utils' target from 'level-zero-loader' includes path which is prefixed # in the source directory, this breaks the installation of 'utils' target. if(TARGET level_zero_utils) diff --git a/unified-runtime/source/adapters/opencl/CMakeLists.txt b/unified-runtime/source/adapters/opencl/CMakeLists.txt index fec5b83709566..05882e1968ef8 100644 --- a/unified-runtime/source/adapters/opencl/CMakeLists.txt +++ b/unified-runtime/source/adapters/opencl/CMakeLists.txt @@ -10,7 +10,13 @@ find_package(Threads REQUIRED) set(TARGET_NAME ur_adapter_opencl) -add_ur_adapter(${TARGET_NAME} SHARED +# Determine adapter library type (SHARED or STATIC) +set(ADAPTER_LIB_TYPE SHARED) +if(UR_STATIC_ADAPTER_OPENCL) + set(ADAPTER_LIB_TYPE STATIC) +endif() + +add_ur_adapter(${TARGET_NAME} ${ADAPTER_LIB_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/adapter.hpp ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp @@ -61,11 +67,45 @@ target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../" ) -target_link_libraries(${TARGET_NAME} PRIVATE - ${PROJECT_NAME}::headers - ${PROJECT_NAME}::common - ${PROJECT_NAME}::umf - Threads::Threads - OpenCL-Headers - ${OpenCL_LIBRARY} -) +if(UR_STATIC_ADAPTER_OPENCL) + # When building as static library, add dynamic loading source files + # and don't link against OpenCL library + target_sources(${TARGET_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/ocl_dynamic_lib.cpp + ) + target_compile_definitions(${TARGET_NAME} PRIVATE + UR_STATIC_ADAPTER_OPENCL + # Neutralize dllexport/dllimport annotations on MSVC: the generated .cpp + # files use UR_APIEXPORT/UR_APICALL but a static library must not have + # __declspec(dllimport) on definitions (causes C2375) and does not need + # __declspec(dllexport) either. + UR_APIEXPORT= + UR_APICALL= + UR_DLLEXPORT= + ) + target_link_libraries(${TARGET_NAME} PRIVATE + ${PROJECT_NAME}::headers + ${PROJECT_NAME}::common + ${PROJECT_NAME}::umf + Threads::Threads + ) + # OpenCL-Headers is header-only; use its include path directly to avoid + # exporting a target whose INTERFACE_INCLUDE_DIRECTORIES points into the + # build tree, which CMake rejects in install(EXPORT ...). + target_include_directories(${TARGET_NAME} PRIVATE + $) + # Add dl library for dynamic loading on Unix systems + if(UNIX) + target_link_libraries(${TARGET_NAME} PRIVATE ${CMAKE_DL_LIBS}) + endif() +else() + # Standard shared library build - link against OpenCL library + target_link_libraries(${TARGET_NAME} PRIVATE + ${PROJECT_NAME}::headers + ${PROJECT_NAME}::common + ${PROJECT_NAME}::umf + Threads::Threads + OpenCL-Headers + ${OpenCL_LIBRARY} + ) +endif() diff --git a/unified-runtime/source/adapters/opencl/adapter.cpp b/unified-runtime/source/adapters/opencl/adapter.cpp index 031829d7fb6cd..66f1f46def10c 100644 --- a/unified-runtime/source/adapters/opencl/adapter.cpp +++ b/unified-runtime/source/adapters/opencl/adapter.cpp @@ -11,18 +11,39 @@ #include "common.hpp" #include "ur/ur.hpp" +#ifndef UR_STATIC_ADAPTER_OPENCL #ifdef _MSC_VER #include #else #include #endif +#endif // There can only be one OpenCL adapter alive at a time. // If it is alive (more get/retains than releases called), this is a pointer to // it. static ur_adapter_handle_t liveAdapter = nullptr; -ur_adapter_handle_t_::ur_adapter_handle_t_() : handle_base() { +ur::opencl::ur_adapter_handle_t_::ur_adapter_handle_t_() : handle_base() { +#ifdef UR_STATIC_ADAPTER_OPENCL + if (!ocl::loadOCLLibrary()) { + return; + } + // Probe for platforms: a stub ICD may load successfully but expose none. + cl_uint NumPlatforms = 0; + cl_int Result = ocl::clGetPlatformIDs_ptr(0, nullptr, &NumPlatforms); + if (Result != CL_SUCCESS || NumPlatforms == 0) { + // No platforms found — unload immediately. This adapter never escapes + // urAdapterGet, so no teardown paths can hold function pointers yet. + ocl::unloadOCLLibrary(); + return; + } + // Mirror the symbols already resolved by ocl_dynamic_lib into the per-adapter + // struct fields, so call sites like `getAdapter()->FIELD(...)` work. +#define CL_CORE_FUNCTION(FUNC, FIELD) FIELD = ocl::FUNC##_ptr; +#include "core_functions.def" +#undef CL_CORE_FUNCTION +#else #ifdef _MSC_VER // Retrieving handle of an already linked OpenCL.dll library doesn't increase @@ -30,27 +51,35 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() : handle_base() { auto handle = GetModuleHandleA("OpenCL.dll"); assert(handle); -#define CL_CORE_FUNCTION(FUNC) \ - FUNC = reinterpret_cast(GetProcAddress(handle, #FUNC)); +#define CL_CORE_FUNCTION(FUNC, FIELD) \ + FIELD = reinterpret_cast(GetProcAddress(handle, #FUNC)); #include "core_functions.def" #undef CL_CORE_FUNCTION #else // _MSC_VER // Use the default shared object search order (RTLD_DEFAULT) since the // OpenCL-ICD-Loader has already been loaded into the process. -#define CL_CORE_FUNCTION(FUNC) \ - FUNC = reinterpret_cast(dlsym(RTLD_DEFAULT, #FUNC)); +#define CL_CORE_FUNCTION(FUNC, FIELD) \ + FIELD = reinterpret_cast(dlsym(RTLD_DEFAULT, #FUNC)); #include "core_functions.def" #undef CL_CORE_FUNCTION #endif // _MSC_VER +#endif // UR_STATIC_ADAPTER_OPENCL assert(!liveAdapter); - liveAdapter = this; + liveAdapter = cast(this); } -ur_adapter_handle_t_::~ur_adapter_handle_t_() { - assert(liveAdapter == this); +ur::opencl::ur_adapter_handle_t_::~ur_adapter_handle_t_() { +#ifdef UR_STATIC_ADAPTER_OPENCL + if (liveAdapter == cast(this)) { + liveAdapter = nullptr; + ocl::unloadOCLLibrary(); + } +#else + assert(liveAdapter == cast(this)); liveAdapter = nullptr; +#endif } ur_adapter_handle_t ur::cl::getAdapter() { @@ -60,26 +89,37 @@ ur_adapter_handle_t ur::cl::getAdapter() { return liveAdapter; } +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urAdapterGet(uint32_t NumEntries, ur_adapter_handle_t *phAdapters, uint32_t *pNumAdapters) { static std::mutex AdapterConstructionMutex{}; - if (NumEntries > 0 && phAdapters) { - std::lock_guard Lock{AdapterConstructionMutex}; + // Always construct the adapter, even for count-only queries (NumEntries==0), + // because the loader may bypass the intercept and call this directly when + // there is only one platform registered. + std::lock_guard Lock{AdapterConstructionMutex}; + if (!liveAdapter) { + ur::opencl::ur_adapter_handle_t_ *newAdapter = + new ur::opencl::ur_adapter_handle_t_(); if (!liveAdapter) { - *phAdapters = new ur_adapter_handle_t_(); - } else { - *phAdapters = liveAdapter; + delete newAdapter; + if (pNumAdapters) { + *pNumAdapters = 0; + } + return UR_RESULT_ERROR_UNINITIALIZED; } + } - auto &adapter = *phAdapters; - adapter->RefCount.retain(); + if (NumEntries > 0 && phAdapters) { + *phAdapters = liveAdapter; + cast(liveAdapter)->RefCount.retain(); } if (pNumAdapters) { - *pNumAdapters = 1; + *pNumAdapters = liveAdapter != nullptr ? 1 : 0; } return UR_RESULT_SUCCESS; @@ -87,14 +127,16 @@ urAdapterGet(uint32_t NumEntries, ur_adapter_handle_t *phAdapters, UR_APIEXPORT ur_result_t UR_APICALL urAdapterRetain(ur_adapter_handle_t hAdapter) { - hAdapter->RefCount.retain(); + auto Adapter = cast(hAdapter); + Adapter->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urAdapterRelease(ur_adapter_handle_t hAdapter) { - if (hAdapter->RefCount.release()) { - delete hAdapter; + auto Adapter = cast(hAdapter); + if (Adapter->RefCount.release()) { + delete Adapter; } return UR_RESULT_SUCCESS; } @@ -115,8 +157,10 @@ urAdapterGetInfo(ur_adapter_handle_t hAdapter, ur_adapter_info_t propName, switch (propName) { case UR_ADAPTER_INFO_BACKEND: return ReturnValue(UR_BACKEND_OPENCL); - case UR_ADAPTER_INFO_REFERENCE_COUNT: - return ReturnValue(hAdapter->RefCount.getCount()); + case UR_ADAPTER_INFO_REFERENCE_COUNT: { + auto Adapter = cast(hAdapter); + return ReturnValue(Adapter->RefCount.getCount()); + } case UR_ADAPTER_INFO_VERSION: return ReturnValue(uint32_t{1}); default: @@ -131,7 +175,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterSetLoggerCallback( void *pUserData, ur_logger_level_t level = UR_LOGGER_LEVEL_QUIET) { if (hAdapter) { - hAdapter->log.setCallbackSink(pfnLoggerCallback, pUserData, level); + auto Adapter = cast(hAdapter); + Adapter->log.setCallbackSink(pfnLoggerCallback, pUserData, level); } return UR_RESULT_SUCCESS; @@ -141,8 +186,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterSetLoggerCallbackLevel( ur_adapter_handle_t hAdapter, ur_logger_level_t level) { if (hAdapter) { - hAdapter->log.setCallbackLevel(level); + auto Adapter = cast(hAdapter); + Adapter->log.setCallbackLevel(level); } return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/adapter.hpp b/unified-runtime/source/adapters/opencl/adapter.hpp index 7b17c2dff0ce5..78d809b8a69e5 100644 --- a/unified-runtime/source/adapters/opencl/adapter.hpp +++ b/unified-runtime/source/adapters/opencl/adapter.hpp @@ -17,7 +17,9 @@ #include "common/ur_ref_count.hpp" #include "logger/ur_logger.hpp" -struct ur_adapter_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_adapter_handle_t_ : handle_base { ur_adapter_handle_t_(); ~ur_adapter_handle_t_(); @@ -35,11 +37,13 @@ struct ur_adapter_handle_t_ : ur::opencl::handle_base { // Function pointers to core OpenCL entry points which may not exist in older // versions of the OpenCL-ICD-Loader are tracked here and initialized by // dynamically loading the symbol by name. -#define CL_CORE_FUNCTION(FUNC) decltype(::FUNC) *FUNC = nullptr; +#define CL_CORE_FUNCTION(FUNC, FIELD) decltype(::FUNC) *FIELD = nullptr; #include "core_functions.def" #undef CL_CORE_FUNCTION }; +} // namespace ur::opencl + namespace ur { namespace cl { ur_adapter_handle_t getAdapter(); diff --git a/unified-runtime/source/adapters/opencl/async_alloc.cpp b/unified-runtime/source/adapters/opencl/async_alloc.cpp index 9052477f15b85..94e76596fc0bf 100644 --- a/unified-runtime/source/adapters/opencl/async_alloc.cpp +++ b/unified-runtime/source/adapters/opencl/async_alloc.cpp @@ -9,6 +9,8 @@ #include +namespace ur::opencl { + UR_APIEXPORT ur_result_t urEnqueueUSMDeviceAllocExp( ur_queue_handle_t, ur_usm_pool_handle_t, const size_t, const ur_exp_async_usm_alloc_properties_t *, uint32_t, @@ -37,3 +39,5 @@ UR_APIEXPORT ur_result_t urEnqueueUSMFreeExp(ur_queue_handle_t, ur_event_handle_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/command_buffer.cpp b/unified-runtime/source/adapters/opencl/command_buffer.cpp index 65a3ead00dce8..4f0e7567b7463 100644 --- a/unified-runtime/source/adapters/opencl/command_buffer.cpp +++ b/unified-runtime/source/adapters/opencl/command_buffer.cpp @@ -17,10 +17,12 @@ #include "queue.hpp" #include "sampler.hpp" +namespace ur::opencl { + /// The ur_exp_command_buffer_handle_t_ destructor calls CL release /// command-buffer to free the underlying object. ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() { - urQueueRelease(hInternalQueue); + ur::opencl::urQueueRelease(cast(hInternalQueue)); if (LastSubmission) { clReleaseEvent(LastSubmission); } @@ -30,7 +32,7 @@ ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() { cl_int Res = cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clReleaseCommandBufferKHRCache, + cast(ur::cl::getAdapter())->fnCache.clReleaseCommandBufferKHRCache, cl_ext::ReleaseCommandBufferName, &clReleaseCommandBufferKHR); assert(Res == CL_SUCCESS); (void)Res; @@ -43,18 +45,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( const ur_exp_command_buffer_desc_t *pCommandBufferDesc, ur_exp_command_buffer_handle_t *phCommandBuffer) { - cl_context CLContext = hContext->CLContext; + auto Context = cast(hContext); + cl_context CLContext = Context->CLContext; cl_ext::clCreateCommandBufferKHR_fn clCreateCommandBufferKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clCreateCommandBufferKHRCache, + cast(ur::cl::getAdapter())->fnCache.clCreateCommandBufferKHRCache, cl_ext::CreateCommandBufferName, &clCreateCommandBufferKHR)); const bool IsUpdatable = pCommandBufferDesc->isUpdatable; ur_device_command_buffer_update_capability_flags_t UpdateCapabilities; - cl_device_id CLDevice = hDevice->CLDevice; + auto Device = cast(hDevice); + cl_device_id CLDevice = Device->CLDevice; CL_RETURN_ON_FAILURE( getDeviceCommandBufferUpdateCapabilities(CLDevice, UpdateCapabilities)); bool DeviceSupportsUpdate = UpdateCapabilities > 0; @@ -81,21 +85,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( QueueProperties.flags = UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE; } UR_RETURN_ON_FAILURE( - urQueueCreate(hContext, hDevice, &QueueProperties, &Queue)); + ur::opencl::urQueueCreate(hContext, hDevice, &QueueProperties, &Queue)); cl_int Res = CL_SUCCESS; - const cl_command_queue CLQueue = Queue->CLQueue; + auto QueuePtr = cast(Queue); + const cl_command_queue CLQueue = QueuePtr->CLQueue; auto CLCommandBuffer = clCreateCommandBufferKHR(1, &CLQueue, Properties, &Res); if (Res != CL_SUCCESS) { - urQueueRelease(Queue); + ur::opencl::urQueueRelease(Queue); CL_RETURN_ON_FAILURE_AND_SET_NULL(Res, phCommandBuffer); } try { auto URCommandBuffer = std::make_unique( - Queue, hContext, hDevice, CLCommandBuffer, IsUpdatable, IsInOrder); - *phCommandBuffer = URCommandBuffer.release(); + QueuePtr, cast(hContext), cast(hDevice), CLCommandBuffer, IsUpdatable, + IsInOrder); + *phCommandBuffer = cast(URCommandBuffer.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -108,14 +114,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) { - hCommandBuffer->RefCount.retain(); + auto CommandBuffer = cast(hCommandBuffer); + CommandBuffer->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) { - if (hCommandBuffer->RefCount.release()) { - delete hCommandBuffer; + auto CommandBuffer = cast(hCommandBuffer); + if (CommandBuffer->RefCount.release()) { + delete CommandBuffer; } return UR_RESULT_SUCCESS; @@ -123,18 +131,20 @@ urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) { UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer) { - UR_ASSERT(!hCommandBuffer->IsFinalized, UR_RESULT_ERROR_INVALID_OPERATION); - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + UR_ASSERT(!CommandBuffer->IsFinalized, UR_RESULT_ERROR_INVALID_OPERATION); + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; cl_ext::clFinalizeCommandBufferKHR_fn clFinalizeCommandBufferKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clFinalizeCommandBufferKHRCache, + cast(ur::cl::getAdapter())->fnCache.clFinalizeCommandBufferKHRCache, cl_ext::FinalizeCommandBufferName, &clFinalizeCommandBufferKHR)); CL_RETURN_ON_FAILURE( - clFinalizeCommandBufferKHR(hCommandBuffer->CLCommandBuffer)); - hCommandBuffer->IsFinalized = true; + clFinalizeCommandBufferKHR(CommandBuffer->CLCommandBuffer)); + CommandBuffer->IsFinalized = true; return UR_RESULT_SUCCESS; } @@ -155,49 +165,55 @@ urCommandBufferAppendKernelLaunchWithArgsExp( (void)phEventWaitList; (void)phEvent; + auto CommandBuffer = cast(hCommandBuffer); + auto Kernel = cast(hKernel); // Command handles can only be obtained from updatable command-buffers - UR_ASSERT(!(phCommandHandle && !hCommandBuffer->IsUpdatable), + UR_ASSERT(!(phCommandHandle && !CommandBuffer->IsUpdatable), UR_RESULT_ERROR_INVALID_OPERATION); - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; clSetKernelArgMemPointerINTEL_fn SetKernelArgMemPointerPtr = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clSetKernelArgMemPointerINTELCache, + cast(ur::cl::getAdapter()) + ->fnCache.clSetKernelArgMemPointerINTELCache, cl_ext::SetKernelArgMemPointerName, &SetKernelArgMemPointerPtr)); for (uint32_t i = 0; i < numArgs; i++) { switch (pArgs[i].type) { case UR_EXP_KERNEL_ARG_TYPE_LOCAL: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, nullptr)); break; case UR_EXP_KERNEL_ARG_TYPE_VALUE: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, pArgs[i].value.value)); break; case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: { - cl_mem mem = pArgs[i].value.memObjTuple.hMem - ? pArgs[i].value.memObjTuple.hMem->CLMemory - : nullptr; - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + auto Mem = pArgs[i].value.memObjTuple.hMem + ? cast(pArgs[i].value.memObjTuple.hMem) + : nullptr; + cl_mem mem = Mem ? Mem->CLMemory : nullptr; + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, &mem)); break; } case UR_EXP_KERNEL_ARG_TYPE_POINTER: CL_RETURN_ON_FAILURE(SetKernelArgMemPointerPtr( - hKernel->CLKernel, static_cast(pArgs[i].index), + Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].value.pointer)); break; case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { - CL_RETURN_ON_FAILURE(clSetKernelArg( - hKernel->CLKernel, static_cast(pArgs[i].index), - pArgs[i].size, &pArgs[i].value.sampler->CLSampler)); + auto Sampler = cast(pArgs[i].value.sampler); + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, + static_cast(pArgs[i].index), + pArgs[i].size, &Sampler->CLSampler)); break; } default: @@ -205,7 +221,7 @@ urCommandBufferAppendKernelLaunchWithArgsExp( } } - return urCommandBufferAppendKernelLaunchExp( + return ur::opencl::urCommandBufferAppendKernelLaunchExp( hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, 0, nullptr, numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, phEvent, @@ -227,21 +243,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( (void)phEventWaitList; (void)phEvent; + auto CommandBuffer = cast(hCommandBuffer); + auto Kernel = cast(hKernel); // Command handles can only be obtained from updatable command-buffers - UR_ASSERT(!(phCommandHandle && !hCommandBuffer->IsUpdatable), + UR_ASSERT(!(phCommandHandle && !CommandBuffer->IsUpdatable), UR_RESULT_ERROR_INVALID_OPERATION); - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; cl_ext::clCommandNDRangeKernelKHR_fn clCommandNDRangeKernelKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clCommandNDRangeKernelKHRCache, + cast(ur::cl::getAdapter())->fnCache.clCommandNDRangeKernelKHRCache, cl_ext::CommandNRRangeKernelName, &clCommandNDRangeKernelKHR)); cl_mutable_command_khr CommandHandle = nullptr; cl_mutable_command_khr *OutCommandHandle = - hCommandBuffer->IsUpdatable ? &CommandHandle : nullptr; + CommandBuffer->IsUpdatable ? &CommandHandle : nullptr; cl_command_properties_khr UpdateProperties[] = { CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR, @@ -252,27 +271,27 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( 0}; cl_command_properties_khr *Properties = - hCommandBuffer->IsUpdatable ? UpdateProperties : nullptr; + CommandBuffer->IsUpdatable ? UpdateProperties : nullptr; - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandNDRangeKernelKHR( - hCommandBuffer->CLCommandBuffer, nullptr, Properties, hKernel->CLKernel, + CommandBuffer->CLCommandBuffer, nullptr, Properties, Kernel->CLKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, WaitListSize, SyncPointWaitList, RetSyncPoint, OutCommandHandle)); try { auto Handle = std::make_unique( - hCommandBuffer, CommandHandle, hKernel, workDim, + CommandBuffer, CommandHandle, Kernel, workDim, pLocalWorkSize != nullptr); if (phCommandHandle) { - *phCommandHandle = Handle.get(); + *phCommandHandle = cast(Handle.get()); } - hCommandBuffer->CommandHandles.push_back(std::move(Handle)); + CommandBuffer->CommandHandles.push_back(std::move(Handle)); } catch (...) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } @@ -288,20 +307,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) { // No extension entry-point exists for USM memcpy, use SVM memcpy in // preparation for USVM. - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; cl_ext::clCommandSVMMemcpyKHR_fn clCommandSVMMemcpyKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clCommandSVMMemcpyKHRCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clCommandSVMMemcpyKHRCache, cl_ext::CommandSVMMemcpyName, &clCommandSVMMemcpyKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandSVMMemcpyKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, pDst, pSrc, size, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, pDst, pSrc, size, WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr)); return UR_RESULT_SUCCESS; @@ -316,20 +338,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *) { // No extension entry-point exists for USM fill, use SVM fill in preparation // for USVM. - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; cl_ext::clCommandSVMMemFillKHR_fn clCommandSVMMemFillKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clCommandSVMMemFillKHRCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clCommandSVMMemFillKHRCache, cl_ext::CommandSVMMemFillName, &clCommandSVMMemFillKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE( - clCommandSVMMemFillKHR(hCommandBuffer->CLCommandBuffer, nullptr, nullptr, + clCommandSVMMemFillKHR(CommandBuffer->CLCommandBuffer, nullptr, nullptr, pMemory, pPattern, patternSize, size, WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr)); @@ -348,21 +373,26 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( (void)phEventWaitList; (void)phEvent; (void)phCommand; - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto SrcMem = cast(hSrcMem); + auto DstMem = cast(hDstMem); + auto Context = CommandBuffer->hContext; + cl_context CLContext = Context->CLContext; cl_ext::clCommandCopyBufferKHR_fn clCommandCopyBufferKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clCommandCopyBufferKHRCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clCommandCopyBufferKHRCache, cl_ext::CommandCopyBufferName, &clCommandCopyBufferKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandCopyBufferKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, hSrcMem->CLMemory, - hDstMem->CLMemory, srcOffset, dstOffset, size, WaitListSize, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, SrcMem->CLMemory, + DstMem->CLMemory, srcOffset, dstOffset, size, WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr)); return UR_RESULT_SUCCESS; @@ -390,22 +420,25 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( size_t OpenCLDstRect[3]{dstOrigin.x, dstOrigin.y, dstOrigin.z}; size_t OpenCLRegion[3]{region.width, region.height, region.depth}; - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto SrcMem = cast(hSrcMem); + auto DstMem = cast(hDstMem); + cl_context CLContext = CommandBuffer->hContext->CLContext; cl_ext::clCommandCopyBufferRectKHR_fn clCommandCopyBufferRectKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clCommandCopyBufferRectKHRCache, + cast(ur::cl::getAdapter())->fnCache.clCommandCopyBufferRectKHRCache, cl_ext::CommandCopyBufferRectName, &clCommandCopyBufferRectKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandCopyBufferRectKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, hSrcMem->CLMemory, - hDstMem->CLMemory, OpenCLOriginRect, OpenCLDstRect, OpenCLRegion, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, SrcMem->CLMemory, + DstMem->CLMemory, OpenCLOriginRect, OpenCLDstRect, OpenCLRegion, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr)); @@ -499,20 +532,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( [[maybe_unused]] ur_event_handle_t *phEvent, [[maybe_unused]] ur_exp_command_buffer_command_handle_t *phCommand) { - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto Buffer = cast(hBuffer); + cl_context CLContext = CommandBuffer->hContext->CLContext; cl_ext::clCommandFillBufferKHR_fn clCommandFillBufferKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clCommandFillBufferKHRCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clCommandFillBufferKHRCache, cl_ext::CommandFillBufferName, &clCommandFillBufferKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandFillBufferKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, hBuffer->CLMemory, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, Buffer->CLMemory, pPattern, patternSize, offset, size, WaitListSize, SyncPointWaitList, RetSyncPoint, nullptr)); @@ -570,44 +606,47 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp( uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + auto Queue = cast(hQueue); + cl_context CLContext = CommandBuffer->hContext->CLContext; cl_ext::clEnqueueCommandBufferKHR_fn clEnqueueCommandBufferKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clEnqueueCommandBufferKHRCache, + cast(ur::cl::getAdapter())->fnCache.clEnqueueCommandBufferKHRCache, cl_ext::EnqueueCommandBufferName, &clEnqueueCommandBufferKHR)); // If we've submitted the command-buffer before, then add a dependency on the // last submission. - bool AddExtraDep = hCommandBuffer->LastSubmission != nullptr; + bool AddExtraDep = CommandBuffer->LastSubmission != nullptr; uint32_t CLWaitListLength = AddExtraDep ? numEventsInWaitList + 1 : numEventsInWaitList; std::vector CLWaitEvents(CLWaitListLength); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } if (AddExtraDep) { - CLWaitEvents[numEventsInWaitList] = hCommandBuffer->LastSubmission; + CLWaitEvents[numEventsInWaitList] = CommandBuffer->LastSubmission; } // Always get an event as we need it to serialize any future submissions // of the command-buffer with. cl_event Event; - cl_command_queue CLQueue = hQueue->CLQueue; + cl_command_queue CLQueue = Queue->CLQueue; CL_RETURN_ON_FAILURE( - clEnqueueCommandBufferKHR(1, &CLQueue, hCommandBuffer->CLCommandBuffer, + clEnqueueCommandBufferKHR(1, &CLQueue, CommandBuffer->CLCommandBuffer, CLWaitListLength, CLWaitEvents.data(), &Event)); // Retain event so that if a user manually destroys the returned UR // event the adapter still has a valid handle. clRetainEvent(Event); - if (hCommandBuffer->LastSubmission) { - clReleaseEvent(hCommandBuffer->LastSubmission); + if (CommandBuffer->LastSubmission) { + clReleaseEvent(CommandBuffer->LastSubmission); } - hCommandBuffer->LastSubmission = Event; + CommandBuffer->LastSubmission = Event; - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); return UR_RESULT_SUCCESS; } @@ -647,7 +686,7 @@ void updateKernelArgs(std::vector &CLArgs, for (uint32_t i = 0; i < NumMemobjArgs; i++) { const ur_exp_command_buffer_update_memobj_arg_desc_t &URMemObjArg = ArgMemobjList[i]; - cl_mem arg_value = URMemObjArg.hNewMemObjArg->CLMemory; + cl_mem arg_value = cast(URMemObjArg.hNewMemObjArg)->CLMemory; cl_mutable_dispatch_arg_khr CLArg{ URMemObjArg.argIndex, // arg_index sizeof(cl_mem), // arg_size @@ -670,19 +709,20 @@ void updateKernelArgs(std::vector &CLArgs, } ur_result_t validateCommandDesc( - ur_exp_command_buffer_handle_t CommandBuffer, + ur_exp_command_buffer_handle_t hCommandBuffer, const ur_exp_command_buffer_update_kernel_launch_desc_t &UpdateDesc) { + auto CommandBuffer = cast(hCommandBuffer); if (!CommandBuffer->IsFinalized || !CommandBuffer->IsUpdatable) { return UR_RESULT_ERROR_INVALID_OPERATION; } - auto Command = UpdateDesc.hCommand; - if (CommandBuffer != Command->hCommandBuffer) { + auto Command = cast(UpdateDesc.hCommand); + if (hCommandBuffer != cast(Command->hCommandBuffer)) { return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP; } // Kernel handle updates are not yet supported. - if (UpdateDesc.hNewKernel && UpdateDesc.hNewKernel != Command->Kernel) { + if (UpdateDesc.hNewKernel && UpdateDesc.hNewKernel != cast(Command->Kernel)) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -694,7 +734,7 @@ ur_result_t validateCommandDesc( // Verify that the device supports updating the aspects of the kernel that // the user is requesting. - ur_device_handle_t URDevice = CommandBuffer->hDevice; + auto URDevice = CommandBuffer->hDevice; cl_device_id CLDevice = URDevice->CLDevice; ur_device_command_buffer_update_capability_flags_t UpdateCapabilities = 0; @@ -747,13 +787,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( validateCommandDesc(hCommandBuffer, pUpdateKernelLaunch[i])); } - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + cl_context CLContext = CommandBuffer->hContext->CLContext; cl_ext::clUpdateMutableCommandsKHR_fn clUpdateMutableCommandsKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clUpdateMutableCommandsKHRCache, + cast(ur::cl::getAdapter())->fnCache.clUpdateMutableCommandsKHRCache, cl_ext::UpdateMutableCommandsName, &clUpdateMutableCommandsKHR)); std::vector ConfigList(numKernelUpdates); @@ -790,7 +831,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( updateKernelArgs(CLArgs, UpdateDesc); // Find the updated ND-Range configuration of the kernel. - auto Command = UpdateDesc.hCommand; + auto Command = cast(UpdateDesc.hCommand); cl_uint &CommandWorkDim = Command->WorkDim; if (auto GlobalWorkOffsetPtr = UpdateDesc.pNewGlobalWorkOffset) { @@ -829,7 +870,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( ConfigPtrs[i] = &ConfigList[i]; } CL_RETURN_ON_FAILURE(clUpdateMutableCommandsKHR( - hCommandBuffer->CLCommandBuffer, NumConfigs, ConfigTypes.data(), + CommandBuffer->CLCommandBuffer, NumConfigs, ConfigTypes.data(), (const void **)ConfigPtrs.data())); return UR_RESULT_SUCCESS; @@ -854,15 +895,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetInfoExp( size_t *pPropSizeRet) { UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + auto CommandBuffer = cast(hCommandBuffer); switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(hCommandBuffer->RefCount.getCount()); + return ReturnValue(CommandBuffer->RefCount.getCount()); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; Descriptor.pNext = nullptr; - Descriptor.isUpdatable = hCommandBuffer->IsUpdatable; + Descriptor.isUpdatable = CommandBuffer->IsUpdatable; Descriptor.isInOrder = false; Descriptor.enableProfiling = false; @@ -883,23 +925,25 @@ ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( uint32_t numSyncPointsInWaitList, const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ur_exp_command_buffer_sync_point_t *pSyncPoint) { - cl_context CLContext = hCommandBuffer->hContext->CLContext; + auto CommandBuffer = cast(hCommandBuffer); + cl_context CLContext = CommandBuffer->hContext->CLContext; cl_ext::clCommandBarrierWithWaitListKHR_fn clCommandBarrierWithWaitListKHR = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clCommandBarrierWithWaitListKHRCache, + cast(ur::cl::getAdapter()) + ->fnCache.clCommandBarrierWithWaitListKHRCache, cl_ext::CommandBarrierWithWaitListName, &clCommandBarrierWithWaitListKHR)); - const bool IsInOrder = hCommandBuffer->IsInOrder; + const bool IsInOrder = CommandBuffer->IsInOrder; cl_sync_point_khr *RetSyncPoint = IsInOrder ? nullptr : pSyncPoint; const cl_sync_point_khr *SyncPointWaitList = IsInOrder ? nullptr : pSyncPointWaitList; uint32_t WaitListSize = IsInOrder ? 0 : numSyncPointsInWaitList; CL_RETURN_ON_FAILURE(clCommandBarrierWithWaitListKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, WaitListSize, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, WaitListSize, SyncPointWaitList, nullptr, nullptr)); // Call user-defined function immediately @@ -907,7 +951,7 @@ ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( // Barrier on all commands after user defined commands. CL_RETURN_ON_FAILURE(clCommandBarrierWithWaitListKHR( - hCommandBuffer->CLCommandBuffer, nullptr, nullptr, 0, nullptr, + CommandBuffer->CLCommandBuffer, nullptr, nullptr, 0, nullptr, RetSyncPoint, nullptr)); return UR_RESULT_SUCCESS; @@ -916,7 +960,10 @@ ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, ur_native_handle_t *phNativeCommandBuffer) { + auto CommandBuffer = cast(hCommandBuffer); *phNativeCommandBuffer = - reinterpret_cast(hCommandBuffer->CLCommandBuffer); + reinterpret_cast(CommandBuffer->CLCommandBuffer); return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/command_buffer.hpp b/unified-runtime/source/adapters/opencl/command_buffer.hpp index 3bfabd78c06f4..31badc7e899da 100644 --- a/unified-runtime/source/adapters/opencl/command_buffer.hpp +++ b/unified-runtime/source/adapters/opencl/command_buffer.hpp @@ -11,22 +11,24 @@ #include "common/ur_ref_count.hpp" #include +namespace ur::opencl { + /// Handle to a kernel command. -struct ur_exp_command_buffer_command_handle_t_ : ur::opencl::handle_base { +struct ur_exp_command_buffer_command_handle_t_ : handle_base { /// Command-buffer this command belongs to. - ur_exp_command_buffer_handle_t hCommandBuffer; + ur_exp_command_buffer_handle_t_ *hCommandBuffer; /// OpenCL command-handle. cl_mutable_command_khr CLMutableCommand; /// Kernel associated with this command handle - ur_kernel_handle_t Kernel; + ur_kernel_handle_t_ *Kernel; /// Work-dimension the command was originally created with. cl_uint WorkDim; /// Set to true if the user set the local work size on command creation. bool UserDefinedLocalSize; ur_exp_command_buffer_command_handle_t_( - ur_exp_command_buffer_handle_t hCommandBuffer, - cl_mutable_command_khr CLMutableCommand, ur_kernel_handle_t Kernel, + ur_exp_command_buffer_handle_t_ *hCommandBuffer, + cl_mutable_command_khr CLMutableCommand, ur_kernel_handle_t_ *Kernel, cl_uint WorkDim, bool UserDefinedLocalSize) : handle_base(), hCommandBuffer(hCommandBuffer), CLMutableCommand(CLMutableCommand), Kernel(Kernel), WorkDim(WorkDim), @@ -34,13 +36,13 @@ struct ur_exp_command_buffer_command_handle_t_ : ur::opencl::handle_base { }; /// Handle to a command-buffer object. -struct ur_exp_command_buffer_handle_t_ : ur::opencl::handle_base { +struct ur_exp_command_buffer_handle_t_ : handle_base { /// UR queue belonging to the command-buffer, required for OpenCL creation. - ur_queue_handle_t hInternalQueue; + ur_queue_handle_t_ *hInternalQueue; /// Context the command-buffer is created for. - ur_context_handle_t hContext; + ur_context_handle_t_ *hContext; /// Device the command-buffer is created for. - ur_device_handle_t hDevice; + ur_device_handle_t_ *hDevice; /// OpenCL command-buffer object. cl_command_buffer_khr CLCommandBuffer; /// Set to true if the kernel commands in the command-buffer can be updated, @@ -58,9 +60,9 @@ struct ur_exp_command_buffer_handle_t_ : ur::opencl::handle_base { ur::RefCount RefCount; - ur_exp_command_buffer_handle_t_(ur_queue_handle_t hQueue, - ur_context_handle_t hContext, - ur_device_handle_t hDevice, + ur_exp_command_buffer_handle_t_(ur_queue_handle_t_ *hQueue, + ur_context_handle_t_ *hContext, + ur_device_handle_t_ *hDevice, cl_command_buffer_khr CLCommandBuffer, bool IsUpdatable, bool IsInOrder) : handle_base(), hInternalQueue(hQueue), hContext(hContext), @@ -70,3 +72,5 @@ struct ur_exp_command_buffer_handle_t_ : ur::opencl::handle_base { ~ur_exp_command_buffer_handle_t_(); }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/common.hpp b/unified-runtime/source/adapters/opencl/common.hpp index aefea1f6c55a0..527e126f8fe5e 100644 --- a/unified-runtime/source/adapters/opencl/common.hpp +++ b/unified-runtime/source/adapters/opencl/common.hpp @@ -14,6 +14,12 @@ #include #include + +// In static-adapter builds this redirects OpenCL calls to dynamically-loaded +// function pointers. The header is self-guarded and expands to nothing +// otherwise. +#include "ocl_dynamic_lib.hpp" + #include #include #include @@ -162,11 +168,143 @@ extern thread_local char ErrorMessage[MaxMessageSize]; [[maybe_unused]] void setErrorMessage(const char *Message, int32_t ErrorCode); } // namespace cl_adapter +#include "ur_interface_loader.hpp" + namespace ur::opencl { -struct ddi_getter { - const static ur_dditable_t *value(); -}; using handle_base = ur::handle_base; + +// Forward declarations of the adapter's internal handle types. Full +// definitions live in the per-type headers (context.hpp, device.hpp, ...). +// They reside in `namespace ur::opencl` so that the global opaque +// `::ur_X_handle_t` types from ur_api.h do not collide when several adapters +// are statically linked into the same image. +struct ur_adapter_handle_t_; +struct ur_platform_handle_t_; +struct ur_device_handle_t_; +struct ur_context_handle_t_; +struct ur_event_handle_t_; +struct ur_program_handle_t_; +struct ur_kernel_handle_t_; +struct ur_queue_handle_t_; +struct ur_sampler_handle_t_; +struct ur_mem_handle_t_; +struct ur_physical_mem_handle_t_; +struct ur_exp_command_buffer_handle_t_; +struct ur_exp_command_buffer_command_handle_t_; + +// Cast from an opaque UR handle to the adapter's concrete type and back. The +// handles passed through the UR API are opaque, so adapter code uses these +// helpers to reach the internal members. Both trait mappings are intentionally +// left undefined for unregistered types so that `cast` below participates in +// overload resolution only for known handles. +// +// TODO: the two specialization lists below are mechanical inverses of each +// other. They could be collapsed into a single +// UR_CL_HANDLE_MAP(::ur_context_handle_t, ur_context_handle_t_) +// X-macro that emits both directions at once. The same applies to the L0 +// adapter's v1_handle_traits/v1_opaque_handle_for pair (see #21845); worth +// unifying once both PRs have landed. +namespace detail { +// Maps an opaque handle typedef to its corresponding internal struct. +template struct handle_traits; +template <> struct handle_traits<::ur_adapter_handle_t> { + using type = ur_adapter_handle_t_; +}; +template <> struct handle_traits<::ur_platform_handle_t> { + using type = ur_platform_handle_t_; +}; +template <> struct handle_traits<::ur_device_handle_t> { + using type = ur_device_handle_t_; +}; +template <> struct handle_traits<::ur_context_handle_t> { + using type = ur_context_handle_t_; +}; +template <> struct handle_traits<::ur_event_handle_t> { + using type = ur_event_handle_t_; +}; +template <> struct handle_traits<::ur_program_handle_t> { + using type = ur_program_handle_t_; +}; +template <> struct handle_traits<::ur_kernel_handle_t> { + using type = ur_kernel_handle_t_; +}; +template <> struct handle_traits<::ur_queue_handle_t> { + using type = ur_queue_handle_t_; +}; +template <> struct handle_traits<::ur_sampler_handle_t> { + using type = ur_sampler_handle_t_; +}; +template <> struct handle_traits<::ur_mem_handle_t> { + using type = ur_mem_handle_t_; +}; +template <> struct handle_traits<::ur_physical_mem_handle_t> { + using type = ur_physical_mem_handle_t_; +}; +template <> struct handle_traits<::ur_exp_command_buffer_handle_t> { + using type = ur_exp_command_buffer_handle_t_; +}; +template <> struct handle_traits<::ur_exp_command_buffer_command_handle_t> { + using type = ur_exp_command_buffer_command_handle_t_; +}; + +template +using internal_t = typename handle_traits::type; + +// Reverse mapping: internal struct -> opaque handle. +template struct opaque_handle_for; +template <> struct opaque_handle_for { + using type = ::ur_adapter_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_platform_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_device_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_context_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_event_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_program_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_kernel_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_queue_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_sampler_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_mem_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_physical_mem_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_exp_command_buffer_handle_t; +}; +template <> struct opaque_handle_for { + using type = ::ur_exp_command_buffer_command_handle_t; +}; +} // namespace detail + +// Opaque handle -> internal pointer. +template +inline detail::internal_t *cast(Opaque handle) { + return reinterpret_cast *>(handle); +} + +// Internal pointer -> opaque handle (reverse direction). +template +inline typename detail::opaque_handle_for::type cast(Internal *ptr) { + return reinterpret_cast::type>( + ptr); +} } // namespace ur::opencl namespace cl_ext { @@ -244,17 +382,17 @@ using clEnqueueReadGlobalVariableINTEL_fn = CL_API_ENTRY cl_event *); using clEnqueueReadHostPipeINTEL_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_queue queue, cl_program program, - const char *pipe_symbol, cl_bool blocking, void *ptr, - size_t size, cl_uint num_events_in_waitlist, - const cl_event *events_waitlist, cl_event *event); + cl_int(CL_API_CALL *)(cl_command_queue queue, cl_program program, + const char *pipe_symbol, cl_bool blocking, void *ptr, + size_t size, cl_uint num_events_in_waitlist, + const cl_event *events_waitlist, cl_event *event); using clEnqueueWriteHostPipeINTEL_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_queue queue, cl_program program, - const char *pipe_symbol, cl_bool blocking, - const void *ptr, size_t size, - cl_uint num_events_in_waitlist, - const cl_event *events_waitlist, cl_event *event); + cl_int(CL_API_CALL *)(cl_command_queue queue, cl_program program, + const char *pipe_symbol, cl_bool blocking, + const void *ptr, size_t size, + cl_uint num_events_in_waitlist, + const cl_event *events_waitlist, cl_event *event); using clCreateCommandBufferKHR_fn = CL_API_ENTRY cl_command_buffer_khr( CL_API_CALL *)(cl_uint num_queues, const cl_command_queue *queues, @@ -262,13 +400,13 @@ using clCreateCommandBufferKHR_fn = CL_API_ENTRY cl_command_buffer_khr( cl_int *errcode_ret); using clRetainCommandBufferKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); + cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); using clReleaseCommandBufferKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); + cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); using clFinalizeCommandBufferKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); + cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer); using clCommandNDRangeKernelKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)( cl_command_buffer_khr command_buffer, cl_command_queue command_queue, @@ -328,26 +466,27 @@ using clCommandSVMMemFillKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)( cl_sync_point_khr *sync_point, cl_mutable_command_khr *mutable_handle); using clEnqueueCommandBufferKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_uint num_queues, cl_command_queue *queues, - cl_command_buffer_khr command_buffer, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, cl_event *event); + cl_int(CL_API_CALL *)(cl_uint num_queues, cl_command_queue *queues, + cl_command_buffer_khr command_buffer, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); using clGetCommandBufferInfoKHR_fn = CL_API_ENTRY cl_int(CL_API_CALL *)( cl_command_buffer_khr command_buffer, cl_command_buffer_info_khr param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret); using clUpdateMutableCommandsKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer, cl_uint num_configs, - const cl_command_buffer_update_type_khr *config_types, - const void **configs); + cl_int(CL_API_CALL *)(cl_command_buffer_khr command_buffer, + cl_uint num_configs, + const cl_command_buffer_update_type_khr *config_types, + const void **configs); using clCreateProgramWithILKHR_fn = CL_API_ENTRY -cl_program(CL_API_CALL *)(cl_context, const void *, size_t, cl_int *); + cl_program(CL_API_CALL *)(cl_context, const void *, size_t, cl_int *); using clGetKernelSubGroupInfoKHR_fn = CL_API_ENTRY -cl_int(CL_API_CALL *)(cl_kernel, cl_device_id, cl_kernel_sub_group_info, size_t, - const void *, size_t, void *, size_t *); + cl_int(CL_API_CALL *)(cl_kernel, cl_device_id, cl_kernel_sub_group_info, + size_t, const void *, size_t, void *, size_t *); template struct FuncPtrCache { std::map Map; diff --git a/unified-runtime/source/adapters/opencl/context.cpp b/unified-runtime/source/adapters/opencl/context.cpp index f4a876cc0dd39..66540ac4aff76 100644 --- a/unified-runtime/source/adapters/opencl/context.cpp +++ b/unified-runtime/source/adapters/opencl/context.cpp @@ -14,6 +14,8 @@ #include #include +namespace ur::opencl { + ur_result_t ur_context_handle_t_::makeWithNative(native_type Ctx, uint32_t DevCount, const ur_device_handle_t *phDevices, @@ -26,16 +28,16 @@ ur_context_handle_t_::makeWithNative(native_type Ctx, uint32_t DevCount, std::vector CLDevices(CLDeviceCount); CL_RETURN_ON_FAILURE(clGetContextInfo( Ctx, CL_CONTEXT_DEVICES, sizeof(CLDevices), CLDevices.data(), nullptr)); - std::vector URDevices; + std::vector URDevices; if (DevCount) { if (DevCount != CLDeviceCount) { return UR_RESULT_ERROR_INVALID_CONTEXT; } for (uint32_t i = 0; i < DevCount; i++) { - if (phDevices[i]->CLDevice != CLDevices[i]) { + if (cast(phDevices[i])->CLDevice != CLDevices[i]) { return UR_RESULT_ERROR_INVALID_CONTEXT; } - URDevices.push_back(phDevices[i]); + URDevices.push_back(cast(phDevices[i])); } } else { DevCount = CLDeviceCount; @@ -43,15 +45,15 @@ ur_context_handle_t_::makeWithNative(native_type Ctx, uint32_t DevCount, ur_device_handle_t UrDevice = nullptr; ur_native_handle_t hNativeHandle = reinterpret_cast(CLDevices[i]); - UR_RETURN_ON_FAILURE(urDeviceCreateWithNativeHandle( + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceCreateWithNativeHandle( hNativeHandle, nullptr, nullptr, &UrDevice)); - URDevices.push_back(UrDevice); + URDevices.push_back(cast(UrDevice)); } } auto URContext = std::make_unique(Ctx, DevCount, URDevices.data()); - Context = URContext.release(); + Context = cast(URContext.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -67,8 +69,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreate( cl_int Ret; std::vector CLDevices(DeviceCount); + std::vector URDevices(DeviceCount); for (size_t i = 0; i < DeviceCount; i++) { - CLDevices[i] = phDevices[i]->CLDevice; + URDevices[i] = cast(phDevices[i]); + CLDevices[i] = URDevices[i]->CLDevice; } try { @@ -76,9 +80,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreate( CLDevices.data(), nullptr, nullptr, static_cast(&Ret)); CL_RETURN_ON_FAILURE(Ret); - auto URContext = - std::make_unique(Ctx, DeviceCount, phDevices); - *phContext = URContext.release(); + auto URContext = std::make_unique(Ctx, DeviceCount, + URDevices.data()); + *phContext = cast(URContext.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -92,6 +96,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextGetInfo(ur_context_handle_t hContext, ur_context_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Context = cast(hContext); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (static_cast(propName)) { @@ -101,13 +106,13 @@ urContextGetInfo(ur_context_handle_t hContext, ur_context_info_t propName, return ReturnValue(false); } case UR_CONTEXT_INFO_NUM_DEVICES: { - return ReturnValue(hContext->DeviceCount); + return ReturnValue(Context->DeviceCount); } case UR_CONTEXT_INFO_DEVICES: { - return ReturnValue(&hContext->Devices[0], hContext->DeviceCount); + return ReturnValue(&Context->Devices[0], Context->DeviceCount); } case UR_CONTEXT_INFO_REFERENCE_COUNT: { - return ReturnValue(hContext->RefCount.getCount()); + return ReturnValue(Context->RefCount.getCount()); } default: return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -116,8 +121,9 @@ urContextGetInfo(ur_context_handle_t hContext, ur_context_info_t propName, UR_APIEXPORT ur_result_t UR_APICALL urContextRelease(ur_context_handle_t hContext) { - if (hContext->RefCount.release()) { - delete hContext; + auto Context = cast(hContext); + if (Context->RefCount.release()) { + delete Context; } return UR_RESULT_SUCCESS; @@ -125,14 +131,16 @@ urContextRelease(ur_context_handle_t hContext) { UR_APIEXPORT ur_result_t UR_APICALL urContextRetain(ur_context_handle_t hContext) { - hContext->RefCount.retain(); + auto Context = cast(hContext); + Context->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urContextGetNativeHandle( ur_context_handle_t hContext, ur_native_handle_t *phNativeContext) { - *phNativeContext = reinterpret_cast(hContext->CLContext); + auto Context = cast(hContext); + *phNativeContext = reinterpret_cast(Context->CLContext); return UR_RESULT_SUCCESS; } @@ -145,7 +153,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreateWithNativeHandle( cl_context NativeHandle = reinterpret_cast(hNativeContext); UR_RETURN_ON_FAILURE(ur_context_handle_t_::makeWithNative( NativeHandle, numDevices, phDevices, *phContext)); - (*phContext)->IsNativeHandleOwned = + cast(*phContext)->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; return UR_RESULT_SUCCESS; } @@ -153,8 +161,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreateWithNativeHandle( UR_APIEXPORT ur_result_t UR_APICALL urContextSetExtendedDeleter( ur_context_handle_t hContext, ur_context_extended_deleter_t pfnDeleter, void *pUserData) { - if (!ur::cl::getAdapter()->clSetContextDestructorCallback) { - UR_LOG_L(ur::cl::getAdapter()->log, WARN, + if (!cast(ur::cl::getAdapter())->clSetContextDestructorCallbackFn) { + UR_LOG_L(cast(ur::cl::getAdapter())->log, WARN, "clSetContextDestructorCallback not found, consider upgrading the " "OpenCL-ICD-Loader to the latest version."); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -203,8 +211,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextSetExtendedDeleter( auto *C = static_cast(pUserData); C->execute(); }; - CL_RETURN_ON_FAILURE(ur::cl::getAdapter()->clSetContextDestructorCallback( - hContext->CLContext, ClCallback, Callback)); + CL_RETURN_ON_FAILURE( + cast(ur::cl::getAdapter()) + ->clSetContextDestructorCallbackFn(cast(hContext)->CLContext, + ClCallback, Callback)); return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/context.hpp b/unified-runtime/source/adapters/opencl/context.hpp index bdf775d51e5b8..0428bc944edb5 100644 --- a/unified-runtime/source/adapters/opencl/context.hpp +++ b/unified-runtime/source/adapters/opencl/context.hpp @@ -15,10 +15,12 @@ #include -struct ur_context_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_context_handle_t_ : handle_base { using native_type = cl_context; native_type CLContext; - std::vector Devices; + std::vector Devices; uint32_t DeviceCount; bool IsNativeHandleOwned = true; ur::RefCount RefCount; @@ -27,11 +29,11 @@ struct ur_context_handle_t_ : ur::opencl::handle_base { ur_context_handle_t_ &operator=(const ur_context_handle_t_ &) = delete; ur_context_handle_t_(native_type Ctx, uint32_t DevCount, - const ur_device_handle_t *phDevices) + ur_device_handle_t_ *const *phDevices) : handle_base(), CLContext(Ctx), DeviceCount(DevCount) { for (uint32_t i = 0; i < DeviceCount; i++) { Devices.emplace_back(phDevices[i]); - urDeviceRetain(phDevices[i]); + ur::opencl::urDeviceRetain(cast(phDevices[i])); } } @@ -43,13 +45,15 @@ struct ur_context_handle_t_ : ur::opencl::handle_base { // clear the ext function pointer cache. This isn't foolproof sadly but it // should drastically reduce the chances of the pathological case described // in the comments in common.hpp. - ur::cl::getAdapter()->fnCache.clearCache(CLContext); + cast(ur::cl::getAdapter())->fnCache.clearCache(CLContext); for (uint32_t i = 0; i < DeviceCount; i++) { - urDeviceRelease(Devices[i]); + ur::opencl::urDeviceRelease(cast(Devices[i])); } if (IsNativeHandleOwned) { clReleaseContext(CLContext); } } }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/core_functions.def b/unified-runtime/source/adapters/opencl/core_functions.def index c1510af4896ce..dc2cb1eb11555 100644 --- a/unified-runtime/source/adapters/opencl/core_functions.def +++ b/unified-runtime/source/adapters/opencl/core_functions.def @@ -8,7 +8,7 @@ // TODO: clGetDeviceAndHostTimer // Introduced in OpenCL 2.2 -CL_CORE_FUNCTION(clSetProgramSpecializationConstant) +CL_CORE_FUNCTION(clSetProgramSpecializationConstant, clSetProgramSpecializationConstantFn) // Introduced in OpenCL 3.0 -CL_CORE_FUNCTION(clSetContextDestructorCallback) +CL_CORE_FUNCTION(clSetContextDestructorCallback, clSetContextDestructorCallbackFn) diff --git a/unified-runtime/source/adapters/opencl/device.cpp b/unified-runtime/source/adapters/opencl/device.cpp index bdd0b9fd2b2ab..637f537fa91e2 100644 --- a/unified-runtime/source/adapters/opencl/device.cpp +++ b/unified-runtime/source/adapters/opencl/device.cpp @@ -14,6 +14,8 @@ #include #include +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urDeviceGet(ur_platform_handle_t hPlatform, ur_device_type_t DeviceType, uint32_t, @@ -46,14 +48,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGet(ur_platform_handle_t hPlatform, return UR_RESULT_ERROR_INVALID_ENUMERATION; } try { - uint32_t AllDevicesNum = hPlatform->Devices.size(); + auto Platform = cast(hPlatform); + uint32_t AllDevicesNum = Platform->Devices.size(); uint32_t DeviceNumIter = 0; for (uint32_t i = 0; i < AllDevicesNum; i++) { - cl_device_type DevTy = hPlatform->Devices[i]->Type; + cl_device_type DevTy = Platform->Devices[i]->Type; if (DevTy == Type || Type == CL_DEVICE_TYPE_ALL || Type == CL_DEVICE_TYPE_DEFAULT) { if (phDevices) { - phDevices[DeviceNumIter] = hPlatform->Devices[i].get(); + phDevices[DeviceNumIter] = cast(Platform->Devices[i].get()); } DeviceNumIter++; // For default, the first device is the only returned device. @@ -127,6 +130,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, * | cl_device_atomic_capabilities | ur_memory_order_capability_flags_t | */ + auto Device = cast(hDevice); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); /* TODO UR: Casting to uint32_t to silence warnings due to some values not @@ -134,7 +138,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, * to UR */ switch (static_cast(propName)) { case UR_DEVICE_INFO_TYPE: { - cl_device_type CLType = hDevice->Type; + cl_device_type CLType = Device->Type; /* TODO UR: If the device is an Accelerator (FPGA, VPU, etc.), there is not * enough information in the OpenCL runtime to know exactly which type it @@ -156,14 +160,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_DEVICE_ID: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_ID_INTEL, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_ID_INTEL, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; @@ -171,7 +175,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_BACKEND_RUNTIME_VERSION: { oclv::OpenCLVersion Version; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(Version)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(Version)); const std::string Results = std::to_string(Version.getMajor()) + "." + std::to_string(Version.getMinor()); @@ -181,11 +185,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, const cl_device_info info_name = CL_DEVICE_PARTITION_PROPERTIES; size_t CLSize; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, info_name, 0, nullptr, &CLSize)); + clGetDeviceInfo(Device->CLDevice, info_name, 0, nullptr, &CLSize)); const size_t NProperties = CLSize / sizeof(cl_device_partition_property); std::vector CLValue(NProperties); - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, info_name, CLSize, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, info_name, CLSize, CLValue.data(), nullptr)); /* The OpenCL implementation returns a value of 0 if no properties are @@ -208,7 +212,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, const cl_device_info info_name = CL_DEVICE_PARTITION_TYPE; size_t CLSize; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, info_name, 0, nullptr, &CLSize)); + clGetDeviceInfo(Device->CLDevice, info_name, 0, nullptr, &CLSize)); const size_t NProperties = CLSize / sizeof(cl_device_partition_property); /* The OpenCL implementation returns either a size of 0 or a value of 0 if @@ -222,8 +226,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, auto CLValue = reinterpret_cast(alloca(CLSize)); - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, info_name, CLSize, - CLValue, nullptr)); + CL_RETURN_ON_FAILURE( + clGetDeviceInfo(Device->CLDevice, info_name, CLSize, CLValue, nullptr)); std::vector URValue(NProperties - 1); @@ -279,11 +283,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, /* Corresponding OpenCL query is only available starting with OpenCL 2.1 * and we have to emulate it on older OpenCL runtimes. */ oclv::OpenCLVersion DevVer; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); if (DevVer >= oclv::V2_1) { cl_uint CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_NUM_SUB_GROUPS, sizeof(cl_uint), &CLValue, nullptr)); @@ -304,7 +308,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_SINGLE_FP_CONFIG: { cl_device_fp_config CLValue; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_SINGLE_FP_CONFIG, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_SINGLE_FP_CONFIG, sizeof(cl_device_fp_config), &CLValue, nullptr)); return ReturnValue(mapCLDeviceFpConfigToUR(CLValue)); @@ -312,7 +316,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_HALF_FP_CONFIG: { bool Supported = false; UR_RETURN_ON_FAILURE( - hDevice->checkDeviceExtensions({"cl_khr_fp16"}, Supported)); + Device->checkDeviceExtensions({"cl_khr_fp16"}, Supported)); if (!Supported) { // If we don't support the extension then our capabilities are 0. @@ -322,7 +326,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, cl_device_fp_config CLValue; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_HALF_FP_CONFIG, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_HALF_FP_CONFIG, sizeof(cl_device_fp_config), &CLValue, nullptr)); return ReturnValue(mapCLDeviceFpConfigToUR(CLValue)); @@ -330,7 +334,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_DOUBLE_FP_CONFIG: { cl_device_fp_config CLValue; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_DOUBLE_FP_CONFIG, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_DOUBLE_FP_CONFIG, sizeof(cl_device_fp_config), &CLValue, nullptr)); return ReturnValue(mapCLDeviceFpConfigToUR(CLValue)); @@ -340,7 +344,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, /* This query is missing before OpenCL 3.0. Check version and handle * appropriately */ oclv::OpenCLVersion DevVer; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); /* Minimum required capability to be returned. For OpenCL 1.2, this is all * that is required */ @@ -351,7 +355,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, /* For OpenCL >=3.0, the query should be implemented */ cl_device_atomic_capabilities CLCapabilities; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_ATOMIC_MEMORY_CAPABILITIES, + Device->CLDevice, CL_DEVICE_ATOMIC_MEMORY_CAPABILITIES, sizeof(cl_device_atomic_capabilities), &CLCapabilities, nullptr)); /* Mask operation to only consider atomic_memory_order* capabilities */ @@ -396,12 +400,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP; oclv::OpenCLVersion DevVer; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); cl_device_atomic_capabilities CLCapabilities; if (DevVer >= oclv::V3_0) { CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_ATOMIC_MEMORY_CAPABILITIES, + Device->CLDevice, CL_DEVICE_ATOMIC_MEMORY_CAPABILITIES, sizeof(cl_device_atomic_capabilities), &CLCapabilities, nullptr)); assert((CLCapabilities & CL_DEVICE_ATOMIC_SCOPE_WORK_GROUP) && @@ -446,12 +450,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, UR_MEMORY_ORDER_CAPABILITY_FLAG_ACQ_REL; oclv::OpenCLVersion DevVer; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); cl_device_atomic_capabilities CLCapabilities; if (DevVer >= oclv::V3_0) { CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_ATOMIC_FENCE_CAPABILITIES, + Device->CLDevice, CL_DEVICE_ATOMIC_FENCE_CAPABILITIES, sizeof(cl_device_atomic_capabilities), &CLCapabilities, nullptr)); assert((CLCapabilities & CL_DEVICE_ATOMIC_ORDER_RELAXED) && @@ -492,7 +496,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, UR_MEMORY_SCOPE_CAPABILITY_FLAG_WORK_GROUP; oclv::OpenCLVersion DevVer; - UR_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + UR_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); auto convertCapabilities = [](cl_device_atomic_capabilities CLCapabilities) { @@ -516,7 +520,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, if (DevVer >= oclv::V3_0) { cl_device_atomic_capabilities CLCapabilities; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_ATOMIC_FENCE_CAPABILITIES, + Device->CLDevice, CL_DEVICE_ATOMIC_FENCE_CAPABILITIES, sizeof(cl_device_atomic_capabilities), &CLCapabilities, nullptr)); assert((CLCapabilities & CL_DEVICE_ATOMIC_SCOPE_WORK_GROUP) && "Violates minimum mandated guarantee"); @@ -535,7 +539,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // not return an error if the query is unsuccessful as this is expected // of an OpenCL 1.2 driver. cl_device_atomic_capabilities CLCapabilities; - if (CL_SUCCESS == clGetDeviceInfo(hDevice->CLDevice, + if (CL_SUCCESS == clGetDeviceInfo(Device->CLDevice, CL_DEVICE_ATOMIC_FENCE_CAPABILITIES, sizeof(cl_device_atomic_capabilities), &CLCapabilities, nullptr)) { @@ -553,7 +557,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_ATOMIC_64: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_khr_int64_base_atomics", "cl_khr_int64_extended_atomics"}, Supported)); @@ -562,7 +566,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_BUILD_ON_SUBDEVICE: { cl_device_type DevType = CL_DEVICE_TYPE_DEFAULT; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(cl_device_type), &DevType, nullptr)); @@ -570,7 +574,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_MEM_CHANNEL_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_mem_channel_property"}, Supported)); return ReturnValue(Supported); @@ -578,12 +582,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_ESIMD_SUPPORT: { bool Supported = false; cl_device_type DevType = CL_DEVICE_TYPE_DEFAULT; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(cl_device_type), &DevType, nullptr)); cl_uint VendorID = 0; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_VENDOR_ID, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_VENDOR_ID, sizeof(VendorID), &VendorID, nullptr)); /* ESIMD is only supported by Intel GPUs. */ @@ -597,11 +601,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_NUM_COMPUTE_UNITS: { bool ExtensionSupported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, ExtensionSupported)); cl_device_type CLType; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(cl_device_type), &CLType, nullptr)); @@ -610,15 +614,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, cl_uint SliceCount = 0; cl_uint SubSlicePerSliceCount = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_NUM_SLICES_INTEL, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NUM_SLICES_INTEL, sizeof(cl_uint), &SliceCount, nullptr)); CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_NUM_SUB_SLICES_PER_SLICE_INTEL, + Device->CLDevice, CL_DEVICE_NUM_SUB_SLICES_PER_SLICE_INTEL, sizeof(cl_uint), &SubSlicePerSliceCount, nullptr)); NumComputeUnits = SliceCount * SubSlicePerSliceCount; } else { CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_MAX_COMPUTE_UNITS, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(cl_uint), &NumComputeUnits, nullptr)); } @@ -632,20 +636,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_program_scope_host_pipe"}, Supported)); return ReturnValue(Supported); } case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_global_variable_access"}, Supported)); return ReturnValue(Supported); } case UR_DEVICE_INFO_QUEUE_PROPERTIES: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_QUEUE_PROPERTIES, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_QUEUE_PROPERTIES, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -653,7 +657,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_QUEUE_ON_DEVICE_PROPERTIES: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_QUEUE_ON_DEVICE_PROPERTIES, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_QUEUE_ON_DEVICE_PROPERTIES, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -661,7 +665,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_QUEUE_ON_HOST_PROPERTIES: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_QUEUE_ON_HOST_PROPERTIES, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_QUEUE_ON_HOST_PROPERTIES, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -669,7 +673,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_TYPE: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_GLOBAL_MEM_CACHE_TYPE, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_GLOBAL_MEM_CACHE_TYPE, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -677,7 +681,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_LOCAL_MEM_TYPE: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_LOCAL_MEM_TYPE, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_LOCAL_MEM_TYPE, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -685,7 +689,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_EXECUTION_CAPABILITIES: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_EXECUTION_CAPABILITIES, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_EXECUTION_CAPABILITIES, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -693,19 +697,19 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_PARTITION_AFFINITY_DOMAIN: { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_PARTITION_AFFINITY_DOMAIN, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PARTITION_AFFINITY_DOMAIN, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); } case UR_DEVICE_INFO_USM_HOST_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_unified_shared_memory"}, Supported)); if (Supported) { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_HOST_MEM_CAPABILITIES_INTEL, + Device->CLDevice, CL_DEVICE_HOST_MEM_CAPABILITIES_INTEL, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); } else { @@ -714,12 +718,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_USM_DEVICE_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_unified_shared_memory"}, Supported)); if (Supported) { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_DEVICE_MEM_CAPABILITIES_INTEL, + Device->CLDevice, CL_DEVICE_DEVICE_MEM_CAPABILITIES_INTEL, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); } else { @@ -728,12 +732,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_unified_shared_memory"}, Supported)); if (Supported) { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_SINGLE_DEVICE_SHARED_MEM_CAPABILITIES_INTEL, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -743,12 +747,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_unified_shared_memory"}, Supported)); if (Supported) { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_CROSS_DEVICE_SHARED_MEM_CAPABILITIES_INTEL, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -758,12 +762,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_unified_shared_memory"}, Supported)); if (Supported) { cl_bitfield CLValue = 0; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL, + Device->CLDevice, CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL, sizeof(cl_bitfield), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); } else { @@ -772,7 +776,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_IMAGE_SUPPORT: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE_SUPPORT, sizeof(cl_bool), &CLValue, nullptr)); @@ -780,7 +784,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_ERROR_CORRECTION_SUPPORT: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_ERROR_CORRECTION_SUPPORT, sizeof(cl_bool), &CLValue, nullptr)); @@ -788,7 +792,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_HOST_UNIFIED_MEMORY: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_HOST_UNIFIED_MEMORY, sizeof(cl_bool), &CLValue, nullptr)); @@ -796,7 +800,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_ENDIAN_LITTLE: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_ENDIAN_LITTLE, sizeof(cl_bool), &CLValue, nullptr)); @@ -804,14 +808,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_AVAILABLE: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_AVAILABLE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_AVAILABLE, sizeof(cl_bool), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); } case UR_DEVICE_INFO_COMPILER_AVAILABLE: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_COMPILER_AVAILABLE, sizeof(cl_bool), &CLValue, nullptr)); @@ -819,7 +823,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_LINKER_AVAILABLE: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_LINKER_AVAILABLE, sizeof(cl_bool), &CLValue, nullptr)); @@ -827,7 +831,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_PREFERRED_INTEROP_USER_SYNC: { cl_bool CLValue; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_INTEROP_USER_SYNC, sizeof(cl_bool), &CLValue, nullptr)); @@ -835,13 +839,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS: { oclv::OpenCLVersion DevVer; - CL_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + CL_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); /* Independent forward progress query is only supported as of OpenCL 2.1 * if version is older we return a default false. */ if (DevVer >= oclv::V2_1) { cl_bool CLValue; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS, + Device->CLDevice, CL_DEVICE_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS, sizeof(cl_bool), &CLValue, nullptr)); return ReturnValue(static_cast(CLValue)); @@ -850,41 +854,41 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } } case UR_DEVICE_INFO_VENDOR_ID: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_VENDOR_ID, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_VENDOR_ID, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_COMPUTE_UNITS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_COMPUTE_UNITS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_CHAR: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_CHAR, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_SHORT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_SHORT, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_INT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_INT, propSize, pPropValue, pPropSizeRet)); @@ -892,14 +896,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG: case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_LONG_LONG: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_LONG, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_FLOAT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_FLOAT, propSize, pPropValue, pPropSizeRet)); @@ -907,34 +911,34 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_DOUBLE: { CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE, propSize, + Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_HALF: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PREFERRED_VECTOR_WIDTH_HALF, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_CHAR, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_SHORT, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_INT, propSize, pPropValue, pPropSizeRet)); @@ -942,270 +946,269 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG: case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG_LONG: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_LONG, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_FLOAT, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_DOUBLE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NATIVE_VECTOR_WIDTH_HALF, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_CLOCK_FREQUENCY: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_CLOCK_FREQUENCY, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_ADDRESS_BITS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_ADDRESS_BITS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_READ_IMAGE_ARGS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_READ_IMAGE_ARGS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_WRITE_IMAGE_ARGS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_READ_WRITE_IMAGE_ARGS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_READ_WRITE_IMAGE_ARGS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MEM_BASE_ADDR_ALIGN, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_SAMPLERS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_SAMPLERS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_GLOBAL_MEM_CACHELINE_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_CONSTANT_ARGS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_CONSTANT_ARGS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_REFERENCE_COUNT: { - return ReturnValue(hDevice->RefCount.getCount()); + return ReturnValue(Device->RefCount.getCount()); } case UR_DEVICE_INFO_PARTITION_MAX_SUB_DEVICES: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PARTITION_MAX_SUB_DEVICES, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_MEM_ALLOC_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_GLOBAL_MEM_CACHE_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_GLOBAL_MEM_CACHE_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_GLOBAL_MEM_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_GLOBAL_MEM_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_CONSTANT_BUFFER_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_LOCAL_MEM_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_LOCAL_MEM_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_WORK_GROUP_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_WORK_GROUP_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE2D_MAX_WIDTH: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE2D_MAX_WIDTH, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE2D_MAX_HEIGHT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE2D_MAX_HEIGHT, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE3D_MAX_WIDTH: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE3D_MAX_WIDTH, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE3D_MAX_HEIGHT: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE3D_MAX_HEIGHT, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE3D_MAX_DEPTH: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE3D_MAX_DEPTH, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE_MAX_BUFFER_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_IMAGE_MAX_ARRAY_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IMAGE_MAX_ARRAY_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_PARAMETER_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_PARAMETER_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PROFILING_TIMER_RESOLUTION: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PROFILING_TIMER_RESOLUTION, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PRINTF_BUFFER_SIZE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PRINTF_BUFFER_SIZE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PLATFORM: { - return ReturnValue(hDevice->Platform); + return ReturnValue(Device->Platform); } case UR_DEVICE_INFO_PARENT_DEVICE: { - return ReturnValue(hDevice->ParentDevice); + return ReturnValue(Device->ParentDevice); } case UR_DEVICE_INFO_IL_VERSION: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, - CL_DEVICE_IL_VERSION, propSize, - pPropValue, pPropSizeRet)); + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IL_VERSION, + propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_NAME: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_NAME, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NAME, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_VENDOR: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_VENDOR, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_VENDOR, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_DRIVER_VERSION: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DRIVER_VERSION, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DRIVER_VERSION, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_PROFILE: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_PROFILE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PROFILE, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_VERSION: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_VERSION, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_VERSION, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_BUILT_IN_KERNELS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_BUILT_IN_KERNELS, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_WORK_ITEM_SIZES, propSize, pPropValue, pPropSizeRet)); @@ -1214,14 +1217,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_PCI_ADDRESS: { bool Supported = false; UR_RETURN_ON_FAILURE( - hDevice->checkDeviceExtensions({"cl_khr_pci_bus_info"}, Supported)); + Device->checkDeviceExtensions({"cl_khr_pci_bus_info"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } cl_device_pci_bus_info_khr PciInfo = {}; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_PCI_BUS_INFO_KHR, sizeof(PciInfo), &PciInfo, nullptr)); @@ -1237,21 +1240,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, * GPUs. */ bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } cl_device_type CLType; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(cl_device_type), &CLType, nullptr)); if (!(CLType & CL_DEVICE_TYPE_GPU)) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_MAX_COMPUTE_UNITS, propSize, pPropValue, pPropSizeRet)); @@ -1259,12 +1262,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_GPU_EU_SLICES: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NUM_SLICES_INTEL, propSize, pPropValue, pPropSizeRet)); @@ -1272,12 +1275,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NUM_EUS_PER_SUB_SLICE_INTEL, propSize, pPropValue, pPropSizeRet)); @@ -1285,25 +1288,25 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_NUM_SUB_SLICES_PER_SLICE_INTEL, propSize, + Device->CLDevice, CL_DEVICE_NUM_SUB_SLICES_PER_SLICE_INTEL, propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_GPU_HW_THREADS_PER_EU: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NUM_THREADS_PER_EU_INTEL, propSize, pPropValue, pPropSizeRet)); @@ -1311,12 +1314,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_IP_VERSION: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_device_attribute_query"}, Supported)); if (!Supported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_IP_VERSION_INTEL, propSize, pPropValue, pPropSizeRet)); @@ -1325,8 +1328,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL: { const cl_device_info info_name = CL_DEVICE_SUB_GROUP_SIZES_INTEL; bool isExtensionSupported = false; - if (hDevice->checkDeviceExtensions({"cl_intel_required_subgroup_size"}, - isExtensionSupported) != + if (Device->checkDeviceExtensions({"cl_intel_required_subgroup_size"}, + isExtensionSupported) != UR_RESULT_SUCCESS || !isExtensionSupported) { std::vector aThreadIsItsOwnSubGroup({1}); @@ -1336,10 +1339,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // Have to convert size_t to uint32_t size_t SubGroupSizesSize = 0; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, info_name, 0, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, info_name, 0, nullptr, &SubGroupSizesSize)); std::vector SubGroupSizes(SubGroupSizesSize / sizeof(size_t)); - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, info_name, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, info_name, SubGroupSizesSize, SubGroupSizes.data(), nullptr)); return ReturnValue.template operator()(SubGroupSizes.data(), @@ -1349,22 +1352,22 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_UUID: { // Use the cl_khr_device_uuid extension, if available. bool isKhrDeviceUuidSupported = false; - if (hDevice->checkDeviceExtensions({"cl_khr_device_uuid"}, - isKhrDeviceUuidSupported) != + if (Device->checkDeviceExtensions({"cl_khr_device_uuid"}, + isKhrDeviceUuidSupported) != UR_RESULT_SUCCESS || !isKhrDeviceUuidSupported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } static_assert(CL_UUID_SIZE_KHR == 16); std::array UUID{}; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_UUID_KHR, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_UUID_KHR, UUID.size(), UUID.data(), nullptr)); return ReturnValue(UUID); } case UR_DEVICE_INFO_2D_BLOCK_ARRAY_CAPABILITIES_EXP: { bool Is2DBlockIOSupported = false; - if (hDevice->checkDeviceExtensions({"cl_intel_subgroup_2d_block_io"}, - Is2DBlockIOSupported) != + if (Device->checkDeviceExtensions({"cl_intel_subgroup_2d_block_io"}, + Is2DBlockIOSupported) != UR_RESULT_SUCCESS || !Is2DBlockIOSupported) { return ReturnValue( @@ -1379,12 +1382,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, return ReturnValue(false); case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_bfloat16_conversions"}, Supported)); return ReturnValue(Supported); } case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: { - cl_device_id Dev = hDevice->CLDevice; + cl_device_id Dev = Device->CLDevice; size_t ExtSize = 0; CL_RETURN_ON_FAILURE( clGetDeviceInfo(Dev, CL_DEVICE_EXTENSIONS, 0, nullptr, &ExtSize)); @@ -1408,7 +1411,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, 0 != (Caps & CL_COMMAND_BUFFER_CAPABILITY_SIMULTANEOUS_USE_KHR)); } case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP: { - cl_device_id Dev = hDevice->CLDevice; + cl_device_id Dev = Device->CLDevice; ur_device_command_buffer_update_capability_flags_t UpdateCapabilities = 0; CL_RETURN_ON_FAILURE( getDeviceCommandBufferUpdateCapabilities(Dev, UpdateCapabilities)); @@ -1416,18 +1419,18 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_PROGRAM_SET_SPECIALIZATION_CONSTANTS: { return ReturnValue( - ur::cl::getAdapter()->clSetProgramSpecializationConstant != nullptr); + cast(ur::cl::getAdapter())->clSetProgramSpecializationConstantFn != + nullptr); } case UR_DEVICE_INFO_USE_NATIVE_ASSERT: { bool Supported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions( {"cl_intel_devicelib_assert"}, Supported)); return ReturnValue(Supported); } case UR_DEVICE_INFO_EXTENSIONS: { - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, - CL_DEVICE_EXTENSIONS, propSize, - pPropValue, pPropSizeRet)); + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_EXTENSIONS, + propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; } case UR_DEVICE_INFO_USM_P2P_SUPPORT_EXP: @@ -1448,8 +1451,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // Use the cl_khr_device_uuid extension, if available. bool isKhrDeviceLuidSupported = false; - if (hDevice->checkDeviceExtensions({"cl_khr_device_uuid"}, - isKhrDeviceLuidSupported) != + if (Device->checkDeviceExtensions({"cl_khr_device_uuid"}, + isKhrDeviceLuidSupported) != UR_RESULT_SUCCESS || !isKhrDeviceLuidSupported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; @@ -1457,7 +1460,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, cl_bool isLuidValid; CL_RETURN_ON_FAILURE( - clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_LUID_VALID_KHR, + clGetDeviceInfo(Device->CLDevice, CL_DEVICE_LUID_VALID_KHR, sizeof(cl_bool), &isLuidValid, nullptr)); if (!isLuidValid) { @@ -1466,7 +1469,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_assert(CL_LUID_SIZE_KHR == 8); std::array UUID{}; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_LUID_KHR, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_LUID_KHR, UUID.size(), UUID.data(), nullptr)); return ReturnValue(UUID); } @@ -1478,8 +1481,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // Use the cl_khr_device_uuid extension, if available. bool isKhrDeviceLuidSupported = false; - if (hDevice->checkDeviceExtensions({"cl_khr_device_uuid"}, - isKhrDeviceLuidSupported) != + if (Device->checkDeviceExtensions({"cl_khr_device_uuid"}, + isKhrDeviceLuidSupported) != UR_RESULT_SUCCESS || !isKhrDeviceLuidSupported) { return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; @@ -1487,7 +1490,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, cl_int nodeMask = 0; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_NODE_MASK_KHR, sizeof(cl_int), &nodeMask, nullptr)); @@ -1499,18 +1502,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, bool Supported = false; size_t ExtSize = 0; - CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_EXTENSIONS, 0, nullptr, &ExtSize)); + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_EXTENSIONS, + 0, nullptr, &ExtSize)); std::string ExtStr(ExtSize, '\0'); - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, - CL_DEVICE_EXTENSIONS, ExtSize, - ExtStr.data(), nullptr)); + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_EXTENSIONS, + ExtSize, ExtStr.data(), nullptr)); if (ExtStr.find("cl_khr_kernel_clock") != std::string::npos) { cl_device_kernel_clock_capabilities_khr caps = 0; CL_RETURN_ON_FAILURE(clGetDeviceInfo( - hDevice->CLDevice, CL_DEVICE_KERNEL_CLOCK_CAPABILITIES_KHR, + Device->CLDevice, CL_DEVICE_KERNEL_CLOCK_CAPABILITIES_KHR, sizeof(cl_device_kernel_clock_capabilities_khr), &caps, nullptr)); if ((propName == UR_DEVICE_INFO_CLOCK_SUB_GROUP_SUPPORT_EXP && @@ -1529,7 +1531,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // TODO: use stable API instead of deprecated CL_DEVICE_HOST_UNIFIED_MEMORY. // Currently CL_DEVICE_HOST_UNIFIED_MEMORY is deprecated by OpenCL 2.0, but // still was not removed even from Intel implementations of OpenCL 3.0. - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_HOST_UNIFIED_MEMORY, sizeof(cl_bool), &CLValue, nullptr)); @@ -1605,6 +1607,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDevicePartition( const ur_device_partition_properties_t *pProperties, uint32_t NumDevices, ur_device_handle_t *phSubDevices, uint32_t *pNumDevicesRet) { + auto Device = cast(hDevice); std::vector CLProperties( pProperties->PropCount + 2); @@ -1642,8 +1645,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDevicePartition( CLProperties[CLProperties.size() - 1] = 0; cl_uint CLNumDevicesRet; - CL_RETURN_ON_FAILURE(clCreateSubDevices( - hDevice->CLDevice, CLProperties.data(), 0, nullptr, &CLNumDevicesRet)); + CL_RETURN_ON_FAILURE(clCreateSubDevices(Device->CLDevice, CLProperties.data(), + 0, nullptr, &CLNumDevicesRet)); if (pNumDevicesRet) { *pNumDevicesRet = CLNumDevicesRet; @@ -1654,23 +1657,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urDevicePartition( if (phSubDevices) { std::vector CLSubDevices(CLNumDevicesRet); CL_RETURN_ON_FAILURE( - clCreateSubDevices(hDevice->CLDevice, CLProperties.data(), + clCreateSubDevices(Device->CLDevice, CLProperties.data(), CLNumDevicesRet, CLSubDevices.data(), nullptr)); for (uint32_t i = 0; i < std::min(CLNumDevicesRet, NumDevices); i++) { try { auto URSubDevice = std::make_unique( - CLSubDevices[i], hDevice->Platform, hDevice); - phSubDevices[i] = URSubDevice.release(); + CLSubDevices[i], Device->Platform, Device); + phSubDevices[i] = cast(URSubDevice.release()); } catch (std::bad_alloc &) { // Delete all the successfully created subdevices before the failed one. for (uint32_t j = 0; j < i; j++) { - delete phSubDevices[j]; + delete cast(phSubDevices[j]); } return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { // Delete all the successfully created subdevices before the failed one. for (uint32_t j = 0; j < i; j++) { - delete phSubDevices[j]; + delete cast(phSubDevices[j]); } return UR_RESULT_ERROR_UNKNOWN; } @@ -1682,8 +1685,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urDevicePartition( // Root devices ref count are unchanged through out the program lifetime. UR_APIEXPORT ur_result_t UR_APICALL urDeviceRetain(ur_device_handle_t hDevice) { - if (hDevice->ParentDevice) { - hDevice->RefCount.retain(); + auto Device = cast(hDevice); + if (Device->ParentDevice) { + Device->RefCount.retain(); } return UR_RESULT_SUCCESS; @@ -1692,9 +1696,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceRetain(ur_device_handle_t hDevice) { // Root devices ref count are unchanged through out the program lifetime. UR_APIEXPORT ur_result_t UR_APICALL urDeviceRelease(ur_device_handle_t hDevice) { - if (hDevice->ParentDevice) { - if (hDevice->RefCount.release()) { - delete hDevice; + auto Device = cast(hDevice); + if (Device->ParentDevice) { + if (Device->RefCount.release()) { + delete Device; } } return UR_RESULT_SUCCESS; @@ -1703,7 +1708,8 @@ urDeviceRelease(ur_device_handle_t hDevice) { UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetNativeHandle( ur_device_handle_t hDevice, ur_native_handle_t *phNativeDevice) { - *phNativeDevice = reinterpret_cast(hDevice->CLDevice); + auto Device = cast(hDevice); + *phNativeDevice = reinterpret_cast(Device->CLDevice); return UR_RESULT_SUCCESS; } @@ -1713,28 +1719,29 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle( ur_device_handle_t *phDevice) { auto SetDeviceProps = [&]() { - (*phDevice)->IsNativeHandleOwned = + cast(*phDevice)->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; }; cl_device_id NativeHandle = reinterpret_cast(hNativeDevice); uint32_t NumPlatforms = 0; - UR_RETURN_ON_FAILURE(urPlatformGet(nullptr, 0, nullptr, &NumPlatforms)); - std::vector Platforms(NumPlatforms); UR_RETURN_ON_FAILURE( - urPlatformGet(nullptr, NumPlatforms, Platforms.data(), nullptr)); + ur::opencl::urPlatformGet(nullptr, 0, nullptr, &NumPlatforms)); + std::vector Platforms(NumPlatforms); + UR_RETURN_ON_FAILURE(ur::opencl::urPlatformGet(nullptr, NumPlatforms, + Platforms.data(), nullptr)); for (uint32_t i = 0; i < NumPlatforms; i++) { uint32_t NumDevices = 0; - UR_RETURN_ON_FAILURE( - urDeviceGet(Platforms[i], UR_DEVICE_TYPE_ALL, 0, nullptr, &NumDevices)); + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceGet( + Platforms[i], UR_DEVICE_TYPE_ALL, 0, nullptr, &NumDevices)); std::vector Devices(NumDevices); - UR_RETURN_ON_FAILURE(urDeviceGet(Platforms[i], UR_DEVICE_TYPE_ALL, - NumDevices, Devices.data(), nullptr)); + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceGet( + Platforms[i], UR_DEVICE_TYPE_ALL, NumDevices, Devices.data(), nullptr)); for (auto &Device : Devices) { - if (Device->CLDevice == NativeHandle) { + if (cast(Device)->CLDevice == NativeHandle) { *phDevice = Device; SetDeviceProps(); return UR_RESULT_SUCCESS; @@ -1749,23 +1756,27 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle( if (Parent != nullptr) { ur_device_handle_t ParentUrHandle; // This will either create a new device handle, or return an existing one - UR_RETURN_ON_FAILURE(urDeviceCreateWithNativeHandle( + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceCreateWithNativeHandle( reinterpret_cast(Parent), nullptr, nullptr, &ParentUrHandle)); - ur_platform_handle_t PlatformHandle = ParentUrHandle->Platform; + auto ParentUrDevice = cast(ParentUrHandle); + ur_platform_handle_t PlatformHandle = cast(ParentUrDevice->Platform); assert(PlatformHandle); { - std::lock_guard lock{PlatformHandle->SubDevicesLock}; + auto Platform = cast(PlatformHandle); + std::lock_guard lock{Platform->SubDevicesLock}; - if (PlatformHandle->SubDevices.count(NativeHandle)) { - *phDevice = PlatformHandle->SubDevices[NativeHandle]; + if (Platform->SubDevices.count(NativeHandle)) { + *phDevice = cast(Platform->SubDevices[NativeHandle]); } else { - *phDevice = std::make_unique( - NativeHandle, PlatformHandle, ParentUrHandle) - .release(); - PlatformHandle->SubDevices[NativeHandle] = *phDevice; + auto NewDevice = + std::make_unique( + NativeHandle, cast(PlatformHandle), cast(ParentUrHandle)) + .release(); + Platform->SubDevices[NativeHandle] = NewDevice; + *phDevice = cast(NewDevice); } } @@ -1779,18 +1790,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle( UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetGlobalTimestamps( ur_device_handle_t hDevice, uint64_t *pDeviceTimestamp, uint64_t *pHostTimestamp) { + auto Device = cast(hDevice); oclv::OpenCLVersion DevVer, PlatVer; - cl_device_id DeviceId = hDevice->CLDevice; + cl_device_id DeviceId = Device->CLDevice; // TODO: Cache OpenCL version for each device and platform - auto RetErr = hDevice->getDeviceVersion(DevVer); + auto RetErr = Device->getDeviceVersion(DevVer); if (RetErr == CL_INVALID_OPERATION) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } CL_RETURN_ON_FAILURE(RetErr); - RetErr = hDevice->Platform->getPlatformVersion(PlatVer); + auto Platform = Device->Platform; + RetErr = Platform->getPlatformVersion(PlatVer); if (PlatVer < oclv::V2_1 || DevVer < oclv::V2_1) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -1825,12 +1838,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceSelectBinary( // where context->dispatch is set to the dispatch table provided by PI // plugin for platform/device the ctx was created for. + auto Device = cast(hDevice); // Choose the binary target for the provided device const char *ImageTarget = nullptr; // Get the type of the device cl_device_type DeviceType; constexpr uint32_t InvalidInd = std::numeric_limits::max(); - cl_int RetErr = clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + cl_int RetErr = clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(cl_device_type), &DeviceType, nullptr); if (RetErr != CL_SUCCESS) { *pSelectedBinary = InvalidInd; @@ -1881,3 +1895,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceSelectBinary( UR_APIEXPORT ur_result_t UR_APICALL urDeviceWaitExp(ur_device_handle_t) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/device.hpp b/unified-runtime/source/adapters/opencl/device.hpp index 2bff1555b02c6..1cfd76e8822e9 100644 --- a/unified-runtime/source/adapters/opencl/device.hpp +++ b/unified-runtime/source/adapters/opencl/device.hpp @@ -13,23 +13,25 @@ #include "device.hpp" #include "platform.hpp" -struct ur_device_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_device_handle_t_ : handle_base { using native_type = cl_device_id; native_type CLDevice; - ur_platform_handle_t Platform; + ur_platform_handle_t_ *Platform; cl_device_type Type = 0; - ur_device_handle_t ParentDevice = nullptr; + ur_device_handle_t_ *ParentDevice = nullptr; bool IsNativeHandleOwned = true; ur::RefCount RefCount; ur_device_handle_t_(const ur_device_handle_t_ &) = delete; ur_device_handle_t_ &operator=(const ur_device_handle_t_ &) = delete; - ur_device_handle_t_(native_type Dev, ur_platform_handle_t Plat, - ur_device_handle_t Parent) + ur_device_handle_t_(native_type Dev, ur_platform_handle_t_ *Plat, + ur_device_handle_t_ *Parent) : handle_base(), CLDevice(Dev), Platform(Plat), ParentDevice(Parent) { - if (Parent) { - Type = Parent->Type; + if (ParentDevice) { + Type = ParentDevice->Type; [[maybe_unused]] auto Res = clRetainDevice(CLDevice); assert(Res == CL_SUCCESS); } else { @@ -111,3 +113,5 @@ struct ur_device_handle_t_ : ur::opencl::handle_base { return UR_RESULT_SUCCESS; } }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/enqueue.cpp b/unified-runtime/source/adapters/opencl/enqueue.cpp index 19bbc70c352b9..52468bd761754 100644 --- a/unified-runtime/source/adapters/opencl/enqueue.cpp +++ b/unified-runtime/source/adapters/opencl/enqueue.cpp @@ -38,35 +38,41 @@ cl_map_flags convertURMapFlagsToCL(ur_map_flags_t URFlags) { void MapUREventsToCL(uint32_t numEvents, const ur_event_handle_t *UREvents, std::vector &CLEvents) { for (uint32_t i = 0; i < numEvents; i++) { - CLEvents[i] = UREvents[i]->CLEvent; + CLEvents[i] = ur::opencl::cast(UREvents[i])->CLEvent; } } +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait( ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueMarkerWithWaitList( - hQueue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier( ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueBarrierWithWaitList( - hQueue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -74,22 +80,25 @@ UR_APIEXPORT ur_result_t urEnqueueEventsWaitWithBarrierExt( ur_queue_handle_t hQueue, const ur_exp_enqueue_ext_properties_t *, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - return urEnqueueEventsWaitWithBarrier(hQueue, numEventsInWaitList, - phEventWaitList, phEvent); + return ur::opencl::urEnqueueEventsWaitWithBarrier(hQueue, numEventsInWaitList, + phEventWaitList, phEvent); } UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferRead( ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingRead, size_t offset, size_t size, void *pDst, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueReadBuffer( - hQueue->CLQueue, hBuffer->CLMemory, blockingRead, offset, size, pDst, + Queue->CLQueue, Buffer->CLMemory, blockingRead, offset, size, pDst, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -97,14 +106,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferWrite( ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueWriteBuffer( - hQueue->CLQueue, hBuffer->CLMemory, blockingWrite, offset, size, pSrc, + Queue->CLQueue, Buffer->CLMemory, blockingWrite, offset, size, pSrc, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -115,6 +127,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferReadRect( size_t hostRowPitch, size_t hostSlicePitch, void *pDst, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); const size_t BufferOrigin[3] = {bufferOrigin.x, bufferOrigin.y, bufferOrigin.z}; const size_t HostOrigin[3] = {hostOrigin.x, hostOrigin.y, hostOrigin.z}; @@ -123,12 +137,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferReadRect( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueReadBufferRect( - hQueue->CLQueue, hBuffer->CLMemory, blockingRead, BufferOrigin, - HostOrigin, Region, bufferRowPitch, bufferSlicePitch, hostRowPitch, - hostSlicePitch, pDst, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, Buffer->CLMemory, blockingRead, BufferOrigin, HostOrigin, + Region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, + pDst, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -139,6 +154,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferWriteRect( size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); const size_t BufferOrigin[3] = {bufferOrigin.x, bufferOrigin.y, bufferOrigin.z}; const size_t HostOrigin[3] = {hostOrigin.x, hostOrigin.y, hostOrigin.z}; @@ -147,12 +164,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferWriteRect( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueWriteBufferRect( - hQueue->CLQueue, hBuffer->CLMemory, blockingWrite, BufferOrigin, - HostOrigin, Region, bufferRowPitch, bufferSlicePitch, hostRowPitch, - hostSlicePitch, pSrc, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, Buffer->CLMemory, blockingWrite, BufferOrigin, HostOrigin, + Region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, + pSrc, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -161,15 +179,19 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferCopy( ur_mem_handle_t hBufferDst, size_t srcOffset, size_t dstOffset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto BufferSrc = cast(hBufferSrc); + auto BufferDst = cast(hBufferDst); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueCopyBuffer( - hQueue->CLQueue, hBufferSrc->CLMemory, hBufferDst->CLMemory, srcOffset, + Queue->CLQueue, BufferSrc->CLMemory, BufferDst->CLMemory, srcOffset, dstOffset, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -180,6 +202,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferCopyRect( size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto BufferSrc = cast(hBufferSrc); + auto BufferDst = cast(hBufferDst); const size_t SrcOrigin[3] = {srcOrigin.x, srcOrigin.y, srcOrigin.z}; const size_t DstOrigin[3] = {dstOrigin.x, dstOrigin.y, dstOrigin.z}; const size_t Region[3] = {region.width, region.height, region.depth}; @@ -187,11 +212,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferCopyRect( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueCopyBufferRect( - hQueue->CLQueue, hBufferSrc->CLMemory, hBufferDst->CLMemory, SrcOrigin, + Queue->CLQueue, BufferSrc->CLMemory, BufferDst->CLMemory, SrcOrigin, DstOrigin, Region, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -200,6 +226,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferFill( size_t patternSize, size_t offset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); // CL FillBuffer only allows pattern sizes up to the largest CL type: // long16/double16 if (patternSize <= 128) { @@ -207,11 +235,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferFill( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueFillBuffer( - hQueue->CLQueue, hBuffer->CLMemory, pPattern, patternSize, offset, size, + Queue->CLQueue, Buffer->CLMemory, pPattern, patternSize, offset, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -226,7 +254,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferFill( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); auto ClErr = clEnqueueWriteBuffer( - hQueue->CLQueue, hBuffer->CLMemory, false, offset, size, HostBuffer, + Queue->CLQueue, Buffer->CLMemory, false, offset, size, HostBuffer, numEventsInWaitList, CLWaitEvents.data(), &WriteEvent); if (ClErr != CL_SUCCESS) { delete[] HostBuffer; @@ -249,7 +277,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferFill( if (phEvent) { UR_RETURN_ON_FAILURE( - createUREvent(WriteEvent, hQueue->Context, hQueue, phEvent)); + createUREvent(WriteEvent, cast(Queue->Context), cast(Queue), phEvent)); } else { CL_RETURN_ON_FAILURE(clReleaseEvent(WriteEvent)); } @@ -262,17 +290,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageRead( ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Image = cast(hImage); const size_t Origin[3] = {origin.x, origin.y, origin.z}; const size_t Region[3] = {region.width, region.height, region.depth}; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueReadImage( - hQueue->CLQueue, hImage->CLMemory, blockingRead, Origin, Region, rowPitch, + Queue->CLQueue, Image->CLMemory, blockingRead, Origin, Region, rowPitch, slicePitch, pDst, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -281,16 +312,19 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageWrite( ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Image = cast(hImage); const size_t Origin[3] = {origin.x, origin.y, origin.z}; const size_t Region[3] = {region.width, region.height, region.depth}; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueWriteImage( - hQueue->CLQueue, hImage->CLMemory, blockingWrite, Origin, Region, - rowPitch, slicePitch, pSrc, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, Image->CLMemory, blockingWrite, Origin, Region, rowPitch, + slicePitch, pSrc, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -300,17 +334,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageCopy( ur_rect_offset_t dstOrigin, ur_rect_region_t region, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto ImageSrc = cast(hImageSrc); + auto ImageDst = cast(hImageDst); const size_t SrcOrigin[3] = {srcOrigin.x, srcOrigin.y, srcOrigin.z}; const size_t DstOrigin[3] = {dstOrigin.x, dstOrigin.y, dstOrigin.z}; const size_t Region[3] = {region.width, region.height, region.depth}; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); - CL_RETURN_ON_FAILURE(clEnqueueCopyImage( - hQueue->CLQueue, hImageSrc->CLMemory, hImageDst->CLMemory, SrcOrigin, - DstOrigin, Region, numEventsInWaitList, CLWaitEvents.data(), - ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + CL_RETURN_ON_FAILURE( + clEnqueueCopyImage(Queue->CLQueue, ImageSrc->CLMemory, ImageDst->CLMemory, + SrcOrigin, DstOrigin, Region, numEventsInWaitList, + CLWaitEvents.data(), ifUrEvent(phEvent, Event))); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -319,15 +357,18 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemBufferMap( ur_map_flags_t mapFlags, size_t offset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, void **ppRetMap) { + auto Queue = cast(hQueue); + auto Buffer = cast(hBuffer); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); cl_int Err; - *ppRetMap = clEnqueueMapBuffer( - hQueue->CLQueue, hBuffer->CLMemory, blockingMap, - convertURMapFlagsToCL(mapFlags), offset, size, numEventsInWaitList, - CLWaitEvents.data(), ifUrEvent(phEvent, Event), &Err); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + *ppRetMap = clEnqueueMapBuffer(Queue->CLQueue, Buffer->CLMemory, blockingMap, + convertURMapFlagsToCL(mapFlags), offset, size, + numEventsInWaitList, CLWaitEvents.data(), + ifUrEvent(phEvent, Event), &Err); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return mapCLErrorToUR(Err); } @@ -335,13 +376,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemUnmap( ur_queue_handle_t hQueue, ur_mem_handle_t hMem, void *pMappedPtr, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + auto Mem = cast(hMem); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueUnmapMemObject( - hQueue->CLQueue, hMem->CLMemory, pMappedPtr, numEventsInWaitList, + Queue->CLQueue, Mem->CLMemory, pMappedPtr, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -351,19 +395,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueDeviceGlobalVariableWrite( uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context Ctx = hQueue->Context->CLContext; + auto Queue = cast(hQueue); + auto Program = cast(hProgram); + cl_context Ctx = Queue->Context->CLContext; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); cl_ext::clEnqueueWriteGlobalVariableINTEL_fn F = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - Ctx, ur::cl::getAdapter()->fnCache.clEnqueueWriteGlobalVariableINTELCache, + Ctx, + cast(ur::cl::getAdapter()) + ->fnCache.clEnqueueWriteGlobalVariableINTELCache, cl_ext::EnqueueWriteGlobalVariableName, &F)); - cl_int Res = F(hQueue->CLQueue, hProgram->CLProgram, name, blockingWrite, - count, offset, pSrc, numEventsInWaitList, CLWaitEvents.data(), + cl_int Res = F(Queue->CLQueue, Program->CLProgram, name, blockingWrite, count, + offset, pSrc, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event)); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); return mapCLErrorToUR(Res); } @@ -373,20 +422,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueDeviceGlobalVariableRead( uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context Ctx = hQueue->Context->CLContext; + auto Queue = cast(hQueue); + auto Program = cast(hProgram); + cl_context Ctx = Queue->Context->CLContext; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); cl_ext::clEnqueueReadGlobalVariableINTEL_fn F = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - Ctx, ur::cl::getAdapter()->fnCache.clEnqueueReadGlobalVariableINTELCache, + Ctx, + cast(ur::cl::getAdapter())->fnCache.clEnqueueReadGlobalVariableINTELCache, cl_ext::EnqueueReadGlobalVariableName, &F)); - cl_int Res = F(hQueue->CLQueue, hProgram->CLProgram, name, blockingRead, - count, offset, pDst, numEventsInWaitList, CLWaitEvents.data(), + cl_int Res = F(Queue->CLQueue, Program->CLProgram, name, blockingRead, count, + offset, pDst, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event)); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); return mapCLErrorToUR(Res); } @@ -396,7 +449,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueReadHostPipe( uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context CLContext = hQueue->Context->CLContext; + auto Queue = cast(hQueue); + auto Program = cast(hProgram); + cl_context CLContext = Queue->Context->CLContext; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); @@ -404,16 +459,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueReadHostPipe( UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clEnqueueReadHostPipeINTELCache, + cast(ur::cl::getAdapter())->fnCache.clEnqueueReadHostPipeINTELCache, cl_ext::EnqueueReadHostPipeName, &FuncPtr)); if (FuncPtr) { CL_RETURN_ON_FAILURE(FuncPtr( - hQueue->CLQueue, hProgram->CLProgram, pipe_symbol, blocking, pDst, size, + Queue->CLQueue, Program->CLProgram, pipe_symbol, blocking, pDst, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); } return UR_RESULT_SUCCESS; @@ -425,7 +480,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueWriteHostPipe( uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context CLContext = hQueue->Context->CLContext; + auto Queue = cast(hQueue); + auto Program = cast(hProgram); + cl_context CLContext = Queue->Context->CLContext; cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); @@ -433,15 +490,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueWriteHostPipe( UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clEnqueueWriteHostPipeINTELCache, + cast(ur::cl::getAdapter())->fnCache.clEnqueueWriteHostPipeINTELCache, cl_ext::EnqueueWriteHostPipeName, &FuncPtr)); if (FuncPtr) { CL_RETURN_ON_FAILURE(FuncPtr( - hQueue->CLQueue, hProgram->CLProgram, pipe_symbol, blocking, pSrc, size, + Queue->CLQueue, Program->CLProgram, pipe_symbol, blocking, pSrc, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); } return UR_RESULT_SUCCESS; @@ -485,44 +542,47 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( break; } } + auto Queue = cast(hQueue); + auto Kernel = cast(hKernel); if (hasPointerArgs) { UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - hQueue->Context->CLContext, - ur::cl::getAdapter()->fnCache.clSetKernelArgMemPointerINTELCache, + Queue->Context->CLContext, + cast(ur::cl::getAdapter()) + ->fnCache.clSetKernelArgMemPointerINTELCache, cl_ext::SetKernelArgMemPointerName, &SetKernelArgMemPointerPtr)); } for (uint32_t i = 0; i < numArgs; i++) { switch (pArgs[i].type) { case UR_EXP_KERNEL_ARG_TYPE_LOCAL: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, nullptr)); break; case UR_EXP_KERNEL_ARG_TYPE_VALUE: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, pArgs[i].value.value)); break; case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: { cl_mem mem = pArgs[i].value.memObjTuple.hMem - ? pArgs[i].value.memObjTuple.hMem->CLMemory + ? cast(pArgs[i].value.memObjTuple.hMem)->CLMemory : nullptr; - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, &mem)); break; } case UR_EXP_KERNEL_ARG_TYPE_POINTER: CL_RETURN_ON_FAILURE(SetKernelArgMemPointerPtr( - hKernel->CLKernel, static_cast(pArgs[i].index), + Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].value.pointer)); break; case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { CL_RETURN_ON_FAILURE(clSetKernelArg( - hKernel->CLKernel, static_cast(pArgs[i].index), - pArgs[i].size, &pArgs[i].value.sampler->CLSampler)); + Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, + &cast(pArgs[i].value.sampler)->CLSampler)); break; } default: @@ -534,12 +594,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( if (!pLocalWorkSize) { cl_device_id device = nullptr; CL_RETURN_ON_FAILURE(clGetCommandQueueInfo( - hQueue->CLQueue, CL_QUEUE_DEVICE, sizeof(device), &device, nullptr)); + Queue->CLQueue, CL_QUEUE_DEVICE, sizeof(device), &device, nullptr)); // This query always returns size_t[3], if nothing was specified it // returns all zeroes. size_t queriedLocalWorkSize[3] = {0, 0, 0}; CL_RETURN_ON_FAILURE(clGetKernelWorkGroupInfo( - hKernel->CLKernel, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, + Kernel->CLKernel, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size_t[3]), queriedLocalWorkSize, nullptr)); if (queriedLocalWorkSize[0] != 0) { for (uint32_t i = 0; i < 3; i++) { @@ -552,13 +612,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( std::vector CLWaitEvents(numEventsInWaitList); MapUREventsToCL(numEventsInWaitList, phEventWaitList, CLWaitEvents); CL_RETURN_ON_FAILURE(clEnqueueNDRangeKernel( - hQueue->CLQueue, hKernel->CLKernel, workDim, pGlobalWorkOffset, + Queue->CLQueue, Kernel->CLKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, compiledLocalWorksize.empty() ? pLocalWorkSize : compiledLocalWorksize.data(), numEventsInWaitList, numEventsInWaitList ? CLWaitEvents.data() : nullptr, ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/enqueue_native.cpp b/unified-runtime/source/adapters/opencl/enqueue_native.cpp index ee62b8f48d471..cc4c24ded6ac0 100644 --- a/unified-runtime/source/adapters/opencl/enqueue_native.cpp +++ b/unified-runtime/source/adapters/opencl/enqueue_native.cpp @@ -9,6 +9,8 @@ #include +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urEnqueueNativeCommandExp( ur_queue_handle_t, ur_exp_enqueue_native_command_function_t, void *, uint32_t, const ur_mem_handle_t *, @@ -16,3 +18,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueNativeCommandExp( const ur_event_handle_t *, ur_event_handle_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/event.cpp b/unified-runtime/source/adapters/opencl/event.cpp index 15a16480e3336..762d54009e3d6 100644 --- a/unified-runtime/source/adapters/opencl/event.cpp +++ b/unified-runtime/source/adapters/opencl/event.cpp @@ -110,17 +110,20 @@ ur_command_t convertCLCommandTypeToUR(const cl_command_type &CommandType) { } } +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urEventCreateWithNativeHandle( ur_native_handle_t hNativeEvent, ur_context_handle_t hContext, const ur_event_native_properties_t *pProperties, ur_event_handle_t *phEvent) { cl_event NativeHandle = reinterpret_cast(hNativeEvent); + auto Context = cast(hContext); try { auto UREvent = - std::make_unique(NativeHandle, hContext, nullptr); + std::make_unique(NativeHandle, Context, nullptr); UREvent->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; - *phEvent = UREvent.release(); + *phEvent = cast(UREvent.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -131,24 +134,27 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventCreateWithNativeHandle( UR_APIEXPORT ur_result_t UR_APICALL urEventGetNativeHandle( ur_event_handle_t hEvent, ur_native_handle_t *phNativeEvent) { - return getNativeHandle(hEvent->CLEvent, phNativeEvent); + auto Event = cast(hEvent); + return getNativeHandle(Event->CLEvent, phNativeEvent); } UR_APIEXPORT ur_result_t UR_APICALL urEventRelease(ur_event_handle_t hEvent) { - if (hEvent->RefCount.release()) { - delete hEvent; + auto Event = cast(hEvent); + if (Event->RefCount.release()) { + delete Event; } return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urEventRetain(ur_event_handle_t hEvent) { - hEvent->RefCount.retain(); + auto Event = cast(hEvent); + Event->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urEventWait(uint32_t numEvents, const ur_event_handle_t *phEventWaitList) { - ur_context_handle_t hContext = phEventWaitList[0]->Context; + ur_context_handle_t_ *hContext = cast(phEventWaitList[0])->Context; std::vector CLEvents; CLEvents.reserve(numEvents); @@ -156,13 +162,13 @@ urEventWait(uint32_t numEvents, const ur_event_handle_t *phEventWaitList) { // If the events are from different contexts, we need to wait for each // set of events separately. for (uint32_t i = 0; i < numEvents; i++) { - if (phEventWaitList[i]->Context != hContext) { + if (cast(phEventWaitList[i])->Context != hContext) { CL_RETURN_ON_FAILURE(clWaitForEvents(CLEvents.size(), CLEvents.data())); CLEvents.clear(); } - CLEvents.push_back(phEventWaitList[i]->CLEvent); - hContext = phEventWaitList[i]->Context; + CLEvents.push_back(cast(phEventWaitList[i])->CLEvent); + hContext = cast(phEventWaitList[i])->Context; } if (CLEvents.size()) { CL_RETURN_ON_FAILURE(clWaitForEvents(CLEvents.size(), CLEvents.data())); @@ -175,23 +181,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Event = cast(hEvent); cl_event_info CLEventInfo = convertUREventInfoToCL(propName); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { case UR_EVENT_INFO_CONTEXT: { - return ReturnValue(hEvent->Context); + return ReturnValue(cast(Event->Context)); } case UR_EVENT_INFO_COMMAND_QUEUE: { - hEvent->ensureQueue(); - return ReturnValue(hEvent->Queue); + Event->ensureQueue(); + return ReturnValue(cast(Event->Queue)); } case UR_EVENT_INFO_REFERENCE_COUNT: { - return ReturnValue(hEvent->RefCount.getCount()); + return ReturnValue(Event->RefCount.getCount()); } default: { size_t CheckPropSize = 0; - cl_int RetErr = clGetEventInfo(hEvent->CLEvent, CLEventInfo, propSize, + cl_int RetErr = clGetEventInfo(Event->CLEvent, CLEventInfo, propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize && propName != UR_EVENT_INFO_COMMAND_EXECUTION_STATUS) { @@ -228,8 +235,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent, UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo( ur_event_handle_t hEvent, ur_profiling_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Event = cast(hEvent); cl_profiling_info CLProfilingInfo = convertURProfilingInfoToCL(propName); - cl_int RetErr = clGetEventProfilingInfo(hEvent->CLEvent, CLProfilingInfo, + cl_int RetErr = clGetEventProfilingInfo(Event->CLEvent, CLProfilingInfo, propSize, pPropValue, pPropSizeRet); CL_RETURN_ON_FAILURE(RetErr); return UR_RESULT_SUCCESS; @@ -238,6 +246,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo( UR_APIEXPORT ur_result_t UR_APICALL urEventSetCallback(ur_event_handle_t hEvent, ur_execution_info_t execStatus, ur_event_callback_t pfnNotify, void *pUserData) { + auto Event = cast(hEvent); static std::unordered_map> EventCallbackMap; static std::mutex EventCallbackMutex; @@ -296,7 +305,7 @@ urEventSetCallback(ur_event_handle_t hEvent, ur_execution_info_t execStatus, C->execute(); }; CL_RETURN_ON_FAILURE( - clSetEventCallback(hEvent->CLEvent, CallbackType, ClCallback, Callback)); + clSetEventCallback(Event->CLEvent, CallbackType, ClCallback, Callback)); return UR_RESULT_SUCCESS; } @@ -311,3 +320,20 @@ urEventCreateExp(ur_context_handle_t, ur_device_handle_t, const ur_exp_event_desc_t *, ur_event_handle_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetEventHandleExp(ur_event_handle_t, + void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urIPCPutEventHandleExp(ur_context_handle_t, + void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenEventHandleExp( + ur_context_handle_t, const void *, size_t, ur_event_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/event.hpp b/unified-runtime/source/adapters/opencl/event.hpp index 7160471d77cd9..831f1104c527b 100644 --- a/unified-runtime/source/adapters/opencl/event.hpp +++ b/unified-runtime/source/adapters/opencl/event.hpp @@ -14,27 +14,29 @@ #include -struct ur_event_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_event_handle_t_ : handle_base { using native_type = cl_event; native_type CLEvent; - ur_context_handle_t Context; - ur_queue_handle_t Queue; + ur_context_handle_t_ *Context; + ur_queue_handle_t_ *Queue; bool IsNativeHandleOwned = true; ur::RefCount RefCount; - ur_event_handle_t_(native_type Event, ur_context_handle_t Ctx, - ur_queue_handle_t Queue) + ur_event_handle_t_(native_type Event, ur_context_handle_t_ *Ctx, + ur_queue_handle_t_ *Queue) : handle_base(), CLEvent(Event), Context(Ctx), Queue(Queue) { - urContextRetain(Context); + ur::opencl::urContextRetain(cast(Context)); if (Queue) { - urQueueRetain(Queue); + ur::opencl::urQueueRetain(cast(Queue)); } } ~ur_event_handle_t_() { - urContextRelease(Context); + ur::opencl::urContextRelease(cast(Context)); if (Queue) { - urQueueRelease(Queue); + ur::opencl::urQueueRelease(cast(Queue)); } if (IsNativeHandleOwned) { clReleaseEvent(CLEvent); @@ -47,8 +49,10 @@ struct ur_event_handle_t_ : ur::opencl::handle_base { CL_RETURN_ON_FAILURE(clGetEventInfo(CLEvent, CL_EVENT_COMMAND_QUEUE, sizeof(native_queue), &native_queue, nullptr)); + ur_queue_handle_t OpaqueQueue = nullptr; UR_RETURN_ON_FAILURE(ur_queue_handle_t_::makeWithNative( - native_queue, Context, nullptr, Queue)); + native_queue, cast(Context), nullptr, OpaqueQueue)); + Queue = cast(OpaqueQueue); } return UR_RESULT_SUCCESS; @@ -63,19 +67,22 @@ inline ur_result_t createUREvent(cl_event Event, ur_context_handle_t Context, ur_queue_handle_t Queue, ur_event_handle_t *ReturnedEvent) { assert(Queue); + auto UrQueue = cast(Queue); if (ReturnedEvent) { try { auto UREvent = - std::make_unique(Event, Context, Queue); - *ReturnedEvent = UREvent.release(); - UR_RETURN_ON_FAILURE(Queue->storeLastEvent(*ReturnedEvent)); + std::make_unique(Event, cast(Context), UrQueue); + *ReturnedEvent = cast(UREvent.release()); + UR_RETURN_ON_FAILURE(UrQueue->storeLastEvent(*ReturnedEvent)); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { return UR_RESULT_ERROR_UNKNOWN; } } else { - UR_RETURN_ON_FAILURE(Queue->storeLastEvent(nullptr)); + UR_RETURN_ON_FAILURE(UrQueue->storeLastEvent(nullptr)); } return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/graph.cpp b/unified-runtime/source/adapters/opencl/graph.cpp index 5af875b527841..385c657de5009 100644 --- a/unified-runtime/source/adapters/opencl/graph.cpp +++ b/unified-runtime/source/adapters/opencl/graph.cpp @@ -9,6 +9,8 @@ #include +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urGraphCreateExp( ur_context_handle_t /* hContext */, ur_exp_graph_handle_t * /* phGraph */) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -63,3 +65,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urGraphExecutableGraphGetNativeHandleExp( ur_native_handle_t * /* phNativeExecutableGraph */) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/image.cpp b/unified-runtime/source/adapters/opencl/image.cpp index 13504cbc9c0da..bce3df459dbd7 100644 --- a/unified-runtime/source/adapters/opencl/image.cpp +++ b/unified-runtime/source/adapters/opencl/image.cpp @@ -10,6 +10,8 @@ #include "common.hpp" #include "unified-runtime/ur_api.h" +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urUSMPitchedAllocExp( [[maybe_unused]] ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, @@ -188,6 +190,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesFreeMappedLinearMemoryExp( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } +UR_APIEXPORT ur_result_t UR_APICALL +urBindlessImagesSupportsImportingHandleTypeExp( + [[maybe_unused]] ur_device_handle_t hDevice, + [[maybe_unused]] ur_exp_external_mem_type_t memHandleType, + [[maybe_unused]] ur_bool_t *pSupportedRet) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImportExternalSemaphoreExp( [[maybe_unused]] ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, @@ -224,3 +234,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( [[maybe_unused]] ur_event_handle_t *phEvent) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/kernel.cpp b/unified-runtime/source/adapters/opencl/kernel.cpp index d952178f0da73..5d7e3b394bd35 100644 --- a/unified-runtime/source/adapters/opencl/kernel.cpp +++ b/unified-runtime/source/adapters/opencl/kernel.cpp @@ -19,10 +19,12 @@ #include #include +namespace ur::opencl { + ur_result_t ur_kernel_handle_t_::makeWithNative(native_type NativeKernel, - ur_program_handle_t Program, - ur_context_handle_t Context, - ur_kernel_handle_t &Kernel) { + ur_program_handle_t_ *Program, + ur_context_handle_t_ *Context, + ur_kernel_handle_t_ *&Kernel) { try { cl_context CLContext; CL_RETURN_ON_FAILURE(clGetKernelInfo(NativeKernel, CL_KERNEL_CONTEXT, @@ -43,8 +45,10 @@ ur_result_t ur_kernel_handle_t_::makeWithNative(native_type NativeKernel, } else { ur_native_handle_t hNativeHandle = reinterpret_cast(CLProgram); - UR_RETURN_ON_FAILURE(urProgramCreateWithNativeHandle( - hNativeHandle, Context, nullptr, &Program)); + ur_program_handle_t hProgram = nullptr; + UR_RETURN_ON_FAILURE(ur::opencl::urProgramCreateWithNativeHandle( + hNativeHandle, cast(Context), nullptr, &hProgram)); + Program = cast(hProgram); } auto URKernel = @@ -63,9 +67,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelCreate(ur_program_handle_t hProgram, const char *pKernelName, ur_kernel_handle_t *phKernel) { try { + auto Program = cast(hProgram); cl_int CLResult; cl_kernel Kernel = - clCreateKernel(hProgram->CLProgram, pKernelName, &CLResult); + clCreateKernel(Program->CLProgram, pKernelName, &CLResult); if (CLResult == CL_INVALID_KERNEL_DEFINITION) { cl_adapter::setErrorMessage( @@ -74,9 +79,9 @@ urKernelCreate(ur_program_handle_t hProgram, const char *pKernelName, } CL_RETURN_ON_FAILURE(CLResult); - auto URKernel = std::make_unique(Kernel, hProgram, - hProgram->Context); - *phKernel = URKernel.release(); + auto URKernel = std::make_unique(Kernel, Program, + Program->Context); + *phKernel = cast(URKernel.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -89,9 +94,10 @@ urKernelCreate(ur_program_handle_t hProgram, const char *pKernelName, UR_APIEXPORT ur_result_t UR_APICALL urKernelSetArgLocal(ur_kernel_handle_t hKernel, uint32_t argIndex, size_t argSize, const ur_kernel_arg_local_properties_t *) { + auto Kernel = cast(hKernel); CL_RETURN_ON_FAILURE(clSetKernelArg( - hKernel->CLKernel, static_cast(argIndex), argSize, nullptr)); + Kernel->CLKernel, static_cast(argIndex), argSize, nullptr)); return UR_RESULT_SUCCESS; } @@ -130,23 +136,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetInfo(ur_kernel_handle_t hKernel, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Kernel = cast(hKernel); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { case UR_KERNEL_INFO_PROGRAM: { - return ReturnValue(hKernel->Program); + return ReturnValue(cast(Kernel->Program)); } case UR_KERNEL_INFO_CONTEXT: { - return ReturnValue(hKernel->Context); + return ReturnValue(cast(Kernel->Context)); } case UR_KERNEL_INFO_REFERENCE_COUNT: { - return ReturnValue(hKernel->RefCount.getCount()); + return ReturnValue(Kernel->RefCount.getCount()); } default: { size_t CheckPropSize = 0; cl_int ClResult = - clGetKernelInfo(hKernel->CLKernel, mapURKernelInfoToCL(propName), + clGetKernelInfo(Kernel->CLKernel, mapURKernelInfoToCL(propName), propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -188,6 +195,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, ur_kernel_group_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Kernel = cast(hKernel); + auto Device = cast(hDevice); // From the CL spec for GROUP_INFO_GLOBAL: "If device is not a custom device // and kernel is not a built-in kernel, clGetKernelWorkGroupInfo returns the // error CL_INVALID_VALUE.". Unfortunately there doesn't seem to be a nice @@ -195,7 +204,7 @@ urKernelGetGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, // to deter naive use of the query. if (propName == UR_KERNEL_GROUP_INFO_GLOBAL_WORK_SIZE) { cl_device_type ClDeviceType; - CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_TYPE, + CL_RETURN_ON_FAILURE(clGetDeviceInfo(Device->CLDevice, CL_DEVICE_TYPE, sizeof(ClDeviceType), &ClDeviceType, nullptr)); if (ClDeviceType != CL_DEVICE_TYPE_CUSTOM) { @@ -207,7 +216,7 @@ urKernelGetGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } CL_RETURN_ON_FAILURE(clGetKernelWorkGroupInfo( - hKernel->CLKernel, hDevice->CLDevice, mapURKernelGroupInfoToCL(propName), + Kernel->CLKernel, Device->CLDevice, mapURKernelGroupInfoToCL(propName), propSize, pPropValue, pPropSizeRet)); return UR_RESULT_SUCCESS; @@ -234,6 +243,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, ur_kernel_sub_group_info_t propName, size_t, void *pPropValue, size_t *pPropSizeRet) { + auto Kernel = cast(hKernel); + auto Device = cast(hDevice); std::shared_ptr InputValue; size_t InputValueSize = 0; @@ -244,14 +255,15 @@ urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, // value is given we use the max work item size of the device in the first // dimension to avoid truncation of max sub-group size. uint32_t MaxDims = 0; - ur_result_t URRet = - urDeviceGetInfo(hDevice, UR_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS, - sizeof(uint32_t), &MaxDims, nullptr); + ur_result_t URRet = ur::opencl::urDeviceGetInfo( + hDevice, UR_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS, sizeof(uint32_t), + &MaxDims, nullptr); if (URRet != UR_RESULT_SUCCESS) return URRet; std::shared_ptr WgSizes{new size_t[MaxDims]}; - URRet = urDeviceGetInfo(hDevice, UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES, - MaxDims * sizeof(size_t), WgSizes.get(), nullptr); + URRet = ur::opencl::urDeviceGetInfo( + hDevice, UR_DEVICE_INFO_MAX_WORK_ITEM_SIZES, MaxDims * sizeof(size_t), + WgSizes.get(), nullptr); if (URRet != UR_RESULT_SUCCESS) return URRet; for (size_t i = 1; i < MaxDims; ++i) @@ -265,30 +277,30 @@ urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, cl_ext::clGetKernelSubGroupInfoKHR_fn GetKernelSubGroupInfo = nullptr; oclv::OpenCLVersion DevVer; - CL_RETURN_ON_FAILURE(hDevice->getDeviceVersion(DevVer)); + CL_RETURN_ON_FAILURE(Device->getDeviceVersion(DevVer)); if (DevVer < oclv::V2_1) { bool SubgroupExtSupported = false; - UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions({"cl_khr_subgroups"}, - SubgroupExtSupported)); + UR_RETURN_ON_FAILURE(Device->checkDeviceExtensions({"cl_khr_subgroups"}, + SubgroupExtSupported)); if (!SubgroupExtSupported) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } cl_context Context = nullptr; - CL_RETURN_ON_FAILURE(clGetKernelInfo(hKernel->CLKernel, CL_KERNEL_CONTEXT, + CL_RETURN_ON_FAILURE(clGetKernelInfo(Kernel->CLKernel, CL_KERNEL_CONTEXT, sizeof(Context), &Context, nullptr)); UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - Context, ur::cl::getAdapter()->fnCache.clGetKernelSubGroupInfoKHRCache, + Context, + cast(ur::cl::getAdapter())->fnCache.clGetKernelSubGroupInfoKHRCache, cl_ext::GetKernelSubGroupInfoName, &GetKernelSubGroupInfo)); } else { GetKernelSubGroupInfo = clGetKernelSubGroupInfo; } - cl_int Ret = GetKernelSubGroupInfo(hKernel->CLKernel, hDevice->CLDevice, - mapURKernelSubGroupInfoToCL(propName), - InputValueSize, InputValue.get(), - sizeof(size_t), &RetVal, pPropSizeRet); + cl_int Ret = GetKernelSubGroupInfo( + Kernel->CLKernel, Device->CLDevice, mapURKernelSubGroupInfoToCL(propName), + InputValueSize, InputValue.get(), sizeof(size_t), &RetVal, pPropSizeRet); if (Ret == CL_INVALID_OPERATION) { // clGetKernelSubGroupInfo returns CL_INVALID_OPERATION if the device does @@ -305,14 +317,14 @@ urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, // Two calls to urDeviceGetInfo are needed: the first determines the size // required to store the result, and the second returns the actual size // values. - UR_RETURN_ON_FAILURE(urDeviceGetInfo(hDevice, - UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL, - 0, nullptr, &ResultSize)); + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceGetInfo( + hDevice, UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL, 0, nullptr, + &ResultSize)); assert(ResultSize % sizeof(uint32_t) == 0); std::vector Result(ResultSize / sizeof(uint32_t)); - UR_RETURN_ON_FAILURE(urDeviceGetInfo(hDevice, - UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL, - ResultSize, Result.data(), nullptr)); + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceGetInfo( + hDevice, UR_DEVICE_INFO_SUB_GROUP_SIZES_INTEL, ResultSize, + Result.data(), nullptr)); RetVal = *std::max_element(Result.begin(), Result.end()); Ret = CL_SUCCESS; } else if (propName == UR_KERNEL_SUB_GROUP_INFO_SUB_GROUP_SIZE_INTEL) { @@ -332,14 +344,16 @@ urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, } UR_APIEXPORT ur_result_t UR_APICALL urKernelRetain(ur_kernel_handle_t hKernel) { - hKernel->RefCount.retain(); + auto Kernel = cast(hKernel); + Kernel->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urKernelRelease(ur_kernel_handle_t hKernel) { - if (hKernel->RefCount.release()) { - delete hKernel; + auto Kernel = cast(hKernel); + if (Kernel->RefCount.release()) { + delete Kernel; } return UR_RESULT_SUCCESS; } @@ -349,6 +363,7 @@ urKernelRelease(ur_kernel_handle_t hKernel) { * about every pointer that might be used. */ static ur_result_t usmSetIndirectAccess(ur_kernel_handle_t hKernel) { + auto Kernel = cast(hKernel); cl_bool TrueVal = CL_TRUE; clHostMemAllocINTEL_fn HFunc = nullptr; @@ -358,37 +373,37 @@ static ur_result_t usmSetIndirectAccess(ur_kernel_handle_t hKernel) { /* We test that each alloc type is supported before we actually try to set * KernelExecInfo. */ - CL_RETURN_ON_FAILURE(clGetKernelInfo(hKernel->CLKernel, CL_KERNEL_CONTEXT, + CL_RETURN_ON_FAILURE(clGetKernelInfo(Kernel->CLKernel, CL_KERNEL_CONTEXT, sizeof(cl_context), &CLContext, nullptr)); UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clHostMemAllocINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clHostMemAllocINTELCache, cl_ext::HostMemAllocName, &HFunc)); if (HFunc) { CL_RETURN_ON_FAILURE(clSetKernelExecInfo( - hKernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_HOST_ACCESS_INTEL, + Kernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_HOST_ACCESS_INTEL, sizeof(cl_bool), &TrueVal)); } UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clDeviceMemAllocINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clDeviceMemAllocINTELCache, cl_ext::DeviceMemAllocName, &DFunc)); if (DFunc) { CL_RETURN_ON_FAILURE(clSetKernelExecInfo( - hKernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL, + Kernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL, sizeof(cl_bool), &TrueVal)); } UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clSharedMemAllocINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clSharedMemAllocINTELCache, cl_ext::SharedMemAllocName, &SFunc)); if (SFunc) { CL_RETURN_ON_FAILURE(clSetKernelExecInfo( - hKernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_SHARED_ACCESS_INTEL, + Kernel->CLKernel, CL_KERNEL_EXEC_INFO_INDIRECT_SHARED_ACCESS_INTEL, sizeof(cl_bool), &TrueVal)); } return UR_RESULT_SUCCESS; @@ -397,6 +412,7 @@ static ur_result_t usmSetIndirectAccess(ur_kernel_handle_t hKernel) { UR_APIEXPORT ur_result_t UR_APICALL urKernelSetExecInfo( ur_kernel_handle_t hKernel, ur_kernel_exec_info_t propName, size_t propSize, const ur_kernel_exec_info_properties_t *, const void *pPropValue) { + auto Kernel = cast(hKernel); switch (propName) { case UR_KERNEL_EXEC_INFO_USM_INDIRECT_ACCESS: { @@ -411,7 +427,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelSetExecInfo( return UR_RESULT_SUCCESS; } case UR_KERNEL_EXEC_INFO_USM_PTRS: { - CL_RETURN_ON_FAILURE(clSetKernelExecInfo(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelExecInfo(Kernel->CLKernel, CL_KERNEL_EXEC_INFO_USM_PTRS_INTEL, propSize, pPropValue)); return UR_RESULT_SUCCESS; @@ -424,8 +440,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelSetExecInfo( UR_APIEXPORT ur_result_t UR_APICALL urKernelGetNativeHandle( ur_kernel_handle_t hKernel, ur_native_handle_t *phNativeKernel) { + auto Kernel = cast(hKernel); - *phNativeKernel = reinterpret_cast(hKernel->CLKernel); + *phNativeKernel = reinterpret_cast(Kernel->CLKernel); return UR_RESULT_SUCCESS; } @@ -446,11 +463,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelCreateWithNativeHandle( ur_kernel_handle_t *phKernel) { cl_kernel NativeHandle = reinterpret_cast(hNativeKernel); + ur_kernel_handle_t_ *Kernel = nullptr; UR_RETURN_ON_FAILURE(ur_kernel_handle_t_::makeWithNative( - NativeHandle, hProgram, hContext, *phKernel)); + NativeHandle, cast(hProgram), cast(hContext), Kernel)); - (*phKernel)->IsNativeHandleOwned = + Kernel->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; + *phKernel = cast(Kernel); return UR_RESULT_SUCCESS; } @@ -458,12 +477,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetSuggestedLocalWorkSize( ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t workDim, const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, size_t *pSuggestedLocalWorkSize) { + auto Kernel = cast(hKernel); + auto Queue = cast(hQueue); cl_device_id Device; cl_platform_id Platform; - CL_RETURN_ON_FAILURE(clGetCommandQueueInfo(hQueue->CLQueue, CL_QUEUE_DEVICE, - sizeof(cl_device_id), &Device, - nullptr)); + CL_RETURN_ON_FAILURE(clGetCommandQueueInfo( + Queue->CLQueue, CL_QUEUE_DEVICE, sizeof(cl_device_id), &Device, nullptr)); CL_RETURN_ON_FAILURE(clGetDeviceInfo( Device, CL_DEVICE_PLATFORM, sizeof(cl_platform_id), &Platform, nullptr)); @@ -476,7 +496,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetSuggestedLocalWorkSize( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; CL_RETURN_ON_FAILURE(GetKernelSuggestedLocalWorkSizeFuncPtr( - hQueue->CLQueue, hKernel->CLKernel, workDim, pGlobalWorkOffset, + Queue->CLQueue, Kernel->CLKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pSuggestedLocalWorkSize)); return UR_RESULT_SUCCESS; } @@ -486,55 +506,62 @@ UR_APIEXPORT ur_result_t UR_APICALL urKernelGetSuggestedLocalWorkSizeWithArgs( const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, size_t *pSuggestedLocalWorkSize) { + auto Kernel = cast(hKernel); + auto Queue = cast(hQueue); clSetKernelArgMemPointerINTEL_fn SetKernelArgMemPointerPtr = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - hQueue->Context->CLContext, - ur::cl::getAdapter()->fnCache.clSetKernelArgMemPointerINTELCache, + Queue->Context->CLContext, + cast(ur::cl::getAdapter()) + ->fnCache.clSetKernelArgMemPointerINTELCache, cl_ext::SetKernelArgMemPointerName, &SetKernelArgMemPointerPtr)); for (uint32_t i = 0; i < numArgs; i++) { switch (pArgs[i].type) { case UR_EXP_KERNEL_ARG_TYPE_LOCAL: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, nullptr)); break; case UR_EXP_KERNEL_ARG_TYPE_VALUE: - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, pArgs[i].value.value)); break; case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: { - cl_mem mem = pArgs[i].value.memObjTuple.hMem - ? pArgs[i].value.memObjTuple.hMem->CLMemory - : nullptr; - CL_RETURN_ON_FAILURE(clSetKernelArg(hKernel->CLKernel, + auto Mem = pArgs[i].value.memObjTuple.hMem + ? cast(pArgs[i].value.memObjTuple.hMem) + : nullptr; + cl_mem mem = Mem ? Mem->CLMemory : nullptr; + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].size, &mem)); break; } case UR_EXP_KERNEL_ARG_TYPE_POINTER: CL_RETURN_ON_FAILURE(SetKernelArgMemPointerPtr( - hKernel->CLKernel, static_cast(pArgs[i].index), + Kernel->CLKernel, static_cast(pArgs[i].index), pArgs[i].value.pointer)); break; case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: { - CL_RETURN_ON_FAILURE(clSetKernelArg( - hKernel->CLKernel, static_cast(pArgs[i].index), - pArgs[i].size, &pArgs[i].value.sampler->CLSampler)); + auto Sampler = cast(pArgs[i].value.sampler); + CL_RETURN_ON_FAILURE(clSetKernelArg(Kernel->CLKernel, + static_cast(pArgs[i].index), + pArgs[i].size, &Sampler->CLSampler)); break; } default: return UR_RESULT_ERROR_INVALID_ENUMERATION; } } - return urKernelGetSuggestedLocalWorkSize(hKernel, hQueue, workDim, - pGlobalWorkOffset, pGlobalWorkSize, - pSuggestedLocalWorkSize); + return ur::opencl::urKernelGetSuggestedLocalWorkSize( + hKernel, hQueue, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pSuggestedLocalWorkSize); } UR_APIEXPORT ur_result_t UR_APICALL urKernelSetSpecializationConstants( ur_kernel_handle_t, uint32_t, const ur_specialization_constant_info_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/kernel.hpp b/unified-runtime/source/adapters/opencl/kernel.hpp index 3edd78c45dc4c..a53b4a90c5d5b 100644 --- a/unified-runtime/source/adapters/opencl/kernel.hpp +++ b/unified-runtime/source/adapters/opencl/kernel.hpp @@ -16,11 +16,13 @@ #include -struct ur_kernel_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_kernel_handle_t_ : handle_base { using native_type = cl_kernel; native_type CLKernel; - ur_program_handle_t Program; - ur_context_handle_t Context; + ur_program_handle_t_ *Program; + ur_context_handle_t_ *Context; bool IsNativeHandleOwned = true; clSetKernelArgMemPointerINTEL_fn clSetKernelArgMemPointerINTEL = nullptr; ur::RefCount RefCount; @@ -28,28 +30,30 @@ struct ur_kernel_handle_t_ : ur::opencl::handle_base { ur_kernel_handle_t_(const ur_kernel_handle_t_ &) = delete; ur_kernel_handle_t_ &operator=(const ur_kernel_handle_t_ &) = delete; - ur_kernel_handle_t_(native_type Kernel, ur_program_handle_t Program, - ur_context_handle_t Context) + ur_kernel_handle_t_(native_type Kernel, ur_program_handle_t_ *Program, + ur_context_handle_t_ *Context) : handle_base(), CLKernel(Kernel), Program(Program), Context(Context) { - urProgramRetain(Program); - urContextRetain(Context); + ur::opencl::urProgramRetain(cast(Program)); + ur::opencl::urContextRetain(cast(Context)); cl_ext::getExtFuncFromContext( Context->CLContext, - ur::cl::getAdapter()->fnCache.clSetKernelArgMemPointerINTELCache, + cast(ur::cl::getAdapter())->fnCache.clSetKernelArgMemPointerINTELCache, cl_ext::SetKernelArgMemPointerName, &clSetKernelArgMemPointerINTEL); } ~ur_kernel_handle_t_() { - urProgramRelease(Program); - urContextRelease(Context); + ur::opencl::urProgramRelease(cast(Program)); + ur::opencl::urContextRelease(cast(Context)); if (IsNativeHandleOwned) { clReleaseKernel(CLKernel); } } static ur_result_t makeWithNative(native_type NativeKernel, - ur_program_handle_t Program, - ur_context_handle_t Context, - ur_kernel_handle_t &Kernel); + ur_program_handle_t_ *Program, + ur_context_handle_t_ *Context, + ur_kernel_handle_t_ *&Kernel); }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/memory.cpp b/unified-runtime/source/adapters/opencl/memory.cpp index 1ba798bc1faa7..60aaa25496d05 100644 --- a/unified-runtime/source/adapters/opencl/memory.cpp +++ b/unified-runtime/source/adapters/opencl/memory.cpp @@ -310,6 +310,8 @@ cl_map_flags convertURMemFlagsToCL(ur_mem_flags_t URFlags) { return CLFlags; } +namespace ur::opencl { + ur_result_t ur_mem_handle_t_::makeWithNative(native_type NativeMem, ur_context_handle_t Ctx, ur_mem_handle_t &Mem) { @@ -321,11 +323,12 @@ ur_result_t ur_mem_handle_t_::makeWithNative(native_type NativeMem, CL_RETURN_ON_FAILURE(clGetMemObjectInfo( NativeMem, CL_MEM_CONTEXT, sizeof(CLContext), &CLContext, nullptr)); - if (Ctx->CLContext != CLContext) { + auto Context = cast(Ctx); + if (Context->CLContext != CLContext) { return UR_RESULT_ERROR_INVALID_CONTEXT; } - auto URMem = std::make_unique(NativeMem, Ctx); - Mem = URMem.release(); + auto URMem = std::make_unique(NativeMem, Context); + Mem = cast(URMem.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -338,17 +341,18 @@ ur_result_t ur_mem_handle_t_::makeWithNative(native_type NativeMem, UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreate( ur_context_handle_t hContext, ur_mem_flags_t flags, size_t size, const ur_buffer_properties_t *pProperties, ur_mem_handle_t *phBuffer) { + auto Context = cast(hContext); cl_int RetErr = CL_INVALID_OPERATION; if (pProperties) { // TODO: need to check if all properties are supported by OpenCL RT and // ignore unsupported clCreateBufferWithPropertiesINTEL_fn FuncPtr = nullptr; - cl_context CLContext = hContext->CLContext; + cl_context CLContext = Context->CLContext; // First we need to look up the function pointer RetErr = cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter() + cast(ur::cl::getAdapter()) ->fnCache.clCreateBufferWithPropertiesINTELCache, cl_ext::CreateBufferWithPropertiesName, &FuncPtr); if (FuncPtr) { @@ -380,8 +384,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreate( CLContext, PropertiesIntel.data(), static_cast(flags), size, pProperties->pHost, static_cast(&RetErr)); CL_RETURN_ON_FAILURE(RetErr); - auto URMem = std::make_unique(Buffer, hContext); - *phBuffer = URMem.release(); + auto URMem = std::make_unique(Buffer, Context); + *phBuffer = cast(URMem.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -394,11 +398,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreate( void *HostPtr = pProperties ? pProperties->pHost : nullptr; try { cl_mem Buffer = - clCreateBuffer(hContext->CLContext, static_cast(flags), + clCreateBuffer(Context->CLContext, static_cast(flags), size, HostPtr, static_cast(&RetErr)); CL_RETURN_ON_FAILURE(RetErr); - auto URMem = std::make_unique(Buffer, hContext); - *phBuffer = URMem.release(); + auto URMem = std::make_unique(Buffer, Context); + *phBuffer = cast(URMem.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -413,6 +417,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreate( const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, void *pHost, ur_mem_handle_t *phMem) { + auto Context = cast(hContext); cl_int RetErr = CL_INVALID_OPERATION; cl_image_format ImageFormat = mapURImageFormatToCL(pImageFormat); @@ -421,11 +426,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreate( try { cl_mem Mem = - clCreateImage(hContext->CLContext, MapFlags, &ImageFormat, &ImageDesc, + clCreateImage(Context->CLContext, MapFlags, &ImageFormat, &ImageDesc, pHost, static_cast(&RetErr)); CL_RETURN_ON_FAILURE(RetErr); - auto URMem = std::make_unique(Mem, hContext); - *phMem = URMem.release(); + auto URMem = std::make_unique(Mem, Context); + *phMem = cast(URMem.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -440,6 +445,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferPartition( ur_buffer_create_type_t bufferCreateType, const ur_buffer_region_t *pRegion, ur_mem_handle_t *phMem) { + auto Buffer = cast(hBuffer); cl_int RetErr = CL_INVALID_OPERATION; cl_buffer_create_type BufferCreateType; @@ -455,20 +461,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferPartition( BufferRegion.origin = pRegion->origin; BufferRegion.size = pRegion->size; try { - cl_mem Buffer = clCreateSubBuffer( - hBuffer->CLMemory, static_cast(flags), BufferCreateType, + cl_mem NewBuffer = clCreateSubBuffer( + Buffer->CLMemory, static_cast(flags), BufferCreateType, &BufferRegion, static_cast(&RetErr)); if (RetErr == CL_INVALID_VALUE) { size_t BufferSize = 0; - CL_RETURN_ON_FAILURE(clGetMemObjectInfo(hBuffer->CLMemory, CL_MEM_SIZE, + CL_RETURN_ON_FAILURE(clGetMemObjectInfo(Buffer->CLMemory, CL_MEM_SIZE, sizeof(BufferSize), &BufferSize, nullptr)); if (BufferRegion.size + BufferRegion.origin > BufferSize) return UR_RESULT_ERROR_INVALID_BUFFER_SIZE; } CL_RETURN_ON_FAILURE(RetErr); - auto URMem = std::make_unique(Buffer, hBuffer->Context); - *phMem = URMem.release(); + auto URMem = std::make_unique(NewBuffer, Buffer->Context); + *phMem = cast(URMem.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -479,7 +485,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferPartition( UR_APIEXPORT ur_result_t UR_APICALL urMemGetNativeHandle( ur_mem_handle_t hMem, ur_device_handle_t, ur_native_handle_t *phNativeMem) { - return getNativeHandle(hMem->CLMemory, phNativeMem); + auto Mem = cast(hMem); + return getNativeHandle(Mem->CLMemory, phNativeMem); } UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreateWithNativeHandle( @@ -488,7 +495,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemBufferCreateWithNativeHandle( cl_mem NativeHandle = reinterpret_cast(hNativeMem); UR_RETURN_ON_FAILURE( ur_mem_handle_t_::makeWithNative(NativeHandle, hContext, *phMem)); - (*phMem)->IsNativeHandleOwned = + auto Mem = cast(*phMem); + Mem->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; return UR_RESULT_SUCCESS; } @@ -501,7 +509,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreateWithNativeHandle( cl_mem NativeHandle = reinterpret_cast(hNativeMem); UR_RETURN_ON_FAILURE( ur_mem_handle_t_::makeWithNative(NativeHandle, hContext, *phMem)); - (*phMem)->IsNativeHandleOwned = + auto Mem = cast(*phMem); + Mem->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; return UR_RESULT_SUCCESS; } @@ -512,19 +521,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemGetInfo(ur_mem_handle_t hMemory, void *pPropValue, size_t *pPropSizeRet) { + auto Memory = cast(hMemory); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); const cl_int CLPropName = mapURMemInfoToCL(propName); switch (static_cast(propName)) { case UR_MEM_INFO_CONTEXT: { - return ReturnValue(hMemory->Context); + return ReturnValue(cast(Memory->Context)); } case UR_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hMemory->RefCount.getCount()); + return ReturnValue(Memory->RefCount.getCount()); } default: { size_t CheckPropSize = 0; - auto ClResult = clGetMemObjectInfo(hMemory->CLMemory, CLPropName, propSize, + auto ClResult = clGetMemObjectInfo(Memory->CLMemory, CLPropName, propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -545,11 +555,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageGetInfo(ur_mem_handle_t hMemory, void *pPropValue, size_t *pPropSizeRet) { + auto Memory = cast(hMemory); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); const cl_int CLPropName = mapURMemImageInfoToCL(propName); size_t CheckPropSize = 0; - auto ClResult = clGetImageInfo(hMemory->CLMemory, CLPropName, propSize, + auto ClResult = clGetImageInfo(Memory->CLMemory, CLPropName, propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -568,13 +579,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageGetInfo(ur_mem_handle_t hMemory, } UR_APIEXPORT ur_result_t UR_APICALL urMemRetain(ur_mem_handle_t hMem) { - hMem->RefCount.retain(); + auto Mem = cast(hMem); + Mem->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urMemRelease(ur_mem_handle_t hMem) { - if (hMem->RefCount.release()) { - delete hMem; + auto Mem = cast(hMem); + if (Mem->RefCount.release()) { + delete Mem; } return UR_RESULT_SUCCESS; } @@ -622,3 +635,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/memory.hpp b/unified-runtime/source/adapters/opencl/memory.hpp index 58255eda66f3a..2e4f37df6e0be 100644 --- a/unified-runtime/source/adapters/opencl/memory.hpp +++ b/unified-runtime/source/adapters/opencl/memory.hpp @@ -14,23 +14,25 @@ #include -struct ur_mem_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_mem_handle_t_ : handle_base { using native_type = cl_mem; native_type CLMemory; - ur_context_handle_t Context; + ur_context_handle_t_ *Context; bool IsNativeHandleOwned = true; ur::RefCount RefCount; ur_mem_handle_t_(const ur_mem_handle_t_ &) = delete; ur_mem_handle_t_ &operator=(const ur_mem_handle_t_ &) = delete; - ur_mem_handle_t_(native_type Mem, ur_context_handle_t Ctx) + ur_mem_handle_t_(native_type Mem, ur_context_handle_t_ *Ctx) : handle_base(), CLMemory(Mem), Context(Ctx) { - urContextRetain(Context); + ur::opencl::urContextRetain(cast(Context)); } ~ur_mem_handle_t_() { - urContextRelease(Context); + ur::opencl::urContextRelease(cast(Context)); if (IsNativeHandleOwned) { clReleaseMemObject(CLMemory); } @@ -40,3 +42,5 @@ struct ur_mem_handle_t_ : ur::opencl::handle_base { ur_context_handle_t Ctx, ur_mem_handle_t &Mem); }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/memory_export.cpp b/unified-runtime/source/adapters/opencl/memory_export.cpp index cc937f6d2fd27..1ef5c6d6bffd5 100644 --- a/unified-runtime/source/adapters/opencl/memory_export.cpp +++ b/unified-runtime/source/adapters/opencl/memory_export.cpp @@ -10,6 +10,8 @@ #include "common/logger/ur_logger.hpp" #include "unified-runtime/ur_api.h" +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, size_t /*aligment*/, size_t /*size*/, @@ -32,3 +34,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.cpp b/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.cpp new file mode 100644 index 0000000000000..3720c8e3c40aa --- /dev/null +++ b/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.cpp @@ -0,0 +1,145 @@ +//===------- ocl_dynamic_lib.cpp - OpenCL Dynamic Loading -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-----------------------------------------------------------------===// + +// This translation unit is only compiled when UR_STATIC_ADAPTER_OPENCL is set +// (see opencl/CMakeLists.txt). Define OCL_DYNAMIC_LIB_IMPL before including the +// header to suppress the symbol-redirect macros for our own definitions. +#define OCL_DYNAMIC_LIB_IMPL +#include "ocl_dynamic_lib.hpp" + +#include "logger/ur_logger.hpp" + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include + +namespace ocl { + +// Define storage for all function pointers using X-macros +#define OCL_FUNC(name) decltype(::name) *name##_ptr = nullptr; +#define OCL_OPTIONAL_FUNC(name) OCL_FUNC(name) +#include "ocl_functions.def" +#undef OCL_OPTIONAL_FUNC +#undef OCL_FUNC + +static void *OCLLibHandle = nullptr; +static std::mutex OCLLoadMutex; +static bool OCLLoaded = false; + +template +static bool getSymbolAddr(void *handle, const char *name, T *funcPtr) { +#ifdef _WIN32 + *funcPtr = reinterpret_cast(GetProcAddress((HMODULE)handle, name)); +#else + *funcPtr = reinterpret_cast(dlsym(handle, name)); +#endif + return *funcPtr != nullptr; +} + +static void loadOCLLibraryImpl() { +#ifdef _WIN32 + OCLLibHandle = + LoadLibraryExA("OpenCL.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + if (!OCLLibHandle) { + DWORD error = GetLastError(); + UR_LOG(ERR, + "Failed to load OpenCL.dll from system directory (error code: {})", + error); + return; + } + UR_LOG(DEBUG, "Successfully loaded OpenCL.dll"); +#else + OCLLibHandle = dlopen("libOpenCL.so.1", RTLD_NOW | RTLD_LOCAL); + if (!OCLLibHandle) { + const char *error1 = dlerror(); + UR_LOG(DEBUG, "Failed to load libOpenCL.so.1: {}", + error1 ? error1 : "unknown error"); + + OCLLibHandle = dlopen("libOpenCL.so", RTLD_NOW | RTLD_LOCAL); + if (!OCLLibHandle) { + const char *error2 = dlerror(); + UR_LOG(ERR, + "Failed to load OpenCL library. Tried libOpenCL.so.1 and " + "libOpenCL.so: {}", + error2 ? error2 : "unknown error"); + return; + } + UR_LOG(DEBUG, "Successfully loaded libOpenCL.so"); + } else { + UR_LOG(DEBUG, "Successfully loaded libOpenCL.so.1"); + } +#endif + + bool success = true; + int missing = 0; + +#define OCL_FUNC(name) \ + do { \ + if (!getSymbolAddr(OCLLibHandle, #name, &name##_ptr)) { \ + UR_LOG(ERR, "Required OpenCL function not found: {}", #name); \ + missing++; \ + success = false; \ + } \ + } while (0); +#define OCL_OPTIONAL_FUNC(name) \ + (void)getSymbolAddr(OCLLibHandle, #name, &name##_ptr); + +#include "ocl_functions.def" +#undef OCL_OPTIONAL_FUNC +#undef OCL_FUNC + + if (!success) { + UR_LOG(ERR, "Failed to load {} required OpenCL function(s)", missing); + // Required symbols missing — close the handle we opened to avoid a leak. +#ifdef _WIN32 + FreeLibrary((HMODULE)OCLLibHandle); +#else + dlclose(OCLLibHandle); +#endif + OCLLibHandle = nullptr; +#define OCL_FUNC(name) name##_ptr = nullptr; +#define OCL_OPTIONAL_FUNC(name) OCL_FUNC(name) +#include "ocl_functions.def" +#undef OCL_OPTIONAL_FUNC +#undef OCL_FUNC + } +} + +bool loadOCLLibrary() { + std::lock_guard Lock{OCLLoadMutex}; + if (!OCLLoaded) { + loadOCLLibraryImpl(); + OCLLoaded = true; + } + return OCLLibHandle != nullptr; +} + +void unloadOCLLibrary() { + std::lock_guard Lock{OCLLoadMutex}; + if (OCLLibHandle) { +#ifdef _WIN32 + FreeLibrary((HMODULE)OCLLibHandle); +#else + dlclose(OCLLibHandle); +#endif + OCLLibHandle = nullptr; + +#define OCL_FUNC(name) name##_ptr = nullptr; +#define OCL_OPTIONAL_FUNC(name) OCL_FUNC(name) +#include "ocl_functions.def" +#undef OCL_OPTIONAL_FUNC +#undef OCL_FUNC + } + OCLLoaded = false; +} + +} // namespace ocl diff --git a/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.hpp b/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.hpp new file mode 100644 index 0000000000000..a89c6270c990c --- /dev/null +++ b/unified-runtime/source/adapters/opencl/ocl_dynamic_lib.hpp @@ -0,0 +1,108 @@ +//===------- ocl_dynamic_lib.hpp - OpenCL Dynamic Loading ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-----------------------------------------------------------------===// +#pragma once + +#ifdef UR_STATIC_ADAPTER_OPENCL + +// Include OpenCL headers BEFORE any redirection +#include +#include + +namespace ocl { + +// Declare function pointers for all OpenCL functions using X-macros +#define OCL_FUNC(name) extern decltype(::name) *name##_ptr; +#define OCL_OPTIONAL_FUNC(name) OCL_FUNC(name) +#include "ocl_functions.def" +#undef OCL_OPTIONAL_FUNC +#undef OCL_FUNC + +bool loadOCLLibrary(); +void unloadOCLLibrary(); + +} // namespace ocl + +// Only define the redirection macros if we're NOT in the implementation file +// The implementation file needs the original function names for decltype +#ifndef OCL_DYNAMIC_LIB_IMPL + +// Redirect all OpenCL function calls to our dynamically loaded pointers +#define clBuildProgram ocl::clBuildProgram_ptr +#define clCompileProgram ocl::clCompileProgram_ptr +#define clCreateBuffer ocl::clCreateBuffer_ptr +#define clCreateCommandQueue ocl::clCreateCommandQueue_ptr +#define clCreateCommandQueueWithProperties \ + ocl::clCreateCommandQueueWithProperties_ptr +#define clCreateContext ocl::clCreateContext_ptr +#define clCreateImage ocl::clCreateImage_ptr +#define clCreateKernel ocl::clCreateKernel_ptr +#define clCreateProgramWithBinary ocl::clCreateProgramWithBinary_ptr +#define clCreateProgramWithIL ocl::clCreateProgramWithIL_ptr +#define clCreateSampler ocl::clCreateSampler_ptr +#define clCreateSubBuffer ocl::clCreateSubBuffer_ptr +#define clCreateSubDevices ocl::clCreateSubDevices_ptr +#define clEnqueueBarrierWithWaitList ocl::clEnqueueBarrierWithWaitList_ptr +#define clEnqueueCopyBuffer ocl::clEnqueueCopyBuffer_ptr +#define clEnqueueCopyBufferRect ocl::clEnqueueCopyBufferRect_ptr +#define clEnqueueCopyImage ocl::clEnqueueCopyImage_ptr +#define clEnqueueFillBuffer ocl::clEnqueueFillBuffer_ptr +#define clEnqueueMapBuffer ocl::clEnqueueMapBuffer_ptr +#define clEnqueueMarkerWithWaitList ocl::clEnqueueMarkerWithWaitList_ptr +#define clEnqueueNDRangeKernel ocl::clEnqueueNDRangeKernel_ptr +#define clEnqueueReadBuffer ocl::clEnqueueReadBuffer_ptr +#define clEnqueueReadBufferRect ocl::clEnqueueReadBufferRect_ptr +#define clEnqueueReadImage ocl::clEnqueueReadImage_ptr +#define clEnqueueUnmapMemObject ocl::clEnqueueUnmapMemObject_ptr +#define clEnqueueWriteBuffer ocl::clEnqueueWriteBuffer_ptr +#define clEnqueueWriteBufferRect ocl::clEnqueueWriteBufferRect_ptr +#define clEnqueueWriteImage ocl::clEnqueueWriteImage_ptr +#define clFinish ocl::clFinish_ptr +#define clFlush ocl::clFlush_ptr +#define clGetCommandQueueInfo ocl::clGetCommandQueueInfo_ptr +#define clGetContextInfo ocl::clGetContextInfo_ptr +#define clGetDeviceAndHostTimer ocl::clGetDeviceAndHostTimer_ptr +#define clGetDeviceIDs ocl::clGetDeviceIDs_ptr +#define clGetDeviceInfo ocl::clGetDeviceInfo_ptr +#define clGetEventInfo ocl::clGetEventInfo_ptr +#define clGetEventProfilingInfo ocl::clGetEventProfilingInfo_ptr +#define clGetExtensionFunctionAddressForPlatform \ + ocl::clGetExtensionFunctionAddressForPlatform_ptr +#define clGetHostTimer ocl::clGetHostTimer_ptr +#define clGetImageInfo ocl::clGetImageInfo_ptr +#define clGetKernelInfo ocl::clGetKernelInfo_ptr +#define clGetKernelSubGroupInfo ocl::clGetKernelSubGroupInfo_ptr +#define clGetKernelWorkGroupInfo ocl::clGetKernelWorkGroupInfo_ptr +#define clGetMemObjectInfo ocl::clGetMemObjectInfo_ptr +#define clGetPlatformIDs ocl::clGetPlatformIDs_ptr +#define clGetPlatformInfo ocl::clGetPlatformInfo_ptr +#define clGetProgramBuildInfo ocl::clGetProgramBuildInfo_ptr +#define clGetProgramInfo ocl::clGetProgramInfo_ptr +#define clGetSamplerInfo ocl::clGetSamplerInfo_ptr +#define clLinkProgram ocl::clLinkProgram_ptr +#define clReleaseCommandQueue ocl::clReleaseCommandQueue_ptr +#define clReleaseContext ocl::clReleaseContext_ptr +#define clReleaseDevice ocl::clReleaseDevice_ptr +#define clReleaseEvent ocl::clReleaseEvent_ptr +#define clReleaseKernel ocl::clReleaseKernel_ptr +#define clReleaseMemObject ocl::clReleaseMemObject_ptr +#define clReleaseProgram ocl::clReleaseProgram_ptr +#define clReleaseSampler ocl::clReleaseSampler_ptr +#define clRetainContext ocl::clRetainContext_ptr +#define clRetainDevice ocl::clRetainDevice_ptr +#define clRetainEvent ocl::clRetainEvent_ptr +#define clSetEventCallback ocl::clSetEventCallback_ptr +#define clSetKernelArg ocl::clSetKernelArg_ptr +#define clSetKernelExecInfo ocl::clSetKernelExecInfo_ptr +#define clWaitForEvents ocl::clWaitForEvents_ptr +// clSetProgramSpecializationConstant and clSetContextDestructorCallback are +// not redirected here; they are accessed via per-adapter struct fields +// (clSetProgramSpecializationConstantFn / clSetContextDestructorCallbackFn). + +#endif // OCL_DYNAMIC_LIB_IMPL + +#endif // UR_STATIC_ADAPTER_OPENCL diff --git a/unified-runtime/source/adapters/opencl/ocl_functions.def b/unified-runtime/source/adapters/opencl/ocl_functions.def new file mode 100644 index 0000000000000..0cb23f2e01481 --- /dev/null +++ b/unified-runtime/source/adapters/opencl/ocl_functions.def @@ -0,0 +1,74 @@ +// OpenCL function definitions for dynamic loading. +// Format: OCL_FUNC(name) for symbols whose absence aborts adapter init, +// OCL_OPTIONAL_FUNC(name) for symbols treated as optional capabilities +// (call sites null-check the pointer; matches the dynamic-link adapter). + +OCL_FUNC(clBuildProgram) +OCL_FUNC(clCompileProgram) +OCL_FUNC(clCreateBuffer) +OCL_FUNC(clCreateCommandQueue) +OCL_FUNC(clCreateCommandQueueWithProperties) +OCL_FUNC(clCreateContext) +OCL_FUNC(clCreateImage) +OCL_FUNC(clCreateKernel) +OCL_FUNC(clCreateProgramWithBinary) +OCL_FUNC(clCreateProgramWithIL) +OCL_FUNC(clCreateSampler) +OCL_FUNC(clCreateSubBuffer) +OCL_FUNC(clCreateSubDevices) +OCL_FUNC(clEnqueueBarrierWithWaitList) +OCL_FUNC(clEnqueueCopyBuffer) +OCL_FUNC(clEnqueueCopyBufferRect) +OCL_FUNC(clEnqueueCopyImage) +OCL_FUNC(clEnqueueFillBuffer) +OCL_FUNC(clEnqueueMapBuffer) +OCL_FUNC(clEnqueueMarkerWithWaitList) +OCL_FUNC(clEnqueueNDRangeKernel) +OCL_FUNC(clEnqueueReadBuffer) +OCL_FUNC(clEnqueueReadBufferRect) +OCL_FUNC(clEnqueueReadImage) +OCL_FUNC(clEnqueueUnmapMemObject) +OCL_FUNC(clEnqueueWriteBuffer) +OCL_FUNC(clEnqueueWriteBufferRect) +OCL_FUNC(clEnqueueWriteImage) +OCL_FUNC(clFinish) +OCL_FUNC(clFlush) +OCL_FUNC(clGetCommandQueueInfo) +OCL_FUNC(clGetContextInfo) +OCL_FUNC(clGetDeviceAndHostTimer) +OCL_FUNC(clGetDeviceIDs) +OCL_FUNC(clGetDeviceInfo) +OCL_FUNC(clGetEventInfo) +OCL_FUNC(clGetEventProfilingInfo) +OCL_FUNC(clGetExtensionFunctionAddressForPlatform) +OCL_FUNC(clGetHostTimer) +OCL_FUNC(clGetImageInfo) +OCL_FUNC(clGetKernelInfo) +OCL_FUNC(clGetKernelSubGroupInfo) +OCL_FUNC(clGetKernelWorkGroupInfo) +OCL_FUNC(clGetMemObjectInfo) +OCL_FUNC(clGetPlatformIDs) +OCL_FUNC(clGetPlatformInfo) +OCL_FUNC(clGetProgramBuildInfo) +OCL_FUNC(clGetProgramInfo) +OCL_FUNC(clGetSamplerInfo) +OCL_FUNC(clLinkProgram) +OCL_FUNC(clReleaseCommandQueue) +OCL_FUNC(clReleaseContext) +OCL_FUNC(clReleaseDevice) +OCL_FUNC(clReleaseEvent) +OCL_FUNC(clReleaseKernel) +OCL_FUNC(clReleaseMemObject) +OCL_FUNC(clReleaseProgram) +OCL_FUNC(clReleaseSampler) +OCL_FUNC(clRetainContext) +OCL_FUNC(clRetainDevice) +OCL_FUNC(clRetainEvent) +OCL_FUNC(clSetEventCallback) +OCL_FUNC(clSetKernelArg) +OCL_FUNC(clSetKernelExecInfo) +OCL_FUNC(clWaitForEvents) + +// OpenCL 2.2 / 3.0 entry points — call sites already gate on a null pointer. +OCL_OPTIONAL_FUNC(clSetContextDestructorCallback) +OCL_OPTIONAL_FUNC(clSetProgramSpecializationConstant) diff --git a/unified-runtime/source/adapters/opencl/physical_mem.cpp b/unified-runtime/source/adapters/opencl/physical_mem.cpp index 88ca5ecb32b44..3e91752660397 100644 --- a/unified-runtime/source/adapters/opencl/physical_mem.cpp +++ b/unified-runtime/source/adapters/opencl/physical_mem.cpp @@ -11,6 +11,8 @@ #include "common.hpp" #include "context.hpp" +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( ur_context_handle_t, ur_device_handle_t, size_t, const ur_physical_mem_properties_t *, ur_physical_mem_handle_t *) { @@ -32,3 +34,5 @@ urPhysicalMemGetInfo(ur_physical_mem_handle_t, ur_physical_mem_info_t, size_t, void *, size_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/physical_mem.hpp b/unified-runtime/source/adapters/opencl/physical_mem.hpp index 3d13d7183a337..3d2229b40c767 100644 --- a/unified-runtime/source/adapters/opencl/physical_mem.hpp +++ b/unified-runtime/source/adapters/opencl/physical_mem.hpp @@ -10,8 +10,12 @@ #include "common.hpp" +namespace ur::opencl { + /// UR queue mapping on physical memory allocations used in virtual memory /// management. /// TODO: Implement. /// -struct ur_physical_mem_handle_t_ : ur::opencl::handle_base {}; +struct ur_physical_mem_handle_t_ : handle_base {}; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/platform.cpp b/unified-runtime/source/adapters/opencl/platform.cpp index aeef7eb2529b6..fcfcef2b564a5 100644 --- a/unified-runtime/source/adapters/opencl/platform.cpp +++ b/unified-runtime/source/adapters/opencl/platform.cpp @@ -73,6 +73,8 @@ static bool isBannedOpenCLDevice(cl_device_id device) { return isBanned; } +namespace ur::opencl { + UR_DLLEXPORT ur_result_t UR_APICALL urPlatformGetInfo(ur_platform_handle_t hPlatform, ur_platform_info_t propName, size_t propSize, void *pPropValue, size_t *pSizeRet) { @@ -90,7 +92,8 @@ urPlatformGetInfo(ur_platform_handle_t hPlatform, ur_platform_info_t propName, case UR_PLATFORM_INFO_VERSION: case UR_PLATFORM_INFO_EXTENSIONS: case UR_PLATFORM_INFO_PROFILE: { - cl_platform_id Plat = hPlatform->CLPlatform; + auto Platform = cast(hPlatform); + cl_platform_id Plat = Platform->CLPlatform; CL_RETURN_ON_FAILURE( clGetPlatformInfo(Plat, CLPropName, propSize, pPropValue, pSizeRet)); @@ -113,15 +116,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformGet(ur_adapter_handle_t, uint32_t NumEntries, ur_platform_handle_t *phPlatforms, uint32_t *pNumPlatforms) { static std::mutex adapterPopulationMutex{}; - ur_adapter_handle_t Adapter = nullptr; - UR_RETURN_ON_FAILURE(urAdapterGet(1, &Adapter, nullptr)); - if (!Adapter) { + ur_adapter_handle_t AdapterHandle = nullptr; + UR_RETURN_ON_FAILURE(ur::opencl::urAdapterGet(1, &AdapterHandle, nullptr)); + if (!AdapterHandle) { // The only operation urAdapterGet really performs is allocating the adapter // handle via new, so no adapter handle here almost certainly means memory // problems. return UR_RESULT_ERROR_OUT_OF_RESOURCES; } + auto Adapter = cast(AdapterHandle); if (Adapter->NumPlatforms == 0) { std::lock_guard guard{adapterPopulationMutex}; @@ -178,7 +182,7 @@ urPlatformGet(ur_adapter_handle_t, uint32_t NumEntries, } if (NumEntries && phPlatforms) { for (uint32_t i = 0; i < NumEntries; i++) { - phPlatforms[i] = Adapter->URPlatforms[i].get(); + phPlatforms[i] = cast(Adapter->URPlatforms[i].get()); } } @@ -187,8 +191,9 @@ urPlatformGet(ur_adapter_handle_t, uint32_t NumEntries, UR_APIEXPORT ur_result_t UR_APICALL urPlatformGetNativeHandle( ur_platform_handle_t hPlatform, ur_native_handle_t *phNativePlatform) { + auto Platform = cast(hPlatform); *phNativePlatform = - reinterpret_cast(hPlatform->CLPlatform); + reinterpret_cast(Platform->CLPlatform); return UR_RESULT_SUCCESS; } @@ -199,14 +204,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformCreateWithNativeHandle( reinterpret_cast(hNativePlatform); uint32_t NumPlatforms = 0; - UR_RETURN_ON_FAILURE(urPlatformGet(nullptr, 0, nullptr, &NumPlatforms)); - std::vector Platforms(NumPlatforms); UR_RETURN_ON_FAILURE( - urPlatformGet(nullptr, NumPlatforms, Platforms.data(), nullptr)); + ur::opencl::urPlatformGet(nullptr, 0, nullptr, &NumPlatforms)); + std::vector Platforms(NumPlatforms); + UR_RETURN_ON_FAILURE(ur::opencl::urPlatformGet(nullptr, NumPlatforms, + Platforms.data(), nullptr)); for (uint32_t i = 0; i < NumPlatforms; i++) { - if (Platforms[i]->CLPlatform == NativeHandle) { - *phPlatform = Platforms[i]; + auto Platform = cast(Platforms[i]); + if (Platform->CLPlatform == NativeHandle) { + *phPlatform = cast(Platform); return UR_RESULT_SUCCESS; } } @@ -295,3 +302,5 @@ ur_result_t ur_platform_handle_t_::InitDevices() { return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/platform.hpp b/unified-runtime/source/adapters/opencl/platform.hpp index 2c2bf9b2cf077..0191c36f067b9 100644 --- a/unified-runtime/source/adapters/opencl/platform.hpp +++ b/unified-runtime/source/adapters/opencl/platform.hpp @@ -12,13 +12,15 @@ #include +namespace ur::opencl { + struct ur_device_handle_t_; -struct ur_platform_handle_t_ : ur::opencl::handle_base { +struct ur_platform_handle_t_ : handle_base { using native_type = cl_platform_id; native_type CLPlatform = nullptr; std::vector> Devices; - std::map SubDevices; + std::map SubDevices; std::mutex SubDevicesLock; ur_platform_handle_t_(const ur_platform_handle_t_ &) = delete; @@ -66,3 +68,5 @@ struct ur_platform_handle_t_ : ur::opencl::handle_base { return UR_RESULT_SUCCESS; } }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/program.cpp b/unified-runtime/source/adapters/opencl/program.cpp index da1f2cf00ab63..05c3d9caf0b6a 100644 --- a/unified-runtime/source/adapters/opencl/program.cpp +++ b/unified-runtime/source/adapters/opencl/program.cpp @@ -16,10 +16,13 @@ #include +namespace ur::opencl { + ur_result_t ur_program_handle_t_::makeWithNative(native_type NativeProg, ur_context_handle_t Context, ur_program_handle_t &Program) { - if (!Context) { + auto UrContext = cast(Context); + if (!UrContext) { return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } try { @@ -27,12 +30,13 @@ ur_result_t ur_program_handle_t_::makeWithNative(native_type NativeProg, CL_RETURN_ON_FAILURE(clGetProgramInfo(NativeProg, CL_PROGRAM_CONTEXT, sizeof(CLContext), &CLContext, nullptr)); - if (Context->CLContext != CLContext) { + if (UrContext->CLContext != CLContext) { return UR_RESULT_ERROR_INVALID_CONTEXT; } auto URProgram = std::make_unique( - NativeProg, Context, Context->DeviceCount, Context->Devices.data()); - Program = URProgram.release(); + NativeProg, UrContext, UrContext->DeviceCount, + UrContext->Devices.data()); + Program = cast(URProgram.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -46,7 +50,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( ur_context_handle_t hContext, const void *pIL, size_t length, const ur_program_properties_t *, ur_program_handle_t *phProgram) { - ur_platform_handle_t CurPlatform = hContext->Devices[0]->Platform; + auto Context = cast(hContext); + auto CurPlatform = Context->Devices[0]->Platform; oclv::OpenCLVersion PlatVer; CL_RETURN_ON_FAILURE_AND_SET_NULL(CurPlatform->getPlatformVersion(PlatVer), @@ -57,10 +62,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( if (PlatVer >= oclv::V2_1) { /* Make sure all devices support CL 2.1 or newer as well. */ - for (ur_device_handle_t URDev : hContext->Devices) { + for (ur::opencl::ur_device_handle_t_ *URDev : Context->Devices) { + auto Device = URDev; oclv::OpenCLVersion DevVer; - CL_RETURN_ON_FAILURE_AND_SET_NULL(URDev->getDeviceVersion(DevVer), + CL_RETURN_ON_FAILURE_AND_SET_NULL(Device->getDeviceVersion(DevVer), phProgram); /* If the device does not support CL 2.1 or greater, we need to make sure @@ -69,7 +75,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( if (DevVer < oclv::V2_1) { bool Supported = false; CL_RETURN_ON_FAILURE_AND_SET_NULL( - URDev->checkDeviceExtensions({"cl_khr_il_program"}, Supported), + Device->checkDeviceExtensions({"cl_khr_il_program"}, Supported), phProgram); if (!Supported) { @@ -78,15 +84,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( } } - Program = clCreateProgramWithIL(hContext->CLContext, pIL, length, &Err); + Program = clCreateProgramWithIL(Context->CLContext, pIL, length, &Err); } else { /* If none of the devices conform with CL 2.1 or newer make sure they all * support the cl_khr_il_program extension. */ - for (ur_device_handle_t URDev : hContext->Devices) { + for (ur::opencl::ur_device_handle_t_ *URDev : Context->Devices) { + auto Device = URDev; bool Supported = false; CL_RETURN_ON_FAILURE_AND_SET_NULL( - URDev->checkDeviceExtensions({"cl_khr_il_program"}, Supported), + Device->checkDeviceExtensions({"cl_khr_il_program"}, Supported), phProgram); if (!Supported) { @@ -97,11 +104,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( cl_ext::clCreateProgramWithILKHR_fn CreateProgramWithIL = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - hContext->CLContext, - ur::cl::getAdapter()->fnCache.clCreateProgramWithILKHRCache, + Context->CLContext, + cast(ur::cl::getAdapter())->fnCache.clCreateProgramWithILKHRCache, cl_ext::CreateProgramWithILName, &CreateProgramWithIL)); - Program = CreateProgramWithIL(hContext->CLContext, pIL, length, &Err); + Program = CreateProgramWithIL(Context->CLContext, pIL, length, &Err); } // INVALID_VALUE is only returned in three circumstances according to the cl @@ -125,8 +132,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithIL( try { auto URProgram = std::make_unique( - Program, hContext, hContext->DeviceCount, hContext->Devices.data()); - *phProgram = URProgram.release(); + Program, Context, Context->DeviceCount, Context->Devices.data()); + *phProgram = cast(URProgram.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -140,18 +147,22 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithBinary( ur_context_handle_t hContext, uint32_t numDevices, ur_device_handle_t *phDevices, size_t *pLengths, const uint8_t **ppBinaries, const ur_program_properties_t *, ur_program_handle_t *phProgram) { + auto Context = cast(hContext); std::vector CLDevices(numDevices); - for (uint32_t i = 0; i < numDevices; ++i) - CLDevices[i] = phDevices[i]->CLDevice; + std::vector InternalDevices(numDevices); + for (uint32_t i = 0; i < numDevices; ++i) { + InternalDevices[i] = cast(phDevices[i]); + CLDevices[i] = InternalDevices[i]->CLDevice; + } std::vector BinaryStatus(numDevices); cl_int CLResult; cl_program Program = clCreateProgramWithBinary( - hContext->CLContext, static_cast(numDevices), CLDevices.data(), + Context->CLContext, static_cast(numDevices), CLDevices.data(), pLengths, ppBinaries, BinaryStatus.data(), &CLResult); CL_RETURN_ON_FAILURE(CLResult); auto URProgram = std::make_unique( - Program, hContext, numDevices, phDevices); - *phProgram = URProgram.release(); + Program, Context, numDevices, InternalDevices.data()); + *phProgram = cast(URProgram.release()); for (uint32_t i = 0; i < numDevices; ++i) { CL_RETURN_ON_FAILURE(BinaryStatus[i]); } @@ -164,13 +175,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCompile([[maybe_unused]] ur_context_handle_t hContext, ur_program_handle_t hProgram, const char *pOptions) { - uint32_t DeviceCount = hProgram->NumDevices; + auto Program = cast(hProgram); + uint32_t DeviceCount = Program->NumDevices; std::vector CLDevicesInProgram(DeviceCount); for (uint32_t i = 0; i < DeviceCount; i++) { - CLDevicesInProgram[i] = hProgram->Devices[i]->CLDevice; + CLDevicesInProgram[i] = Program->Devices[i]->CLDevice; } - CL_RETURN_ON_FAILURE(clCompileProgram(hProgram->CLProgram, DeviceCount, + CL_RETURN_ON_FAILURE(clCompileProgram(Program->CLProgram, DeviceCount, CLDevicesInProgram.data(), pOptions, 0, nullptr, nullptr, nullptr, nullptr)); @@ -206,27 +218,28 @@ static cl_int mapURProgramInfoToCL(ur_program_info_t URPropName) { UR_APIEXPORT ur_result_t UR_APICALL urProgramGetInfo(ur_program_handle_t hProgram, ur_program_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Program = cast(hProgram); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); const cl_program_info CLPropName = mapURProgramInfoToCL(propName); switch (static_cast(propName)) { case UR_PROGRAM_INFO_CONTEXT: { - return ReturnValue(hProgram->Context); + return ReturnValue(Program->Context); } case UR_PROGRAM_INFO_NUM_DEVICES: { - cl_uint DeviceCount = hProgram->NumDevices; + cl_uint DeviceCount = Program->NumDevices; return ReturnValue(DeviceCount); } case UR_PROGRAM_INFO_DEVICES: { - return ReturnValue(hProgram->Devices.data(), hProgram->NumDevices); + return ReturnValue(Program->Devices.data(), Program->NumDevices); } case UR_PROGRAM_INFO_REFERENCE_COUNT: { - return ReturnValue(hProgram->RefCount.getCount()); + return ReturnValue(Program->RefCount.getCount()); } default: { size_t CheckPropSize = 0; - auto ClResult = clGetProgramInfo(hProgram->CLProgram, CLPropName, propSize, + auto ClResult = clGetProgramInfo(Program->CLProgram, CLPropName, propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -245,14 +258,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramBuild([[maybe_unused]] ur_context_handle_t hContext, ur_program_handle_t hProgram, const char *pOptions) { - uint32_t DeviceCount = hProgram->NumDevices; + auto Program = cast(hProgram); + uint32_t DeviceCount = Program->NumDevices; std::vector CLDevicesInProgram(DeviceCount); for (uint32_t i = 0; i < DeviceCount; i++) { - CLDevicesInProgram[i] = hProgram->Devices[i]->CLDevice; + CLDevicesInProgram[i] = Program->Devices[i]->CLDevice; } CL_RETURN_ON_FAILURE( - clBuildProgram(hProgram->CLProgram, CLDevicesInProgram.size(), + clBuildProgram(Program->CLProgram, CLDevicesInProgram.size(), CLDevicesInProgram.data(), pOptions, nullptr, nullptr)); return UR_RESULT_SUCCESS; } @@ -262,13 +276,14 @@ urProgramLink(ur_context_handle_t hContext, uint32_t count, const ur_program_handle_t *phPrograms, const char *pOptions, ur_program_handle_t *phProgram) { + auto Context = cast(hContext); cl_int CLResult; std::vector CLPrograms(count); for (uint32_t i = 0; i < count; i++) { - CLPrograms[i] = phPrograms[i]->CLProgram; + CLPrograms[i] = cast(phPrograms[i])->CLProgram; } cl_program Program = clLinkProgram( - hContext->CLContext, 0, nullptr, pOptions, static_cast(count), + Context->CLContext, 0, nullptr, pOptions, static_cast(count), CLPrograms.data(), nullptr, nullptr, &CLResult); if (CL_INVALID_BINARY == CLResult) { @@ -282,8 +297,8 @@ urProgramLink(ur_context_handle_t hContext, uint32_t count, if (Program != nullptr) { try { auto URProgram = std::make_unique( - Program, hContext, hContext->DeviceCount, hContext->Devices.data()); - *phProgram = URProgram.release(); + Program, Context, Context->DeviceCount, Context->Devices.data()); + *phProgram = cast(URProgram.release()); } catch (...) { return exceptionToResult(std::current_exception()); } @@ -367,17 +382,19 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetBuildInfo(ur_program_handle_t hProgram, ur_device_handle_t hDevice, ur_program_build_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Program = cast(hProgram); + auto Device = cast(hDevice); if (propName == UR_PROGRAM_BUILD_INFO_BINARY_TYPE) { UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); cl_program_binary_type BinaryType; CL_RETURN_ON_FAILURE(clGetProgramBuildInfo( - hProgram->CLProgram, hDevice->CLDevice, + Program->CLProgram, Device->CLDevice, mapURProgramBuildInfoToCL(propName), sizeof(cl_program_binary_type), &BinaryType, nullptr)); return ReturnValue(mapCLBinaryTypeToUR(BinaryType)); } size_t CheckPropSize = 0; - cl_int ClErr = clGetProgramBuildInfo(hProgram->CLProgram, hDevice->CLDevice, + cl_int ClErr = clGetProgramBuildInfo(Program->CLProgram, Device->CLDevice, mapURProgramBuildInfoToCL(propName), propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { @@ -393,14 +410,16 @@ urProgramGetBuildInfo(ur_program_handle_t hProgram, ur_device_handle_t hDevice, UR_APIEXPORT ur_result_t UR_APICALL urProgramRetain(ur_program_handle_t hProgram) { - hProgram->RefCount.retain(); + auto Program = cast(hProgram); + Program->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urProgramRelease(ur_program_handle_t hProgram) { - if (hProgram->RefCount.release()) { - delete hProgram; + auto Program = cast(hProgram); + if (Program->RefCount.release()) { + delete Program; } return UR_RESULT_SUCCESS; } @@ -408,7 +427,8 @@ urProgramRelease(ur_program_handle_t hProgram) { UR_APIEXPORT ur_result_t UR_APICALL urProgramGetNativeHandle( ur_program_handle_t hProgram, ur_native_handle_t *phNativeProgram) { - *phNativeProgram = reinterpret_cast(hProgram->CLProgram); + auto Program = cast(hProgram); + *phNativeProgram = reinterpret_cast(Program->CLProgram); return UR_RESULT_SUCCESS; } @@ -420,7 +440,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithNativeHandle( UR_RETURN_ON_FAILURE( ur_program_handle_t_::makeWithNative(NativeHandle, hContext, *phProgram)); - (*phProgram)->IsNativeHandleOwned = + auto Program = cast(*phProgram); + Program->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; return UR_RESULT_SUCCESS; } @@ -429,21 +450,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramSetSpecializationConstants( ur_program_handle_t hProgram, uint32_t count, const ur_specialization_constant_info_t *pSpecConstants) { - cl_program CLProg = hProgram->CLProgram; - if (!hProgram->Context) { + auto Program = cast(hProgram); + cl_program CLProg = Program->CLProgram; + if (!Program->Context) { return UR_RESULT_ERROR_INVALID_PROGRAM; } - ur_context_handle_t Ctx = hProgram->Context; + auto Ctx = Program->Context; if (!Ctx->DeviceCount || !Ctx->Devices[0]->Platform) { return UR_RESULT_ERROR_INVALID_CONTEXT; } - if (ur::cl::getAdapter()->clSetProgramSpecializationConstant) { + if (cast(ur::cl::getAdapter())->clSetProgramSpecializationConstantFn) { for (uint32_t i = 0; i < count; ++i) { - CL_RETURN_ON_FAILURE( - ur::cl::getAdapter()->clSetProgramSpecializationConstant( - CLProg, pSpecConstants[i].id, pSpecConstants[i].size, - pSpecConstants[i].pValue)); + CL_RETURN_ON_FAILURE(cast(ur::cl::getAdapter()) + ->clSetProgramSpecializationConstantFn( + CLProg, pSpecConstants[i].id, + pSpecConstants[i].size, + pSpecConstants[i].pValue)); } } else { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -482,14 +505,18 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetFunctionPointer( ur_device_handle_t hDevice, ur_program_handle_t hProgram, const char *pFunctionName, void **ppFunctionPointer) { - cl_context CLContext = hProgram->Context->CLContext; + auto Program = cast(hProgram); + auto Device = cast(hDevice); + auto Context = Program->Context; + cl_context CLContext = Context->CLContext; cl_ext::clGetDeviceFunctionPointerINTEL_fn FuncT = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( CLContext, - ur::cl::getAdapter()->fnCache.clGetDeviceFunctionPointerINTELCache, + cast(ur::cl::getAdapter()) + ->fnCache.clGetDeviceFunctionPointerINTELCache, cl_ext::GetDeviceFunctionPointerName, &FuncT)); // Check if the kernel name exists to prevent the OpenCL runtime from throwing @@ -500,12 +527,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetFunctionPointer( *ppFunctionPointer = 0; size_t Size; CL_RETURN_ON_FAILURE(clGetProgramInfo( - hProgram->CLProgram, CL_PROGRAM_KERNEL_NAMES, 0, nullptr, &Size)); + Program->CLProgram, CL_PROGRAM_KERNEL_NAMES, 0, nullptr, &Size)); std::string KernelNames(Size, ' '); CL_RETURN_ON_FAILURE( - clGetProgramInfo(hProgram->CLProgram, CL_PROGRAM_KERNEL_NAMES, + clGetProgramInfo(Program->CLProgram, CL_PROGRAM_KERNEL_NAMES, KernelNames.size(), &KernelNames[0], nullptr)); // Get rid of the null terminator and search for the kernel name. If the @@ -516,7 +543,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetFunctionPointer( } const cl_int CLResult = - FuncT(hDevice->CLDevice, hProgram->CLProgram, pFunctionName, + FuncT(Device->CLDevice, Program->CLProgram, pFunctionName, reinterpret_cast(ppFunctionPointer)); // GPU runtime sometimes returns CL_INVALID_ARG_VALUE if the function address // cannot be found but the kernel exists. As the kernel does exist, return @@ -537,7 +564,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetGlobalVariablePointer( void **ppGlobalVariablePointerRet) { cl_context CLContext = nullptr; - CL_RETURN_ON_FAILURE(clGetProgramInfo(hProgram->CLProgram, CL_PROGRAM_CONTEXT, + auto Program = cast(hProgram); + auto Device = cast(hDevice); + CL_RETURN_ON_FAILURE(clGetProgramInfo(Program->CLProgram, CL_PROGRAM_CONTEXT, sizeof(CLContext), &CLContext, nullptr)); @@ -546,11 +575,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetGlobalVariablePointer( UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext< cl_ext::clGetDeviceGlobalVariablePointerINTEL_fn>( CLContext, - ur::cl::getAdapter()->fnCache.clGetDeviceGlobalVariablePointerINTELCache, + cast(ur::cl::getAdapter()) + ->fnCache.clGetDeviceGlobalVariablePointerINTELCache, cl_ext::GetDeviceGlobalVariablePointerName, &FuncT)); const cl_int CLResult = - FuncT(hDevice->CLDevice, hProgram->CLProgram, pGlobalVariableName, + FuncT(Device->CLDevice, Program->CLProgram, pGlobalVariableName, pGlobalVariableSizeRet, ppGlobalVariablePointerRet); if (CLResult != CL_SUCCESS) { @@ -565,3 +595,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramGetGlobalVariablePointer( return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/program.hpp b/unified-runtime/source/adapters/opencl/program.hpp index 8af815a0cda68..0a642c56944b5 100644 --- a/unified-runtime/source/adapters/opencl/program.hpp +++ b/unified-runtime/source/adapters/opencl/program.hpp @@ -14,26 +14,28 @@ #include -struct ur_program_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_program_handle_t_ : handle_base { using native_type = cl_program; native_type CLProgram; - ur_context_handle_t Context; + ur_context_handle_t_ *Context; bool IsNativeHandleOwned = true; uint32_t NumDevices = 0; - std::vector Devices; + std::vector Devices; ur::RefCount RefCount; - ur_program_handle_t_(native_type Prog, ur_context_handle_t Ctx, - uint32_t NumDevices, ur_device_handle_t *Devs) + ur_program_handle_t_(native_type Prog, ur_context_handle_t_ *Ctx, + uint32_t NumDevices, ur_device_handle_t_ **Devs) : handle_base(), CLProgram(Prog), Context(Ctx), NumDevices(NumDevices) { - urContextRetain(Context); + ur::opencl::urContextRetain(cast(Context)); for (uint32_t i = 0; i < NumDevices; i++) { Devices.push_back(Devs[i]); } } ~ur_program_handle_t_() { - urContextRelease(Context); + ur::opencl::urContextRelease(cast(Context)); if (IsNativeHandleOwned) { clReleaseProgram(CLProgram); } @@ -43,3 +45,5 @@ struct ur_program_handle_t_ : ur::opencl::handle_base { ur_context_handle_t Context, ur_program_handle_t &Program); }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/queue.cpp b/unified-runtime/source/adapters/opencl/queue.cpp index 5b2be47420a70..c99376ca6be43 100644 --- a/unified-runtime/source/adapters/opencl/queue.cpp +++ b/unified-runtime/source/adapters/opencl/queue.cpp @@ -71,33 +71,38 @@ mapCLQueuePropsToUR(const cl_command_queue_properties &Properties) { return Flags; } +namespace ur::opencl { + ur_result_t ur_queue_handle_t_::makeWithNative(native_type NativeQueue, ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t &Queue) { try { + auto UrContext = cast(Context); + auto UrDevice = cast(Device); cl_context CLContext; CL_RETURN_ON_FAILURE(clGetCommandQueueInfo( NativeQueue, CL_QUEUE_CONTEXT, sizeof(CLContext), &CLContext, nullptr)); cl_device_id CLDevice; CL_RETURN_ON_FAILURE(clGetCommandQueueInfo( NativeQueue, CL_QUEUE_DEVICE, sizeof(CLDevice), &CLDevice, nullptr)); - if (Context->CLContext != CLContext) { + if (UrContext->CLContext != CLContext) { return UR_RESULT_ERROR_INVALID_CONTEXT; } - if (Device) { - if (Device->CLDevice != CLDevice) { + if (UrDevice) { + if (UrDevice->CLDevice != CLDevice) { return UR_RESULT_ERROR_INVALID_DEVICE; } } else { ur_native_handle_t hNativeHandle = reinterpret_cast(CLDevice); - UR_RETURN_ON_FAILURE(urDeviceCreateWithNativeHandle( + UR_RETURN_ON_FAILURE(ur::opencl::urDeviceCreateWithNativeHandle( hNativeHandle, nullptr, nullptr, &Device)); + UrDevice = cast(Device); } - auto URQueue = std::make_unique(NativeQueue, Context, - Device, false); - Queue = URQueue.release(); + auto URQueue = std::make_unique(NativeQueue, UrContext, + UrDevice, false); + Queue = cast(URQueue.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -110,7 +115,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreate( ur_context_handle_t hContext, ur_device_handle_t hDevice, const ur_queue_properties_t *pProperties, ur_queue_handle_t *phQueue) { - ur_platform_handle_t CurPlatform = hDevice->Platform; + auto Context = cast(hContext); + auto Device = cast(hDevice); + auto CurPlatform = Device->Platform; cl_command_queue_properties CLProperties = pProperties ? convertURQueuePropertiesToCL(pProperties) : 0; @@ -130,13 +137,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreate( CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE); if (Version < oclv::V2_0) { cl_command_queue Queue = - clCreateCommandQueue(hContext->CLContext, hDevice->CLDevice, + clCreateCommandQueue(Context->CLContext, Device->CLDevice, CLProperties & SupportByOpenCL, &RetErr); CL_RETURN_ON_FAILURE(RetErr); try { - auto URQueue = std::make_unique(Queue, hContext, - hDevice, InOrder); - *phQueue = URQueue.release(); + auto URQueue = + std::make_unique(Queue, Context, Device, InOrder); + *phQueue = cast(URQueue.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -150,12 +157,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreate( cl_queue_properties CreationFlagProperties[] = { CL_QUEUE_PROPERTIES, CLProperties & SupportByOpenCL, 0}; cl_command_queue Queue = clCreateCommandQueueWithProperties( - hContext->CLContext, hDevice->CLDevice, CreationFlagProperties, &RetErr); + Context->CLContext, Device->CLDevice, CreationFlagProperties, &RetErr); CL_RETURN_ON_FAILURE(RetErr); try { auto URQueue = - std::make_unique(Queue, hContext, hDevice, InOrder); - *phQueue = URQueue.release(); + std::make_unique(Queue, Context, Device, InOrder); + *phQueue = cast(URQueue.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -169,14 +176,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Queue = cast(hQueue); cl_command_queue_info CLCommandQueueInfo = mapURQueueInfoToCL(propName); UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); if (propName == UR_QUEUE_INFO_EMPTY) { - if (!hQueue->LastEvent) { + if (!Queue->LastEvent) { // Check the status of the queue under OpenCL backend. cl_event Event; CL_RETURN_ON_FAILURE( - clEnqueueMarkerWithWaitList(hQueue->CLQueue, 0, nullptr, &Event)); + clEnqueueMarkerWithWaitList(Queue->CLQueue, 0, nullptr, &Event)); cl_int QueryResult; CL_RETURN_ON_FAILURE( clGetEventInfo(Event, CL_EVENT_COMMAND_EXECUTION_STATUS, @@ -188,8 +196,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, return ReturnValue(false); } else { ur_event_status_t Status; - UR_RETURN_ON_FAILURE(urEventGetInfo( - hQueue->LastEvent, UR_EVENT_INFO_COMMAND_EXECUTION_STATUS, + UR_RETURN_ON_FAILURE(ur::opencl::urEventGetInfo( + cast(Queue->LastEvent), UR_EVENT_INFO_COMMAND_EXECUTION_STATUS, sizeof(ur_event_status_t), (void *)&Status, nullptr)); if (Status == UR_EVENT_STATUS_COMPLETE) { return ReturnValue(true); @@ -199,16 +207,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, } switch (propName) { case UR_QUEUE_INFO_CONTEXT: { - return ReturnValue(hQueue->Context); + return ReturnValue(Queue->Context); } case UR_QUEUE_INFO_DEVICE: { - return ReturnValue(hQueue->Device); + return ReturnValue(Queue->Device); } case UR_QUEUE_INFO_DEVICE_DEFAULT: { size_t CheckPropSize = 0; ur_queue_handle_t_::native_type NewDefault = 0; cl_int RetErr = clGetCommandQueueInfo( - hQueue->CLQueue, CL_QUEUE_DEVICE_DEFAULT, sizeof(CheckPropSize), + Queue->CLQueue, CL_QUEUE_DEVICE_DEFAULT, sizeof(CheckPropSize), &NewDefault, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -216,20 +224,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, CL_RETURN_ON_FAILURE(RetErr); // If we have an existing default device, release it - if (hQueue->DeviceDefault.has_value()) { - urQueueRelease(*hQueue->DeviceDefault); - hQueue->DeviceDefault.reset(); + if (Queue->DeviceDefault.has_value()) { + ur::opencl::urQueueRelease(cast(*Queue->DeviceDefault)); + Queue->DeviceDefault.reset(); } // Then either return this queue (if it is the device default) or create a // new handle to hold onto - if (hQueue->CLQueue == NewDefault) { + if (Queue->CLQueue == NewDefault) { return ReturnValue(hQueue); } else { ur_queue_handle_t NewHandle; UR_RETURN_ON_FAILURE(ur_queue_handle_t_::makeWithNative( - NewDefault, hQueue->Context, hQueue->Device, NewHandle)); - hQueue->DeviceDefault = NewHandle; + NewDefault, cast(Queue->Context), cast(Queue->Device), NewHandle)); + Queue->DeviceDefault = cast(NewHandle); return ReturnValue(NewHandle); } break; @@ -240,17 +248,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, case UR_QUEUE_INFO_FLAGS: { cl_command_queue_properties QueueProperties = 0; CL_RETURN_ON_FAILURE(clGetCommandQueueInfo( - hQueue->CLQueue, CLCommandQueueInfo, sizeof(QueueProperties), + Queue->CLQueue, CLCommandQueueInfo, sizeof(QueueProperties), &QueueProperties, nullptr)); return ReturnValue(mapCLQueuePropsToUR(QueueProperties)); } case UR_QUEUE_INFO_REFERENCE_COUNT: { - return ReturnValue(hQueue->RefCount.getCount()); + return ReturnValue(Queue->RefCount.getCount()); } default: { size_t CheckPropSize = 0; - cl_int RetErr = clGetCommandQueueInfo(hQueue->CLQueue, CLCommandQueueInfo, + cl_int RetErr = clGetCommandQueueInfo(Queue->CLQueue, CLCommandQueueInfo, propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; @@ -268,7 +276,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetInfo(ur_queue_handle_t hQueue, UR_APIEXPORT ur_result_t UR_APICALL urQueueGetNativeHandle(ur_queue_handle_t hQueue, ur_queue_native_desc_t *, ur_native_handle_t *phNativeQueue) { - return getNativeHandle(hQueue->CLQueue, phNativeQueue); + auto Queue = cast(hQueue); + return getNativeHandle(Queue->CLQueue, phNativeQueue); } UR_APIEXPORT ur_result_t UR_APICALL urQueueCreateWithNativeHandle( @@ -282,32 +291,37 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreateWithNativeHandle( UR_RETURN_ON_FAILURE(ur_queue_handle_t_::makeWithNative( NativeHandle, hContext, hDevice, *phQueue)); - (*phQueue)->IsNativeHandleOwned = + auto Queue = cast(*phQueue); + Queue->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urQueueFinish(ur_queue_handle_t hQueue) { - cl_int RetErr = clFinish(hQueue->CLQueue); + auto Queue = cast(hQueue); + cl_int RetErr = clFinish(Queue->CLQueue); CL_RETURN_ON_FAILURE(RetErr); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urQueueFlush(ur_queue_handle_t hQueue) { - cl_int RetErr = clFlush(hQueue->CLQueue); + auto Queue = cast(hQueue); + cl_int RetErr = clFlush(Queue->CLQueue); CL_RETURN_ON_FAILURE(RetErr); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urQueueRetain(ur_queue_handle_t hQueue) { - hQueue->RefCount.retain(); + auto Queue = cast(hQueue); + Queue->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urQueueRelease(ur_queue_handle_t hQueue) { - if (hQueue->RefCount.release()) { - delete hQueue; + auto Queue = cast(hQueue); + if (Queue->RefCount.release()) { + delete Queue; } return UR_RESULT_SUCCESS; } @@ -355,3 +369,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueHostTaskExp( ur_event_handle_t * /* phEvent */) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/queue.hpp b/unified-runtime/source/adapters/opencl/queue.hpp index 5684e04340b47..b0d657f3a1e9c 100644 --- a/unified-runtime/source/adapters/opencl/queue.hpp +++ b/unified-runtime/source/adapters/opencl/queue.hpp @@ -15,28 +15,30 @@ #include -struct ur_queue_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_queue_handle_t_ : handle_base { using native_type = cl_command_queue; native_type CLQueue; - ur_context_handle_t Context; - ur_device_handle_t Device; + ur_context_handle_t_ *Context; + ur_device_handle_t_ *Device; // Used to keep a handle to the default queue alive if it is different - std::optional DeviceDefault = std::nullopt; + std::optional DeviceDefault = std::nullopt; bool IsNativeHandleOwned = true; // Used to implement UR_QUEUE_INFO_EMPTY query bool IsInOrder; - ur_event_handle_t LastEvent = nullptr; + ur_event_handle_t_ *LastEvent = nullptr; ur::RefCount RefCount; ur_queue_handle_t_(const ur_queue_handle_t_ &) = delete; ur_queue_handle_t_ &operator=(const ur_queue_handle_t_ &) = delete; - ur_queue_handle_t_(native_type Queue, ur_context_handle_t Ctx, - ur_device_handle_t Dev, bool InOrder) + ur_queue_handle_t_(native_type Queue, ur_context_handle_t_ *Ctx, + ur_device_handle_t_ *Dev, bool InOrder) : handle_base(), CLQueue(Queue), Context(Ctx), Device(Dev), IsInOrder(InOrder) { - urDeviceRetain(Device); - urContextRetain(Context); + ur::opencl::urDeviceRetain(cast(Device)); + ur::opencl::urContextRetain(cast(Context)); } static ur_result_t makeWithNative(native_type NativeQueue, @@ -45,13 +47,13 @@ struct ur_queue_handle_t_ : ur::opencl::handle_base { ur_queue_handle_t &Queue); ~ur_queue_handle_t_() { - urDeviceRelease(Device); - urContextRelease(Context); + ur::opencl::urDeviceRelease(cast(Device)); + ur::opencl::urContextRelease(cast(Context)); if (IsNativeHandleOwned) { clReleaseCommandQueue(CLQueue); } if (DeviceDefault.has_value()) { - urQueueRelease(*DeviceDefault); + ur::opencl::urQueueRelease(cast(*DeviceDefault)); } } @@ -62,12 +64,14 @@ struct ur_queue_handle_t_ : ur::opencl::handle_base { return UR_RESULT_SUCCESS; } if (LastEvent) { - UR_RETURN_ON_FAILURE(urEventRelease(LastEvent)); + UR_RETURN_ON_FAILURE(ur::opencl::urEventRelease(cast(LastEvent))); } - LastEvent = Event; + LastEvent = cast(Event); if (LastEvent) { - UR_RETURN_ON_FAILURE(urEventRetain(LastEvent)); + UR_RETURN_ON_FAILURE(ur::opencl::urEventRetain(cast(LastEvent))); } return UR_RESULT_SUCCESS; } }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/sampler.cpp b/unified-runtime/source/adapters/opencl/sampler.cpp index a40ef560371c5..1540cdf4c3bce 100644 --- a/unified-runtime/source/adapters/opencl/sampler.cpp +++ b/unified-runtime/source/adapters/opencl/sampler.cpp @@ -133,9 +133,12 @@ void cl2URSamplerInfoValue(cl_sampler_info Info, void *InfoValue) { } // namespace +namespace ur::opencl { + ur_result_t urSamplerCreate(ur_context_handle_t hContext, const ur_sampler_desc_t *pDesc, ur_sampler_handle_t *phSampler) { + auto Context = cast(hContext); // Initialize properties according to OpenCL 2.1 spec. cl_int ErrorCode; @@ -145,11 +148,11 @@ ur_result_t urSamplerCreate(ur_context_handle_t hContext, try { // Always call OpenCL 1.0 API cl_sampler Sampler = clCreateSampler( - hContext->CLContext, static_cast(pDesc->normalizedCoords), + Context->CLContext, static_cast(pDesc->normalizedCoords), AddressingMode, FilterMode, &ErrorCode); CL_RETURN_ON_FAILURE(ErrorCode); - auto URSampler = std::make_unique(Sampler, hContext); - *phSampler = URSampler.release(); + auto URSampler = std::make_unique(Sampler, Context); + *phSampler = cast(URSampler.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -162,6 +165,7 @@ ur_result_t urSamplerCreate(ur_context_handle_t hContext, UR_APIEXPORT ur_result_t UR_APICALL urSamplerGetInfo(ur_sampler_handle_t hSampler, ur_sampler_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto Sampler = cast(hSampler); cl_sampler_info SamplerInfo = ur2CLSamplerInfo(propName); static_assert(sizeof(cl_addressing_mode) == sizeof(ur_sampler_addressing_mode_t)); @@ -171,17 +175,17 @@ urSamplerGetInfo(ur_sampler_handle_t hSampler, ur_sampler_info_t propName, switch (propName) { case UR_SAMPLER_INFO_CONTEXT: { - return ReturnValue(hSampler->Context); + return ReturnValue(Sampler->Context); } case UR_SAMPLER_INFO_REFERENCE_COUNT: { - return ReturnValue(hSampler->RefCount.getCount()); + return ReturnValue(Sampler->RefCount.getCount()); } // ur_bool_t have a size of uint8_t, but cl_bool size have the size of // uint32_t so this adjust UR_SAMPLER_INFO_NORMALIZED_COORDS info to map // between them. case UR_SAMPLER_INFO_NORMALIZED_COORDS: { cl_bool normalized_coords = false; - Err = mapCLErrorToUR(clGetSamplerInfo(hSampler->CLSampler, SamplerInfo, + Err = mapCLErrorToUR(clGetSamplerInfo(Sampler->CLSampler, SamplerInfo, sizeof(cl_bool), &normalized_coords, nullptr)); if (pPropValue && propSize != sizeof(ur_bool_t)) { @@ -199,9 +203,8 @@ urSamplerGetInfo(ur_sampler_handle_t hSampler, ur_sampler_info_t propName, } default: { size_t CheckPropSize = 0; - ur_result_t Err = - mapCLErrorToUR(clGetSamplerInfo(hSampler->CLSampler, SamplerInfo, - propSize, pPropValue, &CheckPropSize)); + ur_result_t Err = mapCLErrorToUR(clGetSamplerInfo( + Sampler->CLSampler, SamplerInfo, propSize, pPropValue, &CheckPropSize)); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; } @@ -220,21 +223,24 @@ urSamplerGetInfo(ur_sampler_handle_t hSampler, ur_sampler_info_t propName, UR_APIEXPORT ur_result_t UR_APICALL urSamplerRetain(ur_sampler_handle_t hSampler) { - hSampler->RefCount.retain(); + auto Sampler = cast(hSampler); + Sampler->RefCount.retain(); return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urSamplerRelease(ur_sampler_handle_t hSampler) { - if (hSampler->RefCount.release()) { - delete hSampler; + auto Sampler = cast(hSampler); + if (Sampler->RefCount.release()) { + delete Sampler; } return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urSamplerGetNativeHandle( ur_sampler_handle_t hSampler, ur_native_handle_t *phNativeSampler) { - *phNativeSampler = reinterpret_cast(hSampler->CLSampler); + auto Sampler = cast(hSampler); + *phNativeSampler = reinterpret_cast(Sampler->CLSampler); return UR_RESULT_SUCCESS; } @@ -242,13 +248,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urSamplerCreateWithNativeHandle( ur_native_handle_t hNativeSampler, ur_context_handle_t hContext, const ur_sampler_native_properties_t *pProperties, ur_sampler_handle_t *phSampler) { + auto Context = cast(hContext); cl_sampler NativeHandle = reinterpret_cast(hNativeSampler); try { auto URSampler = - std::make_unique(NativeHandle, hContext); + std::make_unique(NativeHandle, Context); URSampler->IsNativeHandleOwned = pProperties ? pProperties->isNativeHandleOwned : false; - *phSampler = URSampler.release(); + *phSampler = cast(URSampler.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -257,3 +264,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urSamplerCreateWithNativeHandle( return UR_RESULT_SUCCESS; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/sampler.hpp b/unified-runtime/source/adapters/opencl/sampler.hpp index 8df0d182fb159..ab29794026a44 100644 --- a/unified-runtime/source/adapters/opencl/sampler.hpp +++ b/unified-runtime/source/adapters/opencl/sampler.hpp @@ -13,25 +13,32 @@ #include -struct ur_sampler_handle_t_ : ur::opencl::handle_base { +namespace ur::opencl { + +struct ur_context_handle_t_; + +struct ur_sampler_handle_t_ : handle_base { using native_type = cl_sampler; native_type CLSampler; - ur_context_handle_t Context; + ur::opencl::ur_context_handle_t_ *Context; bool IsNativeHandleOwned = false; ur::RefCount RefCount; ur_sampler_handle_t_(const ur_sampler_handle_t_ &) = delete; ur_sampler_handle_t_ &operator=(const ur_sampler_handle_t_ &) = delete; - ur_sampler_handle_t_(native_type Sampler, ur_context_handle_t Ctx) + ur_sampler_handle_t_(native_type Sampler, + ur::opencl::ur_context_handle_t_ *Ctx) : handle_base(), CLSampler(Sampler), Context(Ctx) { - urContextRetain(Context); + ur::opencl::urContextRetain(cast(Context)); } ~ur_sampler_handle_t_() { - urContextRelease(Context); + ur::opencl::urContextRelease(cast(Context)); if (IsNativeHandleOwned) { clReleaseSampler(CLSampler); } } }; + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp index 3fe8a08ad44a1..c0be70663e50c 100644 --- a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp @@ -1,572 +1,773 @@ -//===--------- ur_interface_loader.cpp - Unified Runtime ------------===// +//===--------- ur_interface_loader.cpp - Level Zero Adapter ------------===// // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM // Exceptions. See https://llvm.org/LICENSE.txt for license information. +// // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -#include "common.hpp" +#include #include #include -namespace { +#include "ur_interface_loader.hpp" -// TODO - this is a duplicate of what is in the L0 plugin -// We should move this to somewhere common -ur_result_t validateProcInputs(ur_api_version_t Version, void *pDdiTable) { +static ur_result_t validateProcInputs(ur_api_version_t version, + void *pDdiTable) { if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } // Pre 1.0 we enforce loader and adapter must have same version. // Post 1.0 only major version match should be required. - if (Version != UR_API_VERSION_CURRENT) { + if (version != UR_API_VERSION_CURRENT) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } return UR_RESULT_SUCCESS; } -} // namespace +#ifdef UR_STATIC_ADAPTER_OPENCL +namespace ur::opencl { +#else extern "C" { +#endif -UR_DLLEXPORT ur_result_t UR_APICALL urGetAdapterProcAddrTable( +UR_APIEXPORT ur_result_t UR_APICALL urGetAdapterProcAddrTable( ur_api_version_t version, ur_adapter_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnGet = urAdapterGet; - pDdiTable->pfnRelease = urAdapterRelease; - pDdiTable->pfnRetain = urAdapterRetain; - pDdiTable->pfnGetLastError = urAdapterGetLastError; - pDdiTable->pfnGetInfo = urAdapterGetInfo; - pDdiTable->pfnSetLoggerCallback = urAdapterSetLoggerCallback; - pDdiTable->pfnSetLoggerCallbackLevel = urAdapterSetLoggerCallbackLevel; - return UR_RESULT_SUCCESS; -} + pDdiTable->pfnGet = ur::opencl::urAdapterGet; + pDdiTable->pfnRelease = ur::opencl::urAdapterRelease; + pDdiTable->pfnRetain = ur::opencl::urAdapterRetain; + pDdiTable->pfnGetLastError = ur::opencl::urAdapterGetLastError; + pDdiTable->pfnGetInfo = ur::opencl::urAdapterGetInfo; + pDdiTable->pfnSetLoggerCallback = ur::opencl::urAdapterSetLoggerCallback; + pDdiTable->pfnSetLoggerCallbackLevel = + ur::opencl::urAdapterSetLoggerCallbackLevel; -UR_DLLEXPORT ur_result_t UR_APICALL urGetPlatformProcAddrTable( - ur_api_version_t Version, ur_platform_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; - } - pDdiTable->pfnCreateWithNativeHandle = urPlatformCreateWithNativeHandle; - pDdiTable->pfnGet = urPlatformGet; - pDdiTable->pfnGetApiVersion = urPlatformGetApiVersion; - pDdiTable->pfnGetInfo = urPlatformGetInfo; - pDdiTable->pfnGetNativeHandle = urPlatformGetNativeHandle; - pDdiTable->pfnGetBackendOption = urPlatformGetBackendOption; - return UR_RESULT_SUCCESS; + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetContextProcAddrTable( - ur_api_version_t Version, ur_context_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( + ur_api_version_t version, ur_bindless_images_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreate = urContextCreate; - pDdiTable->pfnCreateWithNativeHandle = urContextCreateWithNativeHandle; - pDdiTable->pfnGetInfo = urContextGetInfo; - pDdiTable->pfnGetNativeHandle = urContextGetNativeHandle; - pDdiTable->pfnRelease = urContextRelease; - pDdiTable->pfnRetain = urContextRetain; - pDdiTable->pfnSetExtendedDeleter = urContextSetExtendedDeleter; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnUnsampledImageHandleDestroyExp = + ur::opencl::urBindlessImagesUnsampledImageHandleDestroyExp; + pDdiTable->pfnSampledImageHandleDestroyExp = + ur::opencl::urBindlessImagesSampledImageHandleDestroyExp; + pDdiTable->pfnImageAllocateExp = ur::opencl::urBindlessImagesImageAllocateExp; + pDdiTable->pfnImageFreeExp = ur::opencl::urBindlessImagesImageFreeExp; + pDdiTable->pfnUnsampledImageCreateExp = + ur::opencl::urBindlessImagesUnsampledImageCreateExp; + pDdiTable->pfnSampledImageCreateExp = + ur::opencl::urBindlessImagesSampledImageCreateExp; + pDdiTable->pfnImageCopyExp = ur::opencl::urBindlessImagesImageCopyExp; + pDdiTable->pfnImageGetInfoExp = ur::opencl::urBindlessImagesImageGetInfoExp; + pDdiTable->pfnGetImageMemoryHandleTypeSupportExp = + ur::opencl::urBindlessImagesGetImageMemoryHandleTypeSupportExp; + pDdiTable->pfnGetImageUnsampledHandleSupportExp = + ur::opencl::urBindlessImagesGetImageUnsampledHandleSupportExp; + pDdiTable->pfnGetImageSampledHandleSupportExp = + ur::opencl::urBindlessImagesGetImageSampledHandleSupportExp; + pDdiTable->pfnMipmapGetLevelExp = + ur::opencl::urBindlessImagesMipmapGetLevelExp; + pDdiTable->pfnMipmapFreeExp = ur::opencl::urBindlessImagesMipmapFreeExp; + pDdiTable->pfnImportExternalMemoryExp = + ur::opencl::urBindlessImagesImportExternalMemoryExp; + pDdiTable->pfnMapExternalArrayExp = + ur::opencl::urBindlessImagesMapExternalArrayExp; + pDdiTable->pfnMapExternalLinearMemoryExp = + ur::opencl::urBindlessImagesMapExternalLinearMemoryExp; + pDdiTable->pfnReleaseExternalMemoryExp = + ur::opencl::urBindlessImagesReleaseExternalMemoryExp; + pDdiTable->pfnFreeMappedLinearMemoryExp = + ur::opencl::urBindlessImagesFreeMappedLinearMemoryExp; + pDdiTable->pfnSupportsImportingHandleTypeExp = + ur::opencl::urBindlessImagesSupportsImportingHandleTypeExp; + pDdiTable->pfnImportExternalSemaphoreExp = + ur::opencl::urBindlessImagesImportExternalSemaphoreExp; + pDdiTable->pfnReleaseExternalSemaphoreExp = + ur::opencl::urBindlessImagesReleaseExternalSemaphoreExp; + pDdiTable->pfnWaitExternalSemaphoreExp = + ur::opencl::urBindlessImagesWaitExternalSemaphoreExp; + pDdiTable->pfnSignalExternalSemaphoreExp = + ur::opencl::urBindlessImagesSignalExternalSemaphoreExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetEventProcAddrTable( - ur_api_version_t Version, ur_event_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetCommandBufferExpProcAddrTable( + ur_api_version_t version, ur_command_buffer_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreateWithNativeHandle = urEventCreateWithNativeHandle; - pDdiTable->pfnGetInfo = urEventGetInfo; - pDdiTable->pfnGetNativeHandle = urEventGetNativeHandle; - pDdiTable->pfnGetProfilingInfo = urEventGetProfilingInfo; - pDdiTable->pfnRelease = urEventRelease; - pDdiTable->pfnRetain = urEventRetain; - pDdiTable->pfnSetCallback = urEventSetCallback; - pDdiTable->pfnWait = urEventWait; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreateExp = ur::opencl::urCommandBufferCreateExp; + pDdiTable->pfnRetainExp = ur::opencl::urCommandBufferRetainExp; + pDdiTable->pfnReleaseExp = ur::opencl::urCommandBufferReleaseExp; + pDdiTable->pfnFinalizeExp = ur::opencl::urCommandBufferFinalizeExp; + pDdiTable->pfnAppendKernelLaunchExp = + ur::opencl::urCommandBufferAppendKernelLaunchExp; + pDdiTable->pfnAppendKernelLaunchWithArgsExp = + ur::opencl::urCommandBufferAppendKernelLaunchWithArgsExp; + pDdiTable->pfnAppendUSMMemcpyExp = + ur::opencl::urCommandBufferAppendUSMMemcpyExp; + pDdiTable->pfnAppendUSMFillExp = ur::opencl::urCommandBufferAppendUSMFillExp; + pDdiTable->pfnAppendMemBufferCopyExp = + ur::opencl::urCommandBufferAppendMemBufferCopyExp; + pDdiTable->pfnAppendMemBufferWriteExp = + ur::opencl::urCommandBufferAppendMemBufferWriteExp; + pDdiTable->pfnAppendMemBufferReadExp = + ur::opencl::urCommandBufferAppendMemBufferReadExp; + pDdiTable->pfnAppendMemBufferCopyRectExp = + ur::opencl::urCommandBufferAppendMemBufferCopyRectExp; + pDdiTable->pfnAppendMemBufferWriteRectExp = + ur::opencl::urCommandBufferAppendMemBufferWriteRectExp; + pDdiTable->pfnAppendMemBufferReadRectExp = + ur::opencl::urCommandBufferAppendMemBufferReadRectExp; + pDdiTable->pfnAppendMemBufferFillExp = + ur::opencl::urCommandBufferAppendMemBufferFillExp; + pDdiTable->pfnAppendUSMPrefetchExp = + ur::opencl::urCommandBufferAppendUSMPrefetchExp; + pDdiTable->pfnAppendUSMAdviseExp = + ur::opencl::urCommandBufferAppendUSMAdviseExp; + pDdiTable->pfnAppendNativeCommandExp = + ur::opencl::urCommandBufferAppendNativeCommandExp; + pDdiTable->pfnUpdateKernelLaunchExp = + ur::opencl::urCommandBufferUpdateKernelLaunchExp; + pDdiTable->pfnUpdateSignalEventExp = + ur::opencl::urCommandBufferUpdateSignalEventExp; + pDdiTable->pfnUpdateWaitEventsExp = + ur::opencl::urCommandBufferUpdateWaitEventsExp; + pDdiTable->pfnGetInfoExp = ur::opencl::urCommandBufferGetInfoExp; + pDdiTable->pfnGetNativeHandleExp = + ur::opencl::urCommandBufferGetNativeHandleExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetEventExpProcAddrTable( - ur_api_version_t version, ur_event_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetContextProcAddrTable( + ur_api_version_t version, ur_context_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnCreateExp = urEventCreateExp; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreate = ur::opencl::urContextCreate; + pDdiTable->pfnRetain = ur::opencl::urContextRetain; + pDdiTable->pfnRelease = ur::opencl::urContextRelease; + pDdiTable->pfnGetInfo = ur::opencl::urContextGetInfo; + pDdiTable->pfnGetNativeHandle = ur::opencl::urContextGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urContextCreateWithNativeHandle; + pDdiTable->pfnSetExtendedDeleter = ur::opencl::urContextSetExtendedDeleter; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetProgramProcAddrTable( - ur_api_version_t Version, ur_program_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueProcAddrTable( + ur_api_version_t version, ur_enqueue_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnBuild = urProgramBuild; - pDdiTable->pfnCompile = urProgramCompile; - pDdiTable->pfnCreateWithBinary = urProgramCreateWithBinary; - pDdiTable->pfnCreateWithIL = urProgramCreateWithIL; - pDdiTable->pfnCreateWithNativeHandle = urProgramCreateWithNativeHandle; - pDdiTable->pfnGetBuildInfo = urProgramGetBuildInfo; - pDdiTable->pfnGetFunctionPointer = urProgramGetFunctionPointer; - pDdiTable->pfnGetGlobalVariablePointer = urProgramGetGlobalVariablePointer; - pDdiTable->pfnGetInfo = urProgramGetInfo; - pDdiTable->pfnGetNativeHandle = urProgramGetNativeHandle; - pDdiTable->pfnLink = urProgramLink; - pDdiTable->pfnRelease = urProgramRelease; - pDdiTable->pfnRetain = urProgramRetain; - pDdiTable->pfnSetSpecializationConstants = - urProgramSetSpecializationConstants; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnEventsWait = ur::opencl::urEnqueueEventsWait; + pDdiTable->pfnEventsWaitWithBarrier = + ur::opencl::urEnqueueEventsWaitWithBarrier; + pDdiTable->pfnMemBufferRead = ur::opencl::urEnqueueMemBufferRead; + pDdiTable->pfnMemBufferWrite = ur::opencl::urEnqueueMemBufferWrite; + pDdiTable->pfnMemBufferReadRect = ur::opencl::urEnqueueMemBufferReadRect; + pDdiTable->pfnMemBufferWriteRect = ur::opencl::urEnqueueMemBufferWriteRect; + pDdiTable->pfnMemBufferCopy = ur::opencl::urEnqueueMemBufferCopy; + pDdiTable->pfnMemBufferCopyRect = ur::opencl::urEnqueueMemBufferCopyRect; + pDdiTable->pfnMemBufferFill = ur::opencl::urEnqueueMemBufferFill; + pDdiTable->pfnMemImageRead = ur::opencl::urEnqueueMemImageRead; + pDdiTable->pfnMemImageWrite = ur::opencl::urEnqueueMemImageWrite; + pDdiTable->pfnMemImageCopy = ur::opencl::urEnqueueMemImageCopy; + pDdiTable->pfnMemBufferMap = ur::opencl::urEnqueueMemBufferMap; + pDdiTable->pfnMemUnmap = ur::opencl::urEnqueueMemUnmap; + pDdiTable->pfnUSMFill = ur::opencl::urEnqueueUSMFill; + pDdiTable->pfnUSMMemcpy = ur::opencl::urEnqueueUSMMemcpy; + pDdiTable->pfnUSMPrefetch = ur::opencl::urEnqueueUSMPrefetch; + pDdiTable->pfnUSMAdvise = ur::opencl::urEnqueueUSMAdvise; + pDdiTable->pfnUSMFill2D = ur::opencl::urEnqueueUSMFill2D; + pDdiTable->pfnUSMMemcpy2D = ur::opencl::urEnqueueUSMMemcpy2D; + pDdiTable->pfnDeviceGlobalVariableWrite = + ur::opencl::urEnqueueDeviceGlobalVariableWrite; + pDdiTable->pfnDeviceGlobalVariableRead = + ur::opencl::urEnqueueDeviceGlobalVariableRead; + pDdiTable->pfnReadHostPipe = ur::opencl::urEnqueueReadHostPipe; + pDdiTable->pfnWriteHostPipe = ur::opencl::urEnqueueWriteHostPipe; + pDdiTable->pfnEventsWaitWithBarrierExt = + ur::opencl::urEnqueueEventsWaitWithBarrierExt; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetKernelProcAddrTable( - ur_api_version_t Version, ur_kernel_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( + ur_api_version_t version, ur_enqueue_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreate = urKernelCreate; - pDdiTable->pfnCreateWithNativeHandle = urKernelCreateWithNativeHandle; - pDdiTable->pfnGetGroupInfo = urKernelGetGroupInfo; - pDdiTable->pfnGetInfo = urKernelGetInfo; - pDdiTable->pfnGetNativeHandle = urKernelGetNativeHandle; - pDdiTable->pfnGetSubGroupInfo = urKernelGetSubGroupInfo; - pDdiTable->pfnRelease = urKernelRelease; - pDdiTable->pfnRetain = urKernelRetain; - pDdiTable->pfnSetExecInfo = urKernelSetExecInfo; - pDdiTable->pfnSetSpecializationConstants = urKernelSetSpecializationConstants; - pDdiTable->pfnGetSuggestedLocalWorkSize = urKernelGetSuggestedLocalWorkSize; - pDdiTable->pfnGetSuggestedLocalWorkSizeWithArgs = - urKernelGetSuggestedLocalWorkSizeWithArgs; - pDdiTable->pfnSuggestMaxCooperativeGroupCount = - urKernelSuggestMaxCooperativeGroupCount; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnKernelLaunchWithArgsExp = + ur::opencl::urEnqueueKernelLaunchWithArgsExp; + pDdiTable->pfnUSMDeviceAllocExp = ur::opencl::urEnqueueUSMDeviceAllocExp; + pDdiTable->pfnUSMSharedAllocExp = ur::opencl::urEnqueueUSMSharedAllocExp; + pDdiTable->pfnUSMHostAllocExp = ur::opencl::urEnqueueUSMHostAllocExp; + pDdiTable->pfnUSMFreeExp = ur::opencl::urEnqueueUSMFreeExp; + pDdiTable->pfnTimestampRecordingExp = + ur::opencl::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = ur::opencl::urEnqueueCommandBufferExp; + pDdiTable->pfnHostTaskExp = ur::opencl::urEnqueueHostTaskExp; + pDdiTable->pfnNativeCommandExp = ur::opencl::urEnqueueNativeCommandExp; + pDdiTable->pfnGraphExp = ur::opencl::urEnqueueGraphExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetSamplerProcAddrTable( - ur_api_version_t Version, ur_sampler_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetEventProcAddrTable( + ur_api_version_t version, ur_event_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreate = urSamplerCreate; - pDdiTable->pfnCreateWithNativeHandle = urSamplerCreateWithNativeHandle; - pDdiTable->pfnGetInfo = urSamplerGetInfo; - pDdiTable->pfnGetNativeHandle = urSamplerGetNativeHandle; - pDdiTable->pfnRelease = urSamplerRelease; - pDdiTable->pfnRetain = urSamplerRetain; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnGetInfo = ur::opencl::urEventGetInfo; + pDdiTable->pfnGetProfilingInfo = ur::opencl::urEventGetProfilingInfo; + pDdiTable->pfnWait = ur::opencl::urEventWait; + pDdiTable->pfnRetain = ur::opencl::urEventRetain; + pDdiTable->pfnRelease = ur::opencl::urEventRelease; + pDdiTable->pfnGetNativeHandle = ur::opencl::urEventGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urEventCreateWithNativeHandle; + pDdiTable->pfnSetCallback = ur::opencl::urEventSetCallback; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL -urGetMemProcAddrTable(ur_api_version_t Version, ur_mem_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetEventExpProcAddrTable( + ur_api_version_t version, ur_event_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnBufferCreate = urMemBufferCreate; - pDdiTable->pfnBufferPartition = urMemBufferPartition; - pDdiTable->pfnBufferCreateWithNativeHandle = - urMemBufferCreateWithNativeHandle; - pDdiTable->pfnImageCreateWithNativeHandle = urMemImageCreateWithNativeHandle; - pDdiTable->pfnGetInfo = urMemGetInfo; - pDdiTable->pfnGetNativeHandle = urMemGetNativeHandle; - pDdiTable->pfnImageCreate = urMemImageCreate; - pDdiTable->pfnImageGetInfo = urMemImageGetInfo; - pDdiTable->pfnImageCreateWithNativeHandle = urMemImageCreateWithNativeHandle; - pDdiTable->pfnRelease = urMemRelease; - pDdiTable->pfnRetain = urMemRetain; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreateExp = ur::opencl::urEventCreateExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetEnqueueProcAddrTable( - ur_api_version_t Version, ur_enqueue_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetGraphExpProcAddrTable( + ur_api_version_t version, ur_graph_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnDeviceGlobalVariableRead = urEnqueueDeviceGlobalVariableRead; - pDdiTable->pfnDeviceGlobalVariableWrite = urEnqueueDeviceGlobalVariableWrite; - pDdiTable->pfnEventsWait = urEnqueueEventsWait; - pDdiTable->pfnEventsWaitWithBarrier = urEnqueueEventsWaitWithBarrier; - pDdiTable->pfnEventsWaitWithBarrierExt = urEnqueueEventsWaitWithBarrierExt; - pDdiTable->pfnMemBufferCopy = urEnqueueMemBufferCopy; - pDdiTable->pfnMemBufferCopyRect = urEnqueueMemBufferCopyRect; - pDdiTable->pfnMemBufferFill = urEnqueueMemBufferFill; - pDdiTable->pfnMemBufferMap = urEnqueueMemBufferMap; - pDdiTable->pfnMemBufferRead = urEnqueueMemBufferRead; - pDdiTable->pfnMemBufferReadRect = urEnqueueMemBufferReadRect; - pDdiTable->pfnMemBufferWrite = urEnqueueMemBufferWrite; - pDdiTable->pfnMemBufferWriteRect = urEnqueueMemBufferWriteRect; - pDdiTable->pfnMemImageCopy = urEnqueueMemImageCopy; - pDdiTable->pfnMemImageRead = urEnqueueMemImageRead; - pDdiTable->pfnMemImageWrite = urEnqueueMemImageWrite; - pDdiTable->pfnMemUnmap = urEnqueueMemUnmap; - pDdiTable->pfnUSMFill2D = urEnqueueUSMFill2D; - pDdiTable->pfnUSMFill = urEnqueueUSMFill; - pDdiTable->pfnUSMAdvise = urEnqueueUSMAdvise; - pDdiTable->pfnUSMMemcpy2D = urEnqueueUSMMemcpy2D; - pDdiTable->pfnUSMMemcpy = urEnqueueUSMMemcpy; - pDdiTable->pfnUSMPrefetch = urEnqueueUSMPrefetch; - pDdiTable->pfnReadHostPipe = urEnqueueReadHostPipe; - pDdiTable->pfnWriteHostPipe = urEnqueueWriteHostPipe; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreateExp = ur::opencl::urGraphCreateExp; + pDdiTable->pfnInstantiateGraphExp = ur::opencl::urGraphInstantiateGraphExp; + pDdiTable->pfnDestroyExp = ur::opencl::urGraphDestroyExp; + pDdiTable->pfnExecutableGraphDestroyExp = + ur::opencl::urGraphExecutableGraphDestroyExp; + pDdiTable->pfnIsEmptyExp = ur::opencl::urGraphIsEmptyExp; + pDdiTable->pfnGetIdExp = ur::opencl::urGraphGetIdExp; + pDdiTable->pfnSetDestructionCallbackExp = + ur::opencl::urGraphSetDestructionCallbackExp; + pDdiTable->pfnDumpContentsExp = ur::opencl::urGraphDumpContentsExp; + pDdiTable->pfnGetNativeHandleExp = ur::opencl::urGraphGetNativeHandleExp; + pDdiTable->pfnExecutableGraphGetNativeHandleExp = + ur::opencl::urGraphExecutableGraphGetNativeHandleExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetQueueProcAddrTable( - ur_api_version_t Version, ur_queue_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( + ur_api_version_t version, ur_ipc_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreate = urQueueCreate; - pDdiTable->pfnCreateWithNativeHandle = urQueueCreateWithNativeHandle; - pDdiTable->pfnFinish = urQueueFinish; - pDdiTable->pfnFlush = urQueueFlush; - pDdiTable->pfnGetInfo = urQueueGetInfo; - pDdiTable->pfnGetNativeHandle = urQueueGetNativeHandle; - pDdiTable->pfnRelease = urQueueRelease; - pDdiTable->pfnRetain = urQueueRetain; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnGetMemHandleExp = ur::opencl::urIPCGetMemHandleExp; + pDdiTable->pfnPutMemHandleExp = ur::opencl::urIPCPutMemHandleExp; + pDdiTable->pfnOpenMemHandleExp = ur::opencl::urIPCOpenMemHandleExp; + pDdiTable->pfnCloseMemHandleExp = ur::opencl::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur::opencl::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur::opencl::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = ur::opencl::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = ur::opencl::urIPCClosePhysMemHandleExp; + pDdiTable->pfnGetEventHandleExp = ur::opencl::urIPCGetEventHandleExp; + pDdiTable->pfnPutEventHandleExp = ur::opencl::urIPCPutEventHandleExp; + pDdiTable->pfnOpenEventHandleExp = ur::opencl::urIPCOpenEventHandleExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL -urGetUSMProcAddrTable(ur_api_version_t Version, ur_usm_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetKernelProcAddrTable( + ur_api_version_t version, ur_kernel_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnDeviceAlloc = urUSMDeviceAlloc; - pDdiTable->pfnFree = urUSMFree; - pDdiTable->pfnGetMemAllocInfo = urUSMGetMemAllocInfo; - pDdiTable->pfnHostAlloc = urUSMHostAlloc; - pDdiTable->pfnPoolCreate = urUSMPoolCreate; - pDdiTable->pfnPoolRetain = urUSMPoolRetain; - pDdiTable->pfnPoolRelease = urUSMPoolRelease; - pDdiTable->pfnPoolGetInfo = urUSMPoolGetInfo; - pDdiTable->pfnSharedAlloc = urUSMSharedAlloc; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreate = ur::opencl::urKernelCreate; + pDdiTable->pfnGetInfo = ur::opencl::urKernelGetInfo; + pDdiTable->pfnGetGroupInfo = ur::opencl::urKernelGetGroupInfo; + pDdiTable->pfnGetSubGroupInfo = ur::opencl::urKernelGetSubGroupInfo; + pDdiTable->pfnRetain = ur::opencl::urKernelRetain; + pDdiTable->pfnRelease = ur::opencl::urKernelRelease; + pDdiTable->pfnGetNativeHandle = ur::opencl::urKernelGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urKernelCreateWithNativeHandle; + pDdiTable->pfnGetSuggestedLocalWorkSize = + ur::opencl::urKernelGetSuggestedLocalWorkSize; + pDdiTable->pfnGetSuggestedLocalWorkSizeWithArgs = + ur::opencl::urKernelGetSuggestedLocalWorkSizeWithArgs; + pDdiTable->pfnSetExecInfo = ur::opencl::urKernelSetExecInfo; + pDdiTable->pfnSetSpecializationConstants = + ur::opencl::urKernelSetSpecializationConstants; + pDdiTable->pfnSuggestMaxCooperativeGroupCount = + ur::opencl::urKernelSuggestMaxCooperativeGroupCount; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( - ur_api_version_t Version, ur_usm_exp_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL +urGetMemProcAddrTable(ur_api_version_t version, ur_mem_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnImportExp = urUSMImportExp; - pDdiTable->pfnReleaseExp = urUSMReleaseExp; - pDdiTable->pfnContextMemcpyExp = urUSMContextMemcpyExp; - return UR_RESULT_SUCCESS; + pDdiTable->pfnImageCreate = ur::opencl::urMemImageCreate; + pDdiTable->pfnBufferCreate = ur::opencl::urMemBufferCreate; + pDdiTable->pfnRetain = ur::opencl::urMemRetain; + pDdiTable->pfnRelease = ur::opencl::urMemRelease; + pDdiTable->pfnBufferPartition = ur::opencl::urMemBufferPartition; + pDdiTable->pfnGetNativeHandle = ur::opencl::urMemGetNativeHandle; + pDdiTable->pfnBufferCreateWithNativeHandle = + ur::opencl::urMemBufferCreateWithNativeHandle; + pDdiTable->pfnImageCreateWithNativeHandle = + ur::opencl::urMemImageCreateWithNativeHandle; + pDdiTable->pfnGetInfo = ur::opencl::urMemGetInfo; + pDdiTable->pfnImageGetInfo = ur::opencl::urMemImageGetInfo; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetDeviceProcAddrTable( - ur_api_version_t Version, ur_device_dditable_t *pDdiTable) { - auto Result = validateProcInputs(Version, pDdiTable); - if (UR_RESULT_SUCCESS != Result) { - return Result; +UR_APIEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreateWithNativeHandle = urDeviceCreateWithNativeHandle; - pDdiTable->pfnGet = urDeviceGet; - pDdiTable->pfnGetGlobalTimestamps = urDeviceGetGlobalTimestamps; - pDdiTable->pfnGetInfo = urDeviceGetInfo; - pDdiTable->pfnGetNativeHandle = urDeviceGetNativeHandle; - pDdiTable->pfnPartition = urDevicePartition; - pDdiTable->pfnRelease = urDeviceRelease; - pDdiTable->pfnRetain = urDeviceRetain; - pDdiTable->pfnSelectBinary = urDeviceSelectBinary; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnAllocExportableMemoryExp = + ur::opencl::urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur::opencl::urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = + ur::opencl::urMemoryExportExportMemoryHandleExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetDeviceExpProcAddrTable( - ur_api_version_t version, ur_device_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( + ur_api_version_t version, ur_physical_mem_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnWaitExp = urDeviceWaitExp; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnCreate = ur::opencl::urPhysicalMemCreate; + pDdiTable->pfnRetain = ur::opencl::urPhysicalMemRetain; + pDdiTable->pfnRelease = ur::opencl::urPhysicalMemRelease; + pDdiTable->pfnGetInfo = ur::opencl::urPhysicalMemGetInfo; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetCommandBufferExpProcAddrTable( - ur_api_version_t version, ur_command_buffer_exp_dditable_t *pDdiTable) { - auto retVal = validateProcInputs(version, pDdiTable); - if (UR_RESULT_SUCCESS != retVal) { - return retVal; +UR_APIEXPORT ur_result_t UR_APICALL urGetPlatformProcAddrTable( + ur_api_version_t version, ur_platform_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreateExp = urCommandBufferCreateExp; - pDdiTable->pfnRetainExp = urCommandBufferRetainExp; - pDdiTable->pfnReleaseExp = urCommandBufferReleaseExp; - pDdiTable->pfnFinalizeExp = urCommandBufferFinalizeExp; - pDdiTable->pfnAppendKernelLaunchExp = urCommandBufferAppendKernelLaunchExp; - pDdiTable->pfnAppendKernelLaunchWithArgsExp = - urCommandBufferAppendKernelLaunchWithArgsExp; - pDdiTable->pfnAppendUSMMemcpyExp = urCommandBufferAppendUSMMemcpyExp; - pDdiTable->pfnAppendUSMFillExp = urCommandBufferAppendUSMFillExp; - pDdiTable->pfnAppendMemBufferCopyExp = urCommandBufferAppendMemBufferCopyExp; - pDdiTable->pfnAppendMemBufferCopyRectExp = - urCommandBufferAppendMemBufferCopyRectExp; - pDdiTable->pfnAppendMemBufferReadExp = urCommandBufferAppendMemBufferReadExp; - pDdiTable->pfnAppendMemBufferReadRectExp = - urCommandBufferAppendMemBufferReadRectExp; - pDdiTable->pfnAppendMemBufferWriteExp = - urCommandBufferAppendMemBufferWriteExp; - pDdiTable->pfnAppendMemBufferWriteRectExp = - urCommandBufferAppendMemBufferWriteRectExp; - pDdiTable->pfnAppendUSMPrefetchExp = urCommandBufferAppendUSMPrefetchExp; - pDdiTable->pfnAppendUSMAdviseExp = urCommandBufferAppendUSMAdviseExp; - pDdiTable->pfnAppendMemBufferFillExp = urCommandBufferAppendMemBufferFillExp; - pDdiTable->pfnUpdateKernelLaunchExp = urCommandBufferUpdateKernelLaunchExp; - pDdiTable->pfnGetInfoExp = urCommandBufferGetInfoExp; - pDdiTable->pfnUpdateWaitEventsExp = urCommandBufferUpdateWaitEventsExp; - pDdiTable->pfnUpdateSignalEventExp = urCommandBufferUpdateSignalEventExp; - pDdiTable->pfnAppendNativeCommandExp = urCommandBufferAppendNativeCommandExp; - pDdiTable->pfnGetNativeHandleExp = urCommandBufferGetNativeHandleExp; - - return retVal; + + pDdiTable->pfnGet = ur::opencl::urPlatformGet; + pDdiTable->pfnGetInfo = ur::opencl::urPlatformGetInfo; + pDdiTable->pfnGetNativeHandle = ur::opencl::urPlatformGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urPlatformCreateWithNativeHandle; + pDdiTable->pfnGetApiVersion = ur::opencl::urPlatformGetApiVersion; + pDdiTable->pfnGetBackendOption = ur::opencl::urPlatformGetBackendOption; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetUsmP2PExpProcAddrTable( - ur_api_version_t version, ur_usm_p2p_exp_dditable_t *pDdiTable) { - auto retVal = validateProcInputs(version, pDdiTable); - if (UR_RESULT_SUCCESS != retVal) { - return retVal; +UR_APIEXPORT ur_result_t UR_APICALL urGetProgramProcAddrTable( + ur_api_version_t version, ur_program_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnEnablePeerAccessExp = urUsmP2PEnablePeerAccessExp; - pDdiTable->pfnDisablePeerAccessExp = urUsmP2PDisablePeerAccessExp; - pDdiTable->pfnPeerAccessGetInfoExp = urUsmP2PPeerAccessGetInfoExp; - return retVal; + pDdiTable->pfnCreateWithIL = ur::opencl::urProgramCreateWithIL; + pDdiTable->pfnCreateWithBinary = ur::opencl::urProgramCreateWithBinary; + pDdiTable->pfnBuild = ur::opencl::urProgramBuild; + pDdiTable->pfnCompile = ur::opencl::urProgramCompile; + pDdiTable->pfnLink = ur::opencl::urProgramLink; + pDdiTable->pfnRetain = ur::opencl::urProgramRetain; + pDdiTable->pfnRelease = ur::opencl::urProgramRelease; + pDdiTable->pfnGetFunctionPointer = ur::opencl::urProgramGetFunctionPointer; + pDdiTable->pfnGetGlobalVariablePointer = + ur::opencl::urProgramGetGlobalVariablePointer; + pDdiTable->pfnGetInfo = ur::opencl::urProgramGetInfo; + pDdiTable->pfnGetBuildInfo = ur::opencl::urProgramGetBuildInfo; + pDdiTable->pfnSetSpecializationConstants = + ur::opencl::urProgramSetSpecializationConstants; + pDdiTable->pfnGetNativeHandle = ur::opencl::urProgramGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urProgramCreateWithNativeHandle; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( - ur_api_version_t version, ur_bindless_images_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetProgramExpProcAddrTable( + ur_api_version_t version, ur_program_exp_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnUnsampledImageHandleDestroyExp = - urBindlessImagesUnsampledImageHandleDestroyExp; - pDdiTable->pfnSampledImageHandleDestroyExp = - urBindlessImagesSampledImageHandleDestroyExp; - pDdiTable->pfnImageAllocateExp = urBindlessImagesImageAllocateExp; - pDdiTable->pfnImageFreeExp = urBindlessImagesImageFreeExp; - pDdiTable->pfnUnsampledImageCreateExp = - urBindlessImagesUnsampledImageCreateExp; - pDdiTable->pfnSampledImageCreateExp = urBindlessImagesSampledImageCreateExp; - pDdiTable->pfnImageCopyExp = urBindlessImagesImageCopyExp; - pDdiTable->pfnImageGetInfoExp = urBindlessImagesImageGetInfoExp; - pDdiTable->pfnMipmapGetLevelExp = urBindlessImagesMipmapGetLevelExp; - pDdiTable->pfnMipmapFreeExp = urBindlessImagesMipmapFreeExp; - pDdiTable->pfnImportExternalMemoryExp = - urBindlessImagesImportExternalMemoryExp; - pDdiTable->pfnMapExternalArrayExp = urBindlessImagesMapExternalArrayExp; - pDdiTable->pfnMapExternalLinearMemoryExp = - urBindlessImagesMapExternalLinearMemoryExp; - pDdiTable->pfnReleaseExternalMemoryExp = - urBindlessImagesReleaseExternalMemoryExp; - pDdiTable->pfnFreeMappedLinearMemoryExp = - urBindlessImagesFreeMappedLinearMemoryExp; - pDdiTable->pfnImportExternalSemaphoreExp = - urBindlessImagesImportExternalSemaphoreExp; - pDdiTable->pfnReleaseExternalSemaphoreExp = - urBindlessImagesReleaseExternalSemaphoreExp; - pDdiTable->pfnWaitExternalSemaphoreExp = - urBindlessImagesWaitExternalSemaphoreExp; - pDdiTable->pfnSignalExternalSemaphoreExp = - urBindlessImagesSignalExternalSemaphoreExp; - pDdiTable->pfnGetImageMemoryHandleTypeSupportExp = - urBindlessImagesGetImageMemoryHandleTypeSupportExp; - pDdiTable->pfnGetImageUnsampledHandleSupportExp = - urBindlessImagesGetImageUnsampledHandleSupportExp; - pDdiTable->pfnGetImageSampledHandleSupportExp = - urBindlessImagesGetImageSampledHandleSupportExp; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnDynamicLinkExp = ur::opencl::urProgramDynamicLinkExp; + pDdiTable->pfnBuildExp = ur::opencl::urProgramBuildExp; + pDdiTable->pfnCompileExp = ur::opencl::urProgramCompileExp; + pDdiTable->pfnLinkExp = ur::opencl::urProgramLinkExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( - ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetQueueProcAddrTable( + ur_api_version_t version, ur_queue_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnAllocExportableMemoryExp = - urMemoryExportAllocExportableMemoryExp; - pDdiTable->pfnFreeExportableMemoryExp = urMemoryExportFreeExportableMemoryExp; - pDdiTable->pfnExportMemoryHandleExp = urMemoryExportExportMemoryHandleExp; - return UR_RESULT_SUCCESS; + + pDdiTable->pfnGetInfo = ur::opencl::urQueueGetInfo; + pDdiTable->pfnCreate = ur::opencl::urQueueCreate; + pDdiTable->pfnRetain = ur::opencl::urQueueRetain; + pDdiTable->pfnRelease = ur::opencl::urQueueRelease; + pDdiTable->pfnGetNativeHandle = ur::opencl::urQueueGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urQueueCreateWithNativeHandle; + pDdiTable->pfnFinish = ur::opencl::urQueueFinish; + pDdiTable->pfnFlush = ur::opencl::urQueueFlush; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( - ur_api_version_t version, ur_virtual_mem_dditable_t *pDdiTable) { - auto retVal = validateProcInputs(version, pDdiTable); - if (UR_RESULT_SUCCESS != retVal) { - return retVal; +UR_APIEXPORT ur_result_t UR_APICALL urGetQueueExpProcAddrTable( + ur_api_version_t version, ur_queue_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnFree = urVirtualMemFree; - pDdiTable->pfnGetInfo = urVirtualMemGetInfo; - pDdiTable->pfnGranularityGetInfo = urVirtualMemGranularityGetInfo; - pDdiTable->pfnMap = urVirtualMemMap; - pDdiTable->pfnReserve = urVirtualMemReserve; - pDdiTable->pfnSetAccess = urVirtualMemSetAccess; - pDdiTable->pfnUnmap = urVirtualMemUnmap; + pDdiTable->pfnBeginGraphCaptureExp = ur::opencl::urQueueBeginGraphCaptureExp; + pDdiTable->pfnBeginCaptureIntoGraphExp = + ur::opencl::urQueueBeginCaptureIntoGraphExp; + pDdiTable->pfnEndGraphCaptureExp = ur::opencl::urQueueEndGraphCaptureExp; + pDdiTable->pfnIsGraphCaptureEnabledExp = + ur::opencl::urQueueIsGraphCaptureEnabledExp; + pDdiTable->pfnGetGraphExp = ur::opencl::urQueueGetGraphExp; - return retVal; + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( - ur_api_version_t version, ur_physical_mem_dditable_t *pDdiTable) { - auto retVal = validateProcInputs(version, pDdiTable); - if (UR_RESULT_SUCCESS != retVal) { - return retVal; +UR_APIEXPORT ur_result_t UR_APICALL urGetSamplerProcAddrTable( + ur_api_version_t version, ur_sampler_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; } - pDdiTable->pfnCreate = urPhysicalMemCreate; - pDdiTable->pfnRelease = urPhysicalMemRelease; - pDdiTable->pfnRetain = urPhysicalMemRetain; - pDdiTable->pfnGetInfo = urPhysicalMemGetInfo; + pDdiTable->pfnCreate = ur::opencl::urSamplerCreate; + pDdiTable->pfnRetain = ur::opencl::urSamplerRetain; + pDdiTable->pfnRelease = ur::opencl::urSamplerRelease; + pDdiTable->pfnGetInfo = ur::opencl::urSamplerGetInfo; + pDdiTable->pfnGetNativeHandle = ur::opencl::urSamplerGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urSamplerCreateWithNativeHandle; - return retVal; + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( - ur_api_version_t version, ur_enqueue_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL +urGetUSMProcAddrTable(ur_api_version_t version, ur_usm_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnTimestampRecordingExp = urEnqueueTimestampRecordingExp; - pDdiTable->pfnNativeCommandExp = urEnqueueNativeCommandExp; - pDdiTable->pfnCommandBufferExp = urEnqueueCommandBufferExp; - pDdiTable->pfnKernelLaunchWithArgsExp = urEnqueueKernelLaunchWithArgsExp; - pDdiTable->pfnGraphExp = urEnqueueGraphExp; - - return UR_RESULT_SUCCESS; + pDdiTable->pfnHostAlloc = ur::opencl::urUSMHostAlloc; + pDdiTable->pfnDeviceAlloc = ur::opencl::urUSMDeviceAlloc; + pDdiTable->pfnSharedAlloc = ur::opencl::urUSMSharedAlloc; + pDdiTable->pfnFree = ur::opencl::urUSMFree; + pDdiTable->pfnGetMemAllocInfo = ur::opencl::urUSMGetMemAllocInfo; + pDdiTable->pfnPoolCreate = ur::opencl::urUSMPoolCreate; + pDdiTable->pfnPoolRetain = ur::opencl::urUSMPoolRetain; + pDdiTable->pfnPoolRelease = ur::opencl::urUSMPoolRelease; + pDdiTable->pfnPoolGetInfo = ur::opencl::urUSMPoolGetInfo; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( - ur_api_version_t version, ur_ipc_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( + ur_api_version_t version, ur_usm_exp_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnGetMemHandleExp = urIPCGetMemHandleExp; - pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; - pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; - pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; - pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; - pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; - pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; - pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; - - return UR_RESULT_SUCCESS; + pDdiTable->pfnPoolCreateExp = ur::opencl::urUSMPoolCreateExp; + pDdiTable->pfnPoolDestroyExp = ur::opencl::urUSMPoolDestroyExp; + pDdiTable->pfnPoolGetDefaultDevicePoolExp = + ur::opencl::urUSMPoolGetDefaultDevicePoolExp; + pDdiTable->pfnPoolGetInfoExp = ur::opencl::urUSMPoolGetInfoExp; + pDdiTable->pfnPoolSetInfoExp = ur::opencl::urUSMPoolSetInfoExp; + pDdiTable->pfnPoolSetDevicePoolExp = ur::opencl::urUSMPoolSetDevicePoolExp; + pDdiTable->pfnPoolGetDevicePoolExp = ur::opencl::urUSMPoolGetDevicePoolExp; + pDdiTable->pfnPoolTrimToExp = ur::opencl::urUSMPoolTrimToExp; + pDdiTable->pfnPitchedAllocExp = ur::opencl::urUSMPitchedAllocExp; + pDdiTable->pfnContextMemcpyExp = ur::opencl::urUSMContextMemcpyExp; + pDdiTable->pfnHostAllocUnregisterExp = + ur::opencl::urUSMHostAllocUnregisterExp; + pDdiTable->pfnHostAllocRegisterExp = ur::opencl::urUSMHostAllocRegisterExp; + pDdiTable->pfnImportExp = ur::opencl::urUSMImportExp; + pDdiTable->pfnReleaseExp = ur::opencl::urUSMReleaseExp; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urGetProgramExpProcAddrTable( - ur_api_version_t version, ur_program_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetUsmP2PExpProcAddrTable( + ur_api_version_t version, ur_usm_p2p_exp_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnBuildExp = urProgramBuildExp; - pDdiTable->pfnCompileExp = urProgramCompileExp; - pDdiTable->pfnLinkExp = urProgramLinkExp; - pDdiTable->pfnDynamicLinkExp = urProgramDynamicLinkExp; + pDdiTable->pfnEnablePeerAccessExp = ur::opencl::urUsmP2PEnablePeerAccessExp; + pDdiTable->pfnDisablePeerAccessExp = ur::opencl::urUsmP2PDisablePeerAccessExp; + pDdiTable->pfnPeerAccessGetInfoExp = ur::opencl::urUsmP2PPeerAccessGetInfoExp; - return UR_RESULT_SUCCESS; + return result; } -UR_APIEXPORT ur_result_t UR_APICALL urGetQueueExpProcAddrTable( - ur_api_version_t version, ur_queue_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( + ur_api_version_t version, ur_virtual_mem_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnBeginGraphCaptureExp = urQueueBeginGraphCaptureExp; - pDdiTable->pfnBeginCaptureIntoGraphExp = urQueueBeginCaptureIntoGraphExp; - pDdiTable->pfnEndGraphCaptureExp = urQueueEndGraphCaptureExp; - pDdiTable->pfnIsGraphCaptureEnabledExp = urQueueIsGraphCaptureEnabledExp; - pDdiTable->pfnGetGraphExp = urQueueGetGraphExp; + pDdiTable->pfnGranularityGetInfo = ur::opencl::urVirtualMemGranularityGetInfo; + pDdiTable->pfnReserve = ur::opencl::urVirtualMemReserve; + pDdiTable->pfnFree = ur::opencl::urVirtualMemFree; + pDdiTable->pfnMap = ur::opencl::urVirtualMemMap; + pDdiTable->pfnUnmap = ur::opencl::urVirtualMemUnmap; + pDdiTable->pfnSetAccess = ur::opencl::urVirtualMemSetAccess; + pDdiTable->pfnGetInfo = ur::opencl::urVirtualMemGetInfo; - return UR_RESULT_SUCCESS; + return result; } -UR_APIEXPORT ur_result_t UR_APICALL urGetGraphExpProcAddrTable( - ur_api_version_t version, ur_graph_exp_dditable_t *pDdiTable) { +UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceProcAddrTable( + ur_api_version_t version, ur_device_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); if (UR_RESULT_SUCCESS != result) { return result; } - pDdiTable->pfnCreateExp = urGraphCreateExp; - pDdiTable->pfnInstantiateGraphExp = urGraphInstantiateGraphExp; - pDdiTable->pfnDestroyExp = urGraphDestroyExp; - pDdiTable->pfnExecutableGraphDestroyExp = urGraphExecutableGraphDestroyExp; - pDdiTable->pfnIsEmptyExp = urGraphIsEmptyExp; - pDdiTable->pfnGetIdExp = urGraphGetIdExp; - pDdiTable->pfnSetDestructionCallbackExp = urGraphSetDestructionCallbackExp; - pDdiTable->pfnDumpContentsExp = urGraphDumpContentsExp; - pDdiTable->pfnGetNativeHandleExp = urGraphGetNativeHandleExp; - pDdiTable->pfnExecutableGraphGetNativeHandleExp = - urGraphExecutableGraphGetNativeHandleExp; - return UR_RESULT_SUCCESS; + pDdiTable->pfnGet = ur::opencl::urDeviceGet; + pDdiTable->pfnGetInfo = ur::opencl::urDeviceGetInfo; + pDdiTable->pfnRetain = ur::opencl::urDeviceRetain; + pDdiTable->pfnRelease = ur::opencl::urDeviceRelease; + pDdiTable->pfnPartition = ur::opencl::urDevicePartition; + pDdiTable->pfnSelectBinary = ur::opencl::urDeviceSelectBinary; + pDdiTable->pfnGetNativeHandle = ur::opencl::urDeviceGetNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur::opencl::urDeviceCreateWithNativeHandle; + pDdiTable->pfnGetGlobalTimestamps = ur::opencl::urDeviceGetGlobalTimestamps; + + return result; } -UR_DLLEXPORT ur_result_t UR_APICALL urAllAddrTable(ur_api_version_t version, - ur_dditable_t *pDdiTable) { - urGetAdapterProcAddrTable(version, &pDdiTable->Adapter); - urGetBindlessImagesExpProcAddrTable(version, &pDdiTable->BindlessImagesExp); - urGetCommandBufferExpProcAddrTable(version, &pDdiTable->CommandBufferExp); - urGetContextProcAddrTable(version, &pDdiTable->Context); - urGetEnqueueProcAddrTable(version, &pDdiTable->Enqueue); - urGetEnqueueExpProcAddrTable(version, &pDdiTable->EnqueueExp); - urGetIPCExpProcAddrTable(version, &pDdiTable->IPCExp); - urGetEventProcAddrTable(version, &pDdiTable->Event); - urGetEventExpProcAddrTable(version, &pDdiTable->EventExp); - urGetGraphExpProcAddrTable(version, &pDdiTable->GraphExp); - urGetKernelProcAddrTable(version, &pDdiTable->Kernel); - urGetMemProcAddrTable(version, &pDdiTable->Mem); - urGetPhysicalMemProcAddrTable(version, &pDdiTable->PhysicalMem); - urGetPlatformProcAddrTable(version, &pDdiTable->Platform); - urGetProgramProcAddrTable(version, &pDdiTable->Program); - urGetProgramExpProcAddrTable(version, &pDdiTable->ProgramExp); - urGetQueueProcAddrTable(version, &pDdiTable->Queue); - urGetQueueExpProcAddrTable(version, &pDdiTable->QueueExp); - urGetSamplerProcAddrTable(version, &pDdiTable->Sampler); - urGetUSMProcAddrTable(version, &pDdiTable->USM); - urGetUSMExpProcAddrTable(version, &pDdiTable->USMExp); - urGetUsmP2PExpProcAddrTable(version, &pDdiTable->UsmP2PExp); - urGetVirtualMemProcAddrTable(version, &pDdiTable->VirtualMem); - urGetDeviceProcAddrTable(version, &pDdiTable->Device); - urGetDeviceExpProcAddrTable(version, &pDdiTable->DeviceExp); - urGetMemoryExportExpProcAddrTable(version, &pDdiTable->MemoryExportExp); +UR_APIEXPORT ur_result_t UR_APICALL urGetDeviceExpProcAddrTable( + ur_api_version_t version, ur_device_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnWaitExp = ur::opencl::urDeviceWaitExp; - return UR_RESULT_SUCCESS; + return result; } +#ifdef UR_STATIC_ADAPTER_OPENCL +} // namespace ur::opencl +#else } // extern "C" +#endif + +namespace { +ur_result_t populateDdiTable(ur_dditable_t *ddi) { + if (ddi == nullptr) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + ur_result_t result; + +#ifdef UR_STATIC_ADAPTER_OPENCL +#define NAMESPACE_ ::ur::opencl +#else +#define NAMESPACE_ +#endif + + result = NAMESPACE_::urGetAdapterProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Adapter); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetBindlessImagesExpProcAddrTable( + UR_API_VERSION_CURRENT, &ddi->BindlessImagesExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetCommandBufferExpProcAddrTable( + UR_API_VERSION_CURRENT, &ddi->CommandBufferExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetContextProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Context); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEnqueueProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Enqueue); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEnqueueExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->EnqueueExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = + NAMESPACE_::urGetEventProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Event); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetEventExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->EventExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetGraphExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->GraphExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetIPCExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->IPCExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetKernelProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Kernel); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetMemProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Mem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetMemoryExportExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->MemoryExportExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetPhysicalMemProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->PhysicalMem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetPlatformProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Platform); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetProgramProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Program); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetProgramExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->ProgramExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = + NAMESPACE_::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Queue); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetQueueExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->QueueExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetSamplerProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Sampler); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, &ddi->USM); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUSMExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->USMExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetUsmP2PExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->UsmP2PExp); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetVirtualMemProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->VirtualMem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetDeviceProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->Device); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetDeviceExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->DeviceExp); + if (result != UR_RESULT_SUCCESS) + return result; + +#undef NAMESPACE_ -const ur_dditable_t *ur::opencl::ddi_getter::value() { + return result; +} +} // namespace + +namespace ur::opencl { +const ur_dditable_t *ddi_getter::value() { static std::once_flag flag; static ur_dditable_t table; - std::call_once(flag, - []() { urAllAddrTable(UR_API_VERSION_CURRENT, &table); }); + std::call_once(flag, []() { populateDdiTable(&table); }); return &table; } + +#ifdef UR_STATIC_ADAPTER_OPENCL +ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) { + return populateDdiTable(ddi); +} +#endif +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/ur_interface_loader.hpp b/unified-runtime/source/adapters/opencl/ur_interface_loader.hpp new file mode 100644 index 0000000000000..ffeaa3c4664f7 --- /dev/null +++ b/unified-runtime/source/adapters/opencl/ur_interface_loader.hpp @@ -0,0 +1,912 @@ +//===--------- ur_interface_loader.hpp - Level Zero Adapter ------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include + +namespace ur::opencl { +ur_result_t urAdapterGet(uint32_t NumEntries, ur_adapter_handle_t *phAdapters, + uint32_t *pNumAdapters); +ur_result_t urAdapterRelease(ur_adapter_handle_t hAdapter); +ur_result_t urAdapterRetain(ur_adapter_handle_t hAdapter); +ur_result_t urAdapterGetLastError(ur_adapter_handle_t hAdapter, + const char **ppMessage, int32_t *pError); +ur_result_t urAdapterGetInfo(ur_adapter_handle_t hAdapter, + ur_adapter_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urAdapterSetLoggerCallback(ur_adapter_handle_t hAdapter, + ur_logger_callback_t pfnLoggerCallback, + void *pUserData, + ur_logger_level_t level); +ur_result_t urAdapterSetLoggerCallbackLevel(ur_adapter_handle_t hAdapter, + ur_logger_level_t level); +ur_result_t urPlatformGet(ur_adapter_handle_t hAdapter, uint32_t NumEntries, + ur_platform_handle_t *phPlatforms, + uint32_t *pNumPlatforms); +ur_result_t urPlatformGetInfo(ur_platform_handle_t hPlatform, + ur_platform_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urPlatformGetApiVersion(ur_platform_handle_t hPlatform, + ur_api_version_t *pVersion); +ur_result_t urPlatformGetNativeHandle(ur_platform_handle_t hPlatform, + ur_native_handle_t *phNativePlatform); +ur_result_t urPlatformCreateWithNativeHandle( + ur_native_handle_t hNativePlatform, ur_adapter_handle_t hAdapter, + const ur_platform_native_properties_t *pProperties, + ur_platform_handle_t *phPlatform); +ur_result_t urPlatformGetBackendOption(ur_platform_handle_t hPlatform, + const char *pFrontendOption, + const char **ppPlatformOption); +ur_result_t urDeviceGet(ur_platform_handle_t hPlatform, + ur_device_type_t DeviceType, uint32_t NumEntries, + ur_device_handle_t *phDevices, uint32_t *pNumDevices); +ur_result_t urDeviceGetInfo(ur_device_handle_t hDevice, + ur_device_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urDeviceRetain(ur_device_handle_t hDevice); +ur_result_t urDeviceRelease(ur_device_handle_t hDevice); +ur_result_t +urDevicePartition(ur_device_handle_t hDevice, + const ur_device_partition_properties_t *pProperties, + uint32_t NumDevices, ur_device_handle_t *phSubDevices, + uint32_t *pNumDevicesRet); +ur_result_t urDeviceSelectBinary(ur_device_handle_t hDevice, + const ur_device_binary_t *pBinaries, + uint32_t NumBinaries, + uint32_t *pSelectedBinary); +ur_result_t urDeviceGetNativeHandle(ur_device_handle_t hDevice, + ur_native_handle_t *phNativeDevice); +ur_result_t +urDeviceCreateWithNativeHandle(ur_native_handle_t hNativeDevice, + ur_adapter_handle_t hAdapter, + const ur_device_native_properties_t *pProperties, + ur_device_handle_t *phDevice); +ur_result_t urDeviceGetGlobalTimestamps(ur_device_handle_t hDevice, + uint64_t *pDeviceTimestamp, + uint64_t *pHostTimestamp); +ur_result_t urContextCreate(uint32_t DeviceCount, + const ur_device_handle_t *phDevices, + const ur_context_properties_t *pProperties, + ur_context_handle_t *phContext); +ur_result_t urContextRetain(ur_context_handle_t hContext); +ur_result_t urContextRelease(ur_context_handle_t hContext); +ur_result_t urContextGetInfo(ur_context_handle_t hContext, + ur_context_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urContextGetNativeHandle(ur_context_handle_t hContext, + ur_native_handle_t *phNativeContext); +ur_result_t urContextCreateWithNativeHandle( + ur_native_handle_t hNativeContext, ur_adapter_handle_t hAdapter, + uint32_t numDevices, const ur_device_handle_t *phDevices, + const ur_context_native_properties_t *pProperties, + ur_context_handle_t *phContext); +ur_result_t +urContextSetExtendedDeleter(ur_context_handle_t hContext, + ur_context_extended_deleter_t pfnDeleter, + void *pUserData); +ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, void *pHost, + ur_mem_handle_t *phMem); +ur_result_t urMemBufferCreate(ur_context_handle_t hContext, + ur_mem_flags_t flags, size_t size, + const ur_buffer_properties_t *pProperties, + ur_mem_handle_t *phBuffer); +ur_result_t urMemRetain(ur_mem_handle_t hMem); +ur_result_t urMemRelease(ur_mem_handle_t hMem); +ur_result_t urMemBufferPartition(ur_mem_handle_t hBuffer, ur_mem_flags_t flags, + ur_buffer_create_type_t bufferCreateType, + const ur_buffer_region_t *pRegion, + ur_mem_handle_t *phMem); +ur_result_t urMemGetNativeHandle(ur_mem_handle_t hMem, + ur_device_handle_t hDevice, + ur_native_handle_t *phNativeMem); +ur_result_t urMemBufferCreateWithNativeHandle( + ur_native_handle_t hNativeMem, ur_context_handle_t hContext, + const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem); +ur_result_t urMemImageCreateWithNativeHandle( + ur_native_handle_t hNativeMem, ur_context_handle_t hContext, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem); +ur_result_t urMemGetInfo(ur_mem_handle_t hMemory, ur_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urMemImageGetInfo(ur_mem_handle_t hMemory, ur_image_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urSamplerCreate(ur_context_handle_t hContext, + const ur_sampler_desc_t *pDesc, + ur_sampler_handle_t *phSampler); +ur_result_t urSamplerRetain(ur_sampler_handle_t hSampler); +ur_result_t urSamplerRelease(ur_sampler_handle_t hSampler); +ur_result_t urSamplerGetInfo(ur_sampler_handle_t hSampler, + ur_sampler_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urSamplerGetNativeHandle(ur_sampler_handle_t hSampler, + ur_native_handle_t *phNativeSampler); +ur_result_t urSamplerCreateWithNativeHandle( + ur_native_handle_t hNativeSampler, ur_context_handle_t hContext, + const ur_sampler_native_properties_t *pProperties, + ur_sampler_handle_t *phSampler); +ur_result_t urUSMHostAlloc(ur_context_handle_t hContext, + const ur_usm_desc_t *pUSMDesc, + ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMDeviceAlloc(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMSharedAlloc(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ur_usm_pool_handle_t pool, size_t size, + void **ppMem); +ur_result_t urUSMFree(ur_context_handle_t hContext, void *pMem); +ur_result_t urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem, + ur_usm_alloc_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urUSMPoolCreate(ur_context_handle_t hContext, + ur_usm_pool_desc_t *pPoolDesc, + ur_usm_pool_handle_t *ppPool); +ur_result_t urUSMPoolRetain(ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolRelease(ur_usm_pool_handle_t pPool); +ur_result_t urUSMPoolGetInfo(ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urVirtualMemGranularityGetInfo( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + size_t allocationSize, ur_virtual_mem_granularity_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet); +ur_result_t urVirtualMemReserve(ur_context_handle_t hContext, + const void *pStart, size_t size, + void **ppStart); +ur_result_t urVirtualMemFree(ur_context_handle_t hContext, const void *pStart, + size_t size); +ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, + size_t size, ur_physical_mem_handle_t hPhysicalMem, + size_t offset, ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemUnmap(ur_context_handle_t hContext, const void *pStart, + size_t size); +ur_result_t urVirtualMemSetAccess(ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_access_flags_t flags); +ur_result_t urVirtualMemGetInfo(ur_context_handle_t hContext, + const void *pStart, size_t size, + ur_virtual_mem_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, + ur_device_handle_t hDevice, size_t size, + const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem); +ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem); +ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, + ur_physical_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urProgramCreateWithIL(ur_context_handle_t hContext, const void *pIL, + size_t length, + const ur_program_properties_t *pProperties, + ur_program_handle_t *phProgram); +ur_result_t urProgramCreateWithBinary( + ur_context_handle_t hContext, uint32_t numDevices, + ur_device_handle_t *phDevices, size_t *pLengths, const uint8_t **ppBinaries, + const ur_program_properties_t *pProperties, ur_program_handle_t *phProgram); +ur_result_t urProgramBuild(ur_context_handle_t hContext, + ur_program_handle_t hProgram, const char *pOptions); +ur_result_t urProgramCompile(ur_context_handle_t hContext, + ur_program_handle_t hProgram, + const char *pOptions); +ur_result_t urProgramLink(ur_context_handle_t hContext, uint32_t count, + const ur_program_handle_t *phPrograms, + const char *pOptions, ur_program_handle_t *phProgram); +ur_result_t urProgramRetain(ur_program_handle_t hProgram); +ur_result_t urProgramRelease(ur_program_handle_t hProgram); +ur_result_t urProgramGetFunctionPointer(ur_device_handle_t hDevice, + ur_program_handle_t hProgram, + const char *pFunctionName, + void **ppFunctionPointer); +ur_result_t urProgramGetGlobalVariablePointer( + ur_device_handle_t hDevice, ur_program_handle_t hProgram, + const char *pGlobalVariableName, size_t *pGlobalVariableSizeRet, + void **ppGlobalVariablePointerRet); +ur_result_t urProgramGetInfo(ur_program_handle_t hProgram, + ur_program_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urProgramGetBuildInfo(ur_program_handle_t hProgram, + ur_device_handle_t hDevice, + ur_program_build_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urProgramSetSpecializationConstants( + ur_program_handle_t hProgram, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants); +ur_result_t urProgramGetNativeHandle(ur_program_handle_t hProgram, + ur_native_handle_t *phNativeProgram); +ur_result_t urProgramCreateWithNativeHandle( + ur_native_handle_t hNativeProgram, ur_context_handle_t hContext, + const ur_program_native_properties_t *pProperties, + ur_program_handle_t *phProgram); +ur_result_t urQueueGetInfo(ur_queue_handle_t hQueue, ur_queue_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urQueueCreate(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_queue_properties_t *pProperties, + ur_queue_handle_t *phQueue); +ur_result_t urQueueRetain(ur_queue_handle_t hQueue); +ur_result_t urQueueRelease(ur_queue_handle_t hQueue); +ur_result_t urQueueGetNativeHandle(ur_queue_handle_t hQueue, + ur_queue_native_desc_t *pDesc, + ur_native_handle_t *phNativeQueue); +ur_result_t urQueueCreateWithNativeHandle( + ur_native_handle_t hNativeQueue, ur_context_handle_t hContext, + ur_device_handle_t hDevice, const ur_queue_native_properties_t *pProperties, + ur_queue_handle_t *phQueue); +ur_result_t urQueueFinish(ur_queue_handle_t hQueue); +ur_result_t urQueueFlush(ur_queue_handle_t hQueue); +ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urEventGetProfilingInfo(ur_event_handle_t hEvent, + ur_profiling_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urEventWait(uint32_t numEvents, + const ur_event_handle_t *phEventWaitList); +ur_result_t urEventRetain(ur_event_handle_t hEvent); +ur_result_t urEventRelease(ur_event_handle_t hEvent); +ur_result_t urEventGetNativeHandle(ur_event_handle_t hEvent, + ur_native_handle_t *phNativeEvent); +ur_result_t +urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent, + ur_context_handle_t hContext, + const ur_event_native_properties_t *pProperties, + ur_event_handle_t *phEvent); +ur_result_t urEventSetCallback(ur_event_handle_t hEvent, + ur_execution_info_t execStatus, + ur_event_callback_t pfnNotify, void *pUserData); +ur_result_t urEnqueueEventsWait(ur_queue_handle_t hQueue, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueEventsWaitWithBarrier( + ur_queue_handle_t hQueue, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferRead(ur_queue_handle_t hQueue, + ur_mem_handle_t hBuffer, bool blockingRead, + size_t offset, size_t size, void *pDst, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferWrite( + ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, + size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferReadRect( + ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingRead, + ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pDst, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferWriteRect( + ur_queue_handle_t hQueue, ur_mem_handle_t hBuffer, bool blockingWrite, + ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferCopy(ur_queue_handle_t hQueue, + ur_mem_handle_t hBufferSrc, + ur_mem_handle_t hBufferDst, size_t srcOffset, + size_t dstOffset, size_t size, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferCopyRect( + ur_queue_handle_t hQueue, ur_mem_handle_t hBufferSrc, + ur_mem_handle_t hBufferDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, + size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferFill(ur_queue_handle_t hQueue, + ur_mem_handle_t hBuffer, + const void *pPattern, size_t patternSize, + size_t offset, size_t size, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemImageRead( + ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingRead, + ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemImageWrite( + ur_queue_handle_t hQueue, ur_mem_handle_t hImage, bool blockingWrite, + ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, + size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t +urEnqueueMemImageCopy(ur_queue_handle_t hQueue, ur_mem_handle_t hImageSrc, + ur_mem_handle_t hImageDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueMemBufferMap(ur_queue_handle_t hQueue, + ur_mem_handle_t hBuffer, bool blockingMap, + ur_map_flags_t mapFlags, size_t offset, + size_t size, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent, void **ppRetMap); +ur_result_t urEnqueueMemUnmap(ur_queue_handle_t hQueue, ur_mem_handle_t hMem, + void *pMappedPtr, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill(ur_queue_handle_t hQueue, void *pMem, + size_t patternSize, const void *pPattern, + size_t size, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy(ur_queue_handle_t hQueue, bool blocking, + void *pDst, const void *pSrc, size_t size, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMPrefetch(ur_queue_handle_t hQueue, const void *pMem, + size_t size, ur_usm_migration_flags_t flags, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMAdvise(ur_queue_handle_t hQueue, const void *pMem, + size_t size, ur_usm_advice_flags_t advice, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFill2D(ur_queue_handle_t hQueue, void *pMem, + size_t pitch, size_t patternSize, + const void *pPattern, size_t width, + size_t height, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMMemcpy2D(ur_queue_handle_t hQueue, bool blocking, + void *pDst, size_t dstPitch, const void *pSrc, + size_t srcPitch, size_t width, size_t height, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueDeviceGlobalVariableWrite( + ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, + bool blockingWrite, size_t count, size_t offset, const void *pSrc, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueDeviceGlobalVariableRead( + ur_queue_handle_t hQueue, ur_program_handle_t hProgram, const char *name, + bool blockingRead, size_t count, size_t offset, void *pDst, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueReadHostPipe(ur_queue_handle_t hQueue, + ur_program_handle_t hProgram, + const char *pipe_symbol, bool blocking, + void *pDst, size_t size, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueWriteHostPipe(ur_queue_handle_t hQueue, + ur_program_handle_t hProgram, + const char *pipe_symbol, bool blocking, + void *pSrc, size_t size, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueKernelLaunchWithArgsExp( + ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, + const ur_kernel_launch_ext_properties_t *launchPropList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urKernelCreate(ur_program_handle_t hProgram, + const char *pKernelName, + ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, + ur_kernel_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet); +ur_result_t urKernelGetGroupInfo(ur_kernel_handle_t hKernel, + ur_device_handle_t hDevice, + ur_kernel_group_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urKernelGetSubGroupInfo(ur_kernel_handle_t hKernel, + ur_device_handle_t hDevice, + ur_kernel_sub_group_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urKernelRetain(ur_kernel_handle_t hKernel); +ur_result_t urKernelRelease(ur_kernel_handle_t hKernel); +ur_result_t +urKernelSetExecInfo(ur_kernel_handle_t hKernel, ur_kernel_exec_info_t propName, + size_t propSize, + const ur_kernel_exec_info_properties_t *pProperties, + const void *pPropValue); +ur_result_t urKernelSetSpecializationConstants( + ur_kernel_handle_t hKernel, uint32_t count, + const ur_specialization_constant_info_t *pSpecConstants); +ur_result_t urKernelGetNativeHandle(ur_kernel_handle_t hKernel, + ur_native_handle_t *phNativeKernel); +ur_result_t +urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, + ur_context_handle_t hContext, + ur_program_handle_t hProgram, + const ur_kernel_native_properties_t *pProperties, + ur_kernel_handle_t *phKernel); +ur_result_t urKernelGetSuggestedLocalWorkSize(ur_kernel_handle_t hKernel, + ur_queue_handle_t hQueue, + uint32_t numWorkDim, + const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, + size_t *pSuggestedLocalWorkSize); +ur_result_t urKernelGetSuggestedLocalWorkSizeWithArgs( + ur_kernel_handle_t hKernel, ur_queue_handle_t hQueue, uint32_t numWorkDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, + size_t *pSuggestedLocalWorkSize); +ur_result_t urKernelSuggestMaxCooperativeGroupCount( + ur_kernel_handle_t hKernel, ur_device_handle_t hDevice, uint32_t workDim, + const size_t *pLocalWorkSize, size_t dynamicSharedMemorySize, + uint32_t *pGroupCountRet); +ur_result_t urEnqueueUSMDeviceAllocExp( + ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + void **ppMem, ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMSharedAllocExp( + ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + void **ppMem, ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMHostAllocExp( + ur_queue_handle_t hQueue, ur_usm_pool_handle_t pPool, const size_t size, + const ur_exp_async_usm_alloc_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + void **ppMem, ur_event_handle_t *phEvent); +ur_result_t urEnqueueUSMFreeExp(ur_queue_handle_t hQueue, + ur_usm_pool_handle_t pPool, void *pMem, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urUSMPoolCreateExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_desc_t *pPoolDesc, + ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolDestroyExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDefaultDevicePoolExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolGetInfoExp(ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t urUSMPoolSetInfoExp(ur_usm_pool_handle_t hPool, + ur_usm_pool_info_t propName, void *pPropValue, + size_t propSize); +ur_result_t urUSMPoolSetDevicePoolExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_handle_t hPool); +ur_result_t urUSMPoolGetDevicePoolExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_handle_t *pPool); +ur_result_t urUSMPoolTrimToExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_usm_pool_handle_t hPool, + size_t minBytesToKeep); +ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ur_usm_pool_handle_t pool, size_t widthInBytes, + size_t height, size_t elementSizeBytes, + void **ppMem, size_t *pResultPitch); +ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage); +ur_result_t urBindlessImagesImageAllocateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesImageFreeExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem); +ur_result_t urBindlessImagesUnsampledImageCreateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_native_handle_t *phImage); +ur_result_t urBindlessImagesSampledImageCreateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_sampler_desc_t *pSamplerDesc, + ur_exp_image_native_handle_t *phImage); +ur_result_t urBindlessImagesImageCopyExp( + ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, + ur_exp_image_copy_input_types_t imageCopyInputTypes, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urBindlessImagesImageGetInfoExp( + ur_context_handle_t hContext, ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet); +ur_result_t urBindlessImagesGetImageMemoryHandleTypeSupportExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageUnsampledHandleSupportExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesGetImageSampledHandleSupportExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_desc_t *pImageDesc, const ur_image_format_t *pImageFormat, + ur_exp_image_mem_type_t imageMemHandleType, ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesMipmapGetLevelExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t +urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hMem); +ur_result_t urBindlessImagesImportExternalMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ur_exp_external_mem_handle_t *phExternalMem); +ur_result_t urBindlessImagesMapExternalArrayExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_external_mem_handle_t hExternalMem, + ur_exp_image_mem_native_handle_t *phImageMem); +ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, + uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **ppRetMem); +ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_mem_handle_t hExternalMem); +ur_result_t urBindlessImagesFreeMappedLinearMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, void *pMem); +ur_result_t urBindlessImagesSupportsImportingHandleTypeExp( + ur_device_handle_t hDevice, ur_exp_external_mem_type_t memHandleType, + ur_bool_t *pSupportedRet); +ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ur_exp_external_semaphore_handle_t *phExternalSemaphore); +ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_semaphore_handle_t hExternalSemaphore); +ur_result_t urBindlessImagesWaitExternalSemaphoreExp( + ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + bool hasWaitValue, uint64_t waitValue, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urBindlessImagesSignalExternalSemaphoreExp( + ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, + bool hasSignalValue, uint64_t signalValue, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urDeviceWaitExp(ur_device_handle_t hDevice); +ur_result_t urProgramDynamicLinkExp(ur_context_handle_t hContext, + uint32_t count, + const ur_program_handle_t *phPrograms); +ur_result_t urEnqueueTimestampRecordingExp( + ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urIPCGetMemHandleExp(ur_context_handle_t hContext, void *pMem, + void **ppIPCMemHandleData, + size_t *pIPCMemHandleDataSizeRet); +ur_result_t urIPCPutMemHandleExp(ur_context_handle_t hContext, + void *pIPCMemHandleData); +ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pIPCMemHandleData, + size_t ipcMemHandleDataSize, void **ppMem); +ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet); +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData); +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem); +ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent, + void **ppIPCEventHandleData, + size_t *pIPCEventHandleDataSizeRet); +ur_result_t urIPCPutEventHandleExp(ur_context_handle_t hContext, + void *pIPCEventHandleData); +ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext, + const void *pIPCEventHandleData, + size_t ipcEventHandleDataSize, + ur_event_handle_t *phEvent); +ur_result_t urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, + size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t urMemoryExportFreeExportableMemoryExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pMem); +ur_result_t urMemoryExportExportMemoryHandleExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet); +ur_result_t urProgramBuildExp(ur_program_handle_t hProgram, uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramCompileExp(ur_program_handle_t hProgram, + uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramLinkExp(ur_context_handle_t hContext, uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, uint32_t count, + const ur_program_handle_t *phPrograms, + const char *pOptions, + ur_program_handle_t *phProgram); +ur_result_t urUSMContextMemcpyExp(ur_context_handle_t hContext, void *pDst, + const void *pSrc, size_t size); +ur_result_t urUSMHostAllocRegisterExp( + ur_context_handle_t hContext, void *pHostMem, size_t size, + const ur_exp_usm_host_alloc_register_properties_t *pProperties); +ur_result_t urUSMHostAllocUnregisterExp(ur_context_handle_t hContext, + void *pHostMem); +ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *pMem, + size_t size); +ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice); +ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice); +ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice, + ur_exp_peer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t +urCommandBufferCreateExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + ur_exp_command_buffer_handle_t *phCommandBuffer); +ur_result_t +urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t +urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t +urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer); +ur_result_t urCommandBufferAppendKernelLaunchExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, + uint32_t workDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, + uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendKernelLaunchWithArgsExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel, + uint32_t workDim, const size_t *pGlobalWorkOffset, + const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize, + uint32_t numArgs, const ur_exp_kernel_arg_properties_t *pArgs, + uint32_t numKernelAlternatives, ur_kernel_handle_t *phKernelAlternatives, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMMemcpyExp( + ur_exp_command_buffer_handle_t hCommandBuffer, void *pDst, const void *pSrc, + size_t size, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMFillExp( + ur_exp_command_buffer_handle_t hCommandBuffer, void *pMemory, + const void *pPattern, size_t patternSize, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferCopyExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, + ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferWriteExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + size_t offset, size_t size, const void *pSrc, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferReadExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + size_t offset, size_t size, void *pDst, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferCopyRectExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem, + ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, + size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferWriteRectExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferReadRectExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + ur_rect_offset_t bufferOffset, ur_rect_offset_t hostOffset, + ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, + size_t hostRowPitch, size_t hostSlicePitch, void *pDst, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendMemBufferFillExp( + ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer, + const void *pPattern, size_t patternSize, size_t offset, size_t size, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMPrefetchExp( + ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + size_t size, ur_usm_migration_flags_t flags, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendUSMAdviseExp( + ur_exp_command_buffer_handle_t hCommandBuffer, const void *pMemory, + size_t size, ur_usm_advice_flags_t advice, uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint, ur_event_handle_t *phEvent, + ur_exp_command_buffer_command_handle_t *phCommand); +ur_result_t urCommandBufferAppendNativeCommandExp( + ur_exp_command_buffer_handle_t hCommandBuffer, + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + void *pData, ur_exp_command_buffer_handle_t hChildCommandBuffer, + uint32_t numSyncPointsInWaitList, + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + ur_exp_command_buffer_sync_point_t *pSyncPoint); +ur_result_t urEnqueueCommandBufferExp( + ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urCommandBufferUpdateKernelLaunchExp( + ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates, + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch); +ur_result_t urCommandBufferUpdateSignalEventExp( + ur_exp_command_buffer_command_handle_t hCommand, + ur_event_handle_t *phSignalEvent); +ur_result_t urCommandBufferUpdateWaitEventsExp( + ur_exp_command_buffer_command_handle_t hCommand, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList); +ur_result_t +urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, + ur_exp_command_buffer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); +ur_result_t +urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, + ur_native_handle_t *phNativeCommandBuffer); +ur_result_t urEnqueueHostTaskExp( + ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, + void *data, const ur_exp_host_task_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueEventsWaitWithBarrierExt( + ur_queue_handle_t hQueue, + const ur_exp_enqueue_ext_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueNativeCommandExp( + ur_queue_handle_t hQueue, + ur_exp_enqueue_native_command_function_t pfnNativeEnqueue, void *data, + uint32_t numMemsInMemList, const ur_mem_handle_t *phMemList, + const ur_exp_enqueue_native_command_properties_t *pProperties, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEventCreateExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_exp_event_desc_t *pEventDesc, + ur_event_handle_t *phEvent); +ur_result_t urGraphCreateExp(ur_context_handle_t hContext, + ur_exp_graph_handle_t *phGraph); +ur_result_t urQueueBeginGraphCaptureExp(ur_queue_handle_t hQueue); +ur_result_t urQueueBeginCaptureIntoGraphExp(ur_queue_handle_t hQueue, + ur_exp_graph_handle_t hGraph); +ur_result_t urQueueEndGraphCaptureExp(ur_queue_handle_t hQueue, + ur_exp_graph_handle_t *phGraph); +ur_result_t +urGraphInstantiateGraphExp(ur_exp_graph_handle_t hGraph, + ur_exp_executable_graph_handle_t *phExecGraph); +ur_result_t urEnqueueGraphExp(ur_queue_handle_t hQueue, + ur_exp_executable_graph_handle_t hGraph, + uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urGraphDestroyExp(ur_exp_graph_handle_t hGraph); +ur_result_t urGraphExecutableGraphDestroyExp( + ur_exp_executable_graph_handle_t hExecutableGraph); +ur_result_t urQueueIsGraphCaptureEnabledExp(ur_queue_handle_t hQueue, + bool *pResult); +ur_result_t urQueueGetGraphExp(ur_queue_handle_t hQueue, + ur_exp_graph_handle_t *phGraph); +ur_result_t urGraphIsEmptyExp(ur_exp_graph_handle_t hGraph, bool *pResult); +ur_result_t urGraphGetIdExp(ur_exp_graph_handle_t hGraph, uint64_t *pGraphId); +ur_result_t urGraphSetDestructionCallbackExp( + ur_exp_graph_handle_t hGraph, + ur_exp_graph_destruction_callback_t pfnCallback, void *pUserData); +ur_result_t urGraphDumpContentsExp(ur_exp_graph_handle_t hGraph, + const char *filePath); +ur_result_t urGraphGetNativeHandleExp(ur_exp_graph_handle_t hGraph, + ur_native_handle_t *phNativeGraph); +ur_result_t urGraphExecutableGraphGetNativeHandleExp( + ur_exp_executable_graph_handle_t hExecutableGraph, + ur_native_handle_t *phNativeExecutableGraph); +#ifdef UR_STATIC_ADAPTER_OPENCL +ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi); +#endif + +struct ddi_getter { + const static ur_dditable_t *value(); +}; +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/usm.cpp b/unified-runtime/source/adapters/opencl/usm.cpp index c32b0888b6db4..d874e12adfec3 100644 --- a/unified-runtime/source/adapters/opencl/usm.cpp +++ b/unified-runtime/source/adapters/opencl/usm.cpp @@ -99,10 +99,14 @@ usmDescToCLMemProperties(const ur_base_desc_t *Desc, return UR_RESULT_SUCCESS; } +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urUSMHostAlloc(ur_context_handle_t Context, const ur_usm_desc_t *pUSMDesc, ur_usm_pool_handle_t, size_t size, void **ppMem) { + auto hContext = cast(Context); + void *Ptr = nullptr; uint32_t Alignment = pUSMDesc ? pUSMDesc->align : 0; @@ -119,9 +123,10 @@ urUSMHostAlloc(ur_context_handle_t Context, const ur_usm_desc_t *pUSMDesc, // First we need to look up the function pointer clHostMemAllocINTEL_fn FuncPtr = nullptr; - cl_context CLContext = Context->CLContext; + cl_context CLContext = hContext->CLContext; if (auto UrResult = cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clHostMemAllocINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clHostMemAllocINTELCache, cl_ext::HostMemAllocName, &FuncPtr)) { return UrResult; } @@ -151,6 +156,9 @@ urUSMDeviceAlloc(ur_context_handle_t Context, ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, ur_usm_pool_handle_t, size_t size, void **ppMem) { + auto hContext = cast(Context); + auto Device = cast(hDevice); + void *Ptr = nullptr; uint32_t Alignment = pUSMDesc ? pUSMDesc->align : 0; @@ -167,16 +175,17 @@ urUSMDeviceAlloc(ur_context_handle_t Context, ur_device_handle_t hDevice, // First we need to look up the function pointer clDeviceMemAllocINTEL_fn FuncPtr = nullptr; - cl_context CLContext = Context->CLContext; + cl_context CLContext = hContext->CLContext; if (auto UrResult = cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clDeviceMemAllocINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clDeviceMemAllocINTELCache, cl_ext::DeviceMemAllocName, &FuncPtr)) { return UrResult; } if (FuncPtr) { cl_int ClResult = CL_SUCCESS; - Ptr = FuncPtr(CLContext, hDevice->CLDevice, + Ptr = FuncPtr(CLContext, Device->CLDevice, AllocProperties.empty() ? nullptr : AllocProperties.data(), size, Alignment, &ClResult); if (ClResult == CL_INVALID_BUFFER_SIZE) { @@ -199,6 +208,9 @@ urUSMSharedAlloc(ur_context_handle_t Context, ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, ur_usm_pool_handle_t, size_t size, void **ppMem) { + auto hContext = cast(Context); + auto Device = cast(hDevice); + void *Ptr = nullptr; uint32_t Alignment = pUSMDesc ? pUSMDesc->align : 0; @@ -215,16 +227,17 @@ urUSMSharedAlloc(ur_context_handle_t Context, ur_device_handle_t hDevice, // First we need to look up the function pointer clSharedMemAllocINTEL_fn FuncPtr = nullptr; - cl_context CLContext = Context->CLContext; + cl_context CLContext = hContext->CLContext; if (auto UrResult = cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clSharedMemAllocINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clSharedMemAllocINTELCache, cl_ext::SharedMemAllocName, &FuncPtr)) { return UrResult; } if (FuncPtr) { cl_int ClResult = CL_SUCCESS; - Ptr = FuncPtr(CLContext, hDevice->CLDevice, + Ptr = FuncPtr(CLContext, Device->CLDevice, AllocProperties.empty() ? nullptr : AllocProperties.data(), size, Alignment, static_cast(&ClResult)); if (ClResult == CL_INVALID_BUFFER_SIZE) { @@ -244,14 +257,17 @@ urUSMSharedAlloc(ur_context_handle_t Context, ur_device_handle_t hDevice, UR_APIEXPORT ur_result_t UR_APICALL urUSMFree(ur_context_handle_t Context, void *pMem) { + auto hContext = cast(Context); + // Use a blocking free to avoid issues with indirect access from kernels that // might be still running. clMemBlockingFreeINTEL_fn FuncPtr = nullptr; - cl_context CLContext = Context->CLContext; + cl_context CLContext = hContext->CLContext; ur_result_t RetVal = UR_RESULT_ERROR_INVALID_OPERATION; RetVal = cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clMemBlockingFreeINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clMemBlockingFreeINTELCache, cl_ext::MemBlockingFreeName, &FuncPtr); if (FuncPtr) { @@ -265,27 +281,31 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMFill( ur_queue_handle_t hQueue, void *ptr, size_t patternSize, const void *pPattern, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + + auto Queue = cast(hQueue); + // Have to look up the context from the kernel - cl_context CLContext = hQueue->Context->CLContext; + cl_context CLContext = Queue->Context->CLContext; if (patternSize <= 128 && isPowerOf2(patternSize) && isPointerAlignedTo(patternSize, ptr)) { clEnqueueMemFillINTEL_fn EnqueueMemFill = nullptr; UR_RETURN_ON_FAILURE( cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clEnqueueMemFillINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clEnqueueMemFillINTELCache, cl_ext::EnqueueMemFillName, &EnqueueMemFill)); cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } CL_RETURN_ON_FAILURE(EnqueueMemFill( - hQueue->CLQueue, ptr, pPattern, patternSize, size, numEventsInWaitList, + Queue->CLQueue, ptr, pPattern, patternSize, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -296,12 +316,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMFill( clEnqueueMemcpyINTEL_fn USMMemcpy = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clEnqueueMemcpyINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clEnqueueMemcpyINTELCache, cl_ext::EnqueueMemcpyName, &USMMemcpy)); clMemBlockingFreeINTEL_fn USMFree = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clMemBlockingFreeINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clMemBlockingFreeINTELCache, cl_ext::MemBlockingFreeName, &USMFree)); uint8_t *HostBuffer = new uint8_t[size]; @@ -314,9 +335,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMFill( cl_event CopyEvent = nullptr; std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } - CL_RETURN_ON_FAILURE(USMMemcpy(hQueue->CLQueue, false, ptr, HostBuffer, size, + CL_RETURN_ON_FAILURE(USMMemcpy(Queue->CLQueue, false, ptr, HostBuffer, size, numEventsInWaitList, CLWaitEvents.data(), &CopyEvent)); @@ -326,8 +347,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMFill( CL_RETURN_ON_FAILURE(clRetainEvent(CopyEvent)); try { auto UREvent = std::make_unique( - CopyEvent, hQueue->Context, hQueue); - *phEvent = UREvent.release(); + CopyEvent, Queue->Context, Queue); + *phEvent = cast(UREvent.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -359,23 +380,27 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + auto Queue = cast(hQueue); + // Have to look up the context from the kernel - cl_context CLContext = hQueue->Context->CLContext; + cl_context CLContext = Queue->Context->CLContext; cl_int CLErr = CL_SUCCESS; clGetMemAllocInfoINTEL_fn GetMemAllocInfo = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clGetMemAllocInfoINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clGetMemAllocInfoINTELCache, cl_ext::GetMemAllocInfoName, &GetMemAllocInfo)); clEnqueueMemcpyINTEL_fn USMMemcpy = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clEnqueueMemcpyINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clEnqueueMemcpyINTELCache, cl_ext::EnqueueMemcpyName, &USMMemcpy)); clMemBlockingFreeINTEL_fn USMFree = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clMemBlockingFreeINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clMemBlockingFreeINTELCache, cl_ext::MemBlockingFreeName, &USMFree)); // Check if the two allocations are DEVICE allocations from different @@ -393,7 +418,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( // We need a queue associated with each device, so first figure out which // one we weren't given. cl_device_id QueueDevice = nullptr; - CL_RETURN_ON_FAILURE(clGetCommandQueueInfo(hQueue->CLQueue, CL_QUEUE_DEVICE, + CL_RETURN_ON_FAILURE(clGetCommandQueueInfo(Queue->CLQueue, CL_QUEUE_DEVICE, sizeof(QueueDevice), &QueueDevice, nullptr)); @@ -401,11 +426,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( DstQueue = nullptr; if (QueueDevice == SrcDevice) { MissingQueue = clCreateCommandQueue(CLContext, DstDevice, 0, &CLErr); - SrcQueue = hQueue->CLQueue; + SrcQueue = Queue->CLQueue; DstQueue = MissingQueue; } else { MissingQueue = clCreateCommandQueue(CLContext, SrcDevice, 0, &CLErr); - DstQueue = hQueue->CLQueue; + DstQueue = Queue->CLQueue; SrcQueue = MissingQueue; } CL_RETURN_ON_FAILURE(CLErr); @@ -413,7 +438,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( cl_event HostCopyEvent = nullptr, FinalCopyEvent = nullptr; clHostMemAllocINTEL_fn HostMemAlloc = nullptr; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clHostMemAllocINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clHostMemAllocINTELCache, cl_ext::HostMemAllocName, &HostMemAlloc)); auto HostAlloc = static_cast( @@ -438,7 +463,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } UR_RETURN_ON_FAILURE(checkCLErr( USMMemcpy(SrcQueue, blocking, HostAlloc, pSrc, size, @@ -457,8 +482,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( if (phEvent) { try { auto UREvent = std::make_unique( - FinalCopyEvent, hQueue->Context, hQueue); - *phEvent = UREvent.release(); + FinalCopyEvent, Queue->Context, Queue); + *phEvent = cast(UREvent.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -471,8 +496,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( if (phEvent) { try { auto UREvent = std::make_unique( - FinalCopyEvent, hQueue->Context, hQueue); - *phEvent = UREvent.release(); + FinalCopyEvent, Queue->Context, Queue); + *phEvent = cast(UREvent.release()); } catch (std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_RESOURCES; } catch (...) { @@ -505,13 +530,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy( cl_event Event; std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } - CL_RETURN_ON_FAILURE(USMMemcpy(hQueue->CLQueue, blocking, pDst, pSrc, size, + CL_RETURN_ON_FAILURE(USMMemcpy(Queue->CLQueue, blocking, pDst, pSrc, size, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); } return UR_RESULT_SUCCESS; @@ -523,6 +548,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMPrefetch( [[maybe_unused]] ur_usm_migration_flags_t flags, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + + auto Queue = cast(hQueue); + // TODO: Uncomment implementation when issues with impl are resolved. // cl_mem_migration_flags MigrationFlag; @@ -560,22 +588,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMPrefetch( cl_event Event = nullptr; std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } /* CL_RETURN_ON_FAILURE(EnqueueMigrateMem( - hQueue->CLQueue, pMem, size, MigrationFlag, numEventsInWaitList, + Queue->CLQueue, pMem, size, MigrationFlag, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); */ // TODO: when issues with impl are fully resolved, delete this and use // waitlisting from EnqueueMigrateMem instead. CL_RETURN_ON_FAILURE(clEnqueueMarkerWithWaitList( - hQueue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), + Queue->CLQueue, numEventsInWaitList, CLWaitEvents.data(), ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; } @@ -583,10 +612,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMAdvise( ur_queue_handle_t hQueue, [[maybe_unused]] const void *pMem, [[maybe_unused]] size_t size, [[maybe_unused]] ur_usm_advice_flags_t advice, ur_event_handle_t *phEvent) { + + auto Queue = cast(hQueue); + cl_event Event; - CL_RETURN_ON_FAILURE(clEnqueueMarkerWithWaitList(hQueue->CLQueue, 0, nullptr, + CL_RETURN_ON_FAILURE(clEnqueueMarkerWithWaitList(Queue->CLQueue, 0, nullptr, ifUrEvent(phEvent, Event))); - UR_RETURN_ON_FAILURE(createUREvent(Event, hQueue->Context, hQueue, phEvent)); + UR_RETURN_ON_FAILURE( + createUREvent(Event, cast(Queue->Context), cast(Queue), phEvent)); return UR_RESULT_SUCCESS; /* // Change to use this once drivers support it. @@ -625,11 +658,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy2D( const void *pSrc, size_t srcPitch, size_t width, size_t height, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - cl_context CLContext = hQueue->Context->CLContext; + + auto Queue = cast(hQueue); + + cl_context CLContext = Queue->Context->CLContext; clEnqueueMemcpyINTEL_fn FuncPtr = nullptr; ur_result_t RetVal = cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clEnqueueMemcpyINTELCache, + CLContext, cast(ur::cl::getAdapter())->fnCache.clEnqueueMemcpyINTELCache, cl_ext::EnqueueMemcpyName, &FuncPtr); if (!FuncPtr) { @@ -641,10 +677,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy2D( cl_event Event = nullptr; std::vector CLWaitEvents(numEventsInWaitList); for (uint32_t i = 0; i < numEventsInWaitList; i++) { - CLWaitEvents[i] = phEventWaitList[i]->CLEvent; + CLWaitEvents[i] = cast(phEventWaitList[i])->CLEvent; } auto ClResult = - FuncPtr(hQueue->CLQueue, false, + FuncPtr(Queue->CLQueue, false, static_cast(pDst) + dstPitch * HeightIndex, static_cast(pSrc) + srcPitch * HeightIndex, width, numEventsInWaitList, CLWaitEvents.data(), &Event); @@ -663,10 +699,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy2D( if (phEvent && ClResult == CL_SUCCESS) { cl_event Event; ClResult = - clEnqueueBarrierWithWaitList(hQueue->CLQueue, Events.size(), + clEnqueueBarrierWithWaitList(Queue->CLQueue, Events.size(), Events.data(), ifUrEvent(phEvent, Event)); UR_RETURN_ON_FAILURE( - createUREvent(Event, hQueue->Context, hQueue, phEvent)); + createUREvent(Event, cast(Queue->Context), hQueue, phEvent)); } for (const auto &E : Events) { CL_RETURN_ON_FAILURE(clReleaseEvent(E)); @@ -694,10 +730,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMGetMemAllocInfo( ur_context_handle_t Context, const void *pMem, ur_usm_alloc_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + auto hContext = cast(Context); + clGetMemAllocInfoINTEL_fn GetMemAllocInfo = nullptr; - cl_context CLContext = Context->CLContext; + cl_context CLContext = hContext->CLContext; UR_RETURN_ON_FAILURE(cl_ext::getExtFuncFromContext( - CLContext, ur::cl::getAdapter()->fnCache.clGetMemAllocInfoINTELCache, + CLContext, + cast(ur::cl::getAdapter())->fnCache.clGetMemAllocInfoINTELCache, cl_ext::GetMemAllocInfoName, &GetMemAllocInfo)); cl_mem_info_intel PropNameCL; @@ -719,11 +758,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMGetMemAllocInfo( } UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); if (propName == UR_USM_ALLOC_INFO_DEVICE) { - return ReturnValue(Context->Devices[0]); + return ReturnValue(cast(hContext->Devices[0])); } size_t CheckPropSize = 0; - cl_int ClErr = GetMemAllocInfo(Context->CLContext, pMem, PropNameCL, propSize, - pPropValue, &CheckPropSize); + cl_int ClErr = GetMemAllocInfo(hContext->CLContext, pMem, PropNameCL, + propSize, pPropValue, &CheckPropSize); if (pPropValue && CheckPropSize != propSize) { return UR_RESULT_ERROR_INVALID_SIZE; } @@ -840,3 +879,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMHostAllocUnregisterExp( ur_context_handle_t /*hContext*/, void * /*pHostMem*/) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/usm_p2p.cpp b/unified-runtime/source/adapters/opencl/usm_p2p.cpp index b8bd7ad5de910..a1de07703ad77 100644 --- a/unified-runtime/source/adapters/opencl/usm_p2p.cpp +++ b/unified-runtime/source/adapters/opencl/usm_p2p.cpp @@ -9,6 +9,8 @@ #include "common.hpp" +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp([[maybe_unused]] ur_device_handle_t commandDevice, [[maybe_unused]] ur_device_handle_t peerDevice) { @@ -35,3 +37,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( die("Experimental P2P feature is not implemented for OpenCL adapter."); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/adapters/opencl/virtual_mem.cpp b/unified-runtime/source/adapters/opencl/virtual_mem.cpp index bc10d5f06dbf0..3c463062e4c8b 100644 --- a/unified-runtime/source/adapters/opencl/virtual_mem.cpp +++ b/unified-runtime/source/adapters/opencl/virtual_mem.cpp @@ -11,6 +11,8 @@ #include "context.hpp" #include "physical_mem.hpp" +namespace ur::opencl { + UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( ur_context_handle_t, ur_device_handle_t, size_t, ur_virtual_mem_granularity_info_t, size_t, void *, size_t *) { @@ -51,3 +53,5 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(ur_context_handle_t, size_t *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +} // namespace ur::opencl diff --git a/unified-runtime/source/common/CMakeLists.txt b/unified-runtime/source/common/CMakeLists.txt index 93e3562ef12b2..6729c5e1629ff 100644 --- a/unified-runtime/source/common/CMakeLists.txt +++ b/unified-runtime/source/common/CMakeLists.txt @@ -159,3 +159,14 @@ target_link_libraries(ur_umf INTERFACE umf::umf umf::headers ) + +# Install ur_umf when any static adapter is enabled, so install(EXPORT ...) +# can resolve it for static consumers. +if(UR_STATIC_ADAPTER_L0 OR UR_STATIC_ADAPTER_OPENCL) + install(TARGETS ur_umf + EXPORT ${PROJECT_NAME}-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif() diff --git a/unified-runtime/source/loader/CMakeLists.txt b/unified-runtime/source/loader/CMakeLists.txt index d4324ca231d21..6a4569a4583ba 100644 --- a/unified-runtime/source/loader/CMakeLists.txt +++ b/unified-runtime/source/loader/CMakeLists.txt @@ -86,6 +86,11 @@ if(UR_STATIC_ADAPTER_L0) target_compile_definitions(ur_loader PRIVATE UR_STATIC_ADAPTER_LEVEL_ZERO) endif() +if(UR_STATIC_ADAPTER_OPENCL AND (UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)) + target_link_libraries(ur_loader PRIVATE ur_adapter_opencl) + target_compile_definitions(ur_loader PRIVATE UR_STATIC_ADAPTER_OPENCL) +endif() + if(UR_ENABLE_TRACING) target_link_libraries(ur_loader PRIVATE ${TARGET_XPTI}) target_include_directories(ur_loader PRIVATE ${xpti_SOURCE_DIR}/include) diff --git a/unified-runtime/source/loader/ur_adapter_registry.hpp b/unified-runtime/source/loader/ur_adapter_registry.hpp index f17d1aa3c0cc7..ca382c617c498 100644 --- a/unified-runtime/source/loader/ur_adapter_registry.hpp +++ b/unified-runtime/source/loader/ur_adapter_registry.hpp @@ -394,11 +394,31 @@ class AdapterRegistry { } bool forceLoaded = false; + std::set staticLoadedAdapters; public: + void markAdapterAsStaticallyLoaded(const std::string &adapterName) { + staticLoadedAdapters.insert(adapterName); + } + + bool isStaticallyLoaded(const fs::path &adapterPath) const { + std::string pathStr = adapterPath.string(); + for (const auto &name : staticLoadedAdapters) { + if (pathStr.find(name) != std::string::npos) { + return true; + } + } + return false; + } + void enableMock() { adaptersLoadPaths.clear(); + // Set it to prevent context_t::init() from loading static adapters + // (guarded by !adaptersForceLoaded()): the mock adapter must not coexist + // with other platforms. + forceLoaded = true; + std::vector loadPaths; auto adapterNamePath = fs::path{mockAdapterName}; loadPaths.emplace_back(adapterNamePath); diff --git a/unified-runtime/source/loader/ur_loader.cpp b/unified-runtime/source/loader/ur_loader.cpp index 7e3412b65784b..f216fdb43ea69 100644 --- a/unified-runtime/source/loader/ur_loader.cpp +++ b/unified-runtime/source/loader/ur_loader.cpp @@ -11,6 +11,9 @@ #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO #include "adapters/level_zero/ur_interface_loader.hpp" #endif +#ifdef UR_STATIC_ADAPTER_OPENCL +#include "adapters/opencl/ur_interface_loader.hpp" +#endif namespace ur_loader { /////////////////////////////////////////////////////////////////////////////// @@ -31,16 +34,30 @@ ur_result_t context_t::init() { UINT SavedMode = SetErrorMode(SEM_FAILCRITICALERRORS); #endif -#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO +#if defined(UR_STATIC_ADAPTER_LEVEL_ZERO) || defined(UR_STATIC_ADAPTER_OPENCL) // If the adapters were force loaded, it means the user wants to use // a specific adapter library. Don't load any static adapters. if (!adapter_registry.adaptersForceLoaded()) { +#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO auto &level_zero = platforms.emplace_back(nullptr); ur::level_zero::urAdapterGetDdiTables(&level_zero.dditable); + adapter_registry.markAdapterAsStaticallyLoaded("ur_adapter_level_zero"); +#endif +#ifdef UR_STATIC_ADAPTER_OPENCL + auto &opencl = platforms.emplace_back(nullptr); + ur::opencl::urAdapterGetDdiTables(&opencl.dditable); + adapter_registry.markAdapterAsStaticallyLoaded("ur_adapter_opencl"); +#endif } #endif for (const auto &adapterPaths : adapter_registry) { + // Skip dynamic adapters that have already been statically registered + // to avoid double-registration of the same backend. + if (!adapterPaths.empty() && + adapter_registry.isStaticallyLoaded(adapterPaths[0])) { + continue; + } for (const auto &path : adapterPaths) { auto handle = LibLoader::loadAdapterLibrary(path.string().c_str()); if (handle) {