From 3b578c35cf503499bbd640a625ce9c667c8ac090 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Sun, 12 Feb 2023 19:09:41 -0500 Subject: [PATCH 1/3] Since PR https://github.com/ICLDisco/parsec/pull/482 we don't copy these files in the build directory, so we must point to the source ones --- tools/profiling/python/setup.py.in | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/profiling/python/setup.py.in b/tools/profiling/python/setup.py.in index 1e250c308..fc2305a78 100644 --- a/tools/profiling/python/setup.py.in +++ b/tools/profiling/python/setup.py.in @@ -44,14 +44,14 @@ if 'Debug' == build_type or 'RelWithDebInfo' == build_type: # Cython does not support VPATH, so the generated files will always reside on the # SOURCE_DIR, except if we explicitly move them to the build directory. -extensions = [Extension('pbt2ptt', ['@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pyx', +extensions = [Extension('pbt2ptt', [os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pyx'), os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/../dbpreader.c')], include_dirs=['@PROJECT_SOURCE_DIR@/parsec/include', '@PROJECT_SOURCE_DIR@', '@PROJECT_BINARY_DIR@/parsec/include', '@PROJECT_BINARY_DIR@', '@CMAKE_CURRENT_SOURCE_DIR@/../'], - depends=['pbt2ptt.pxd', + depends=[os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pxd'), os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/../dbpreader.h'), '@PROJECT_SOURCE_DIR@/parsec/parsec_binary_profile.h', '@PROJECT_SOURCE_DIR@/parsec/include/parsec/os-spec-timing.h'], @@ -130,7 +130,9 @@ setup( description='PaRSEC Binary Trace Interface parses and converts the PaRSEC Binary Trace format into a pandas-based Python tabular format', url='http://icl.cs.utk.edu/parsec/', package_dir={ '': '@PYTHON_TOOLS_BIN_DIR@'}, - py_modules=['ptt_utils', 'parsec_trace_tables', 'common_utils'], + py_modules=[os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/ptt_utils'), + os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/parsec_trace_tables'), + os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/common_utils')], cmdclass = {'build_ext': local_compiler_build_ext}, ext_modules = cythonize(extensions, compiler_directives={'language_level' : @Python_VERSION_MAJOR@}, From ab86fa3dac578090dbf6b7ba91ba616100dc528a Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 7 May 2021 09:38:47 -0400 Subject: [PATCH 2/3] Introduce the DPC++ and LevelZero device driver, enable this device in DTD and PTG. This branch is based on common_gpu and should be merged only after common_gpu Add a new level_zero device (WIP) - copy device_cuda in device_level_zero and rename things - module_init and module_fini for level_zero Need to factorize a little bit more. Factorizing (need to do it in base) Port above new common Add DPC++ to the loop... - Add multiple CMake logic files and commands - jdf2c.c now generates dpcpp output files when needed - make DEV_DPCPP be an alias to DEV_LEVEL_ZERO - Command Lists for I/O (streams of id 0 and 1) are still immediate - Command Lists for computations (streams of id >= 2) are now normal lists connected to a queue that queue exists as a compute level-zero queue and as a DPC++ queue - Missing compilation logic to compile generated dpc++ code and link it with the target binary Risk: it is unclear that the user can still push orders / events in the command list, after it is closed, and it is necessary to close it to force the orders to be pushed on the queue. I might need to create a new command list after each close, and attach the command list to the event for garbage collection. Adapt findlevel-zero.cmake to support systems where pkg-config is broken Re-enable Level Zero test; update to latest level zero / oneAPI API Update wrapper to allow testing both CUDA and Level Zero with new Level Zero update use_cuda / use_cuda_index have been renamed to follow proper naming scheme; do the same for level_zero Try to automate DPCPP generated code compilation; fix ordinal of memory allocation request in wrapper. Command Lists need to be sent to the Command Queue if they are not created immediate (and they cannot be immediate if we want to get their Command Queue, which is necessary for the DPC++ interface) Typo and multiple CMake fixes to make CMake link with DPCPP generated files Add a standalone test for Zero Level capability and integration with DPC++ kernels Rebase the entire Level Zero driver based on the susbsystem test Buffer interface is not required. We can use the USM OneMKL interface, it seems to work ok. Need to check for performance. We cannot mix immediate and non-immediate command lists apparently. Or at least it makes the passing of command queues unreliable There is an exception in data.c how we handle GPU copies, it must be ported to Level Zero too. The Level Zero runtime has a atexit procedure to delete command queues, and this seems to conflict with our own actions to delete the command queues... Porting of the DTD GEMM test to Level Zero NULL is not a valid MPI datatype when compiling with a clone of MPICH. The value doesn't matter in this case, just cast Manage LEVEL_ZERO devices in DTD Accept LEVEL_ZERO devices in the PTG generated code Some fixes in device level_zero Temp fix for termination detection -- tag size must be made portable. TODO! Support LEVEL_ZERO devices in the DSL tests Fix the subsystem test. Need to backport fixes in the MCA device Fully functional sketch for level zero Use level-zero fences to synchronize command lists and command queues, because command lists (or work) submitted to the command queues by SYCL (typically oneMKL) can complete in parallel with events belonging to other command lists. Define the set of globals in DPC++ code after the includse happen to avoid polluting their namespace; cleanup some unused variables Install LevelZero driver files; setup the environment to find the same LevelZero library as at compile time in PaRSECConfig.cmake --- CMakeLists.txt | 31 + cmake_modules/FindDPCPP.cmake | 44 + cmake_modules/Findlevel-zero.cmake | 46 + cmake_modules/PaRSECConfig.cmake.in | 17 + cmake_modules/ParsecCompilePTG.cmake | 33 +- parsec/data.c | 3 +- parsec/include/parsec/parsec_options.h.in | 2 + parsec/interfaces/dtd/insert_function.c | 25 +- parsec/interfaces/ptg/ptg-compiler/jdf.h | 10 +- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 519 +++- parsec/interfaces/ptg/ptg-compiler/jdf2c.h | 5 +- parsec/interfaces/ptg/ptg-compiler/main.c | 26 +- parsec/interfaces/ptg/ptg-compiler/parsec.l | 8 +- parsec/interfaces/ptg/ptg-compiler/parsec.y | 30 +- parsec/mca/device/CMakeLists.txt | 4 +- parsec/mca/device/device.c | 2 +- parsec/mca/device/device.h | 15 +- parsec/mca/device/device_gpu.c | 4 +- parsec/mca/device/device_gpu.h | 3 +- .../device/level_zero/ValidateModule.CMake | 37 + .../mca/device/level_zero/device_level_zero.h | 149 + .../level_zero/device_level_zero_component.c | 425 +++ .../level_zero/device_level_zero_dpcpp.h | 42 + .../device_level_zero_dpcpp_interface.cpp | 59 + .../level_zero/device_level_zero_internal.h | 78 + .../level_zero/device_level_zero_module.c | 2678 +++++++++++++++++ parsec/mca/device/parsec_gpu.c | 97 + parsec/mca/device/transfer_gpu.c | 11 +- parsec/parsec_internal.h | 4 + parsec/remote_dep_mpi.c | 4 +- tests/CMakeLists.txt | 1 + tests/dsl/dtd/CMakeLists.txt | 22 +- tests/dsl/dtd/dtd_test_simple_gemm_lz.c | 646 ++++ .../dtd_test_simple_gemm_lz_kernel.dpcpp.C | 77 + tests/dsl/ptg/cuda/CMakeLists.txt | 23 +- tests/dsl/ptg/cuda/nvlink.jdf | 66 + tests/dsl/ptg/cuda/nvlink_wrapper.c | 98 +- tests/subsystem/CMakeLists.txt | 5 + tests/subsystem/level_zero/CMakeLists.txt | 14 + tests/subsystem/level_zero/interface.dpcpp.C | 56 + tests/subsystem/level_zero/interface.dpcpp.h | 40 + tests/subsystem/level_zero/kernel.dpcpp.C | 64 + tests/subsystem/level_zero/main.c | 581 ++++ 43 files changed, 5879 insertions(+), 225 deletions(-) create mode 100644 cmake_modules/FindDPCPP.cmake create mode 100644 cmake_modules/Findlevel-zero.cmake create mode 100644 parsec/mca/device/level_zero/ValidateModule.CMake create mode 100644 parsec/mca/device/level_zero/device_level_zero.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_component.c create mode 100644 parsec/mca/device/level_zero/device_level_zero_dpcpp.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp create mode 100644 parsec/mca/device/level_zero/device_level_zero_internal.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_module.c create mode 100644 parsec/mca/device/parsec_gpu.c create mode 100644 tests/dsl/dtd/dtd_test_simple_gemm_lz.c create mode 100644 tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C create mode 100644 tests/subsystem/CMakeLists.txt create mode 100644 tests/subsystem/level_zero/CMakeLists.txt create mode 100644 tests/subsystem/level_zero/interface.dpcpp.C create mode 100644 tests/subsystem/level_zero/interface.dpcpp.h create mode 100644 tests/subsystem/level_zero/kernel.dpcpp.C create mode 100644 tests/subsystem/level_zero/main.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 27da45a80..6b3adb5cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,9 @@ include(CMakeDependentOption) include(CMakePushCheckState) include(GNUInstallDirs) +# Help debug dependencies with custom targets that PTG uses a lot +SET(GRAPHVIZ_CUSTOM_TARGETS TRUE) + # The current version number # This uses the numbering scheme from libtool -version_number c:r:a # see http://www.sourceware.org/autobook/autobook/autobook_61.html @@ -173,6 +176,12 @@ option(PARSEC_GPU_HIP_ALLOC_PER_TILE "Tile based allocation engine for GPU memory (instead of internal management of a complete allocation)" OFF) mark_as_advanced(PARSEC_GPU_HIP_ALLOC_PER_TILE) +option(PARSEC_GPU_WITH_LEVEL_ZERO + "Enable GPU support using LEVEL_ZERO kernels" ON) +option(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE + "Tile based allocation engine for GPU memory (instead of internal management + of a complete allocation)" OFF) +mark_as_advanced(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) option(PARSEC_GPU_WITH_OPENCL "Enable GPU support using OpenCL kernels" OFF) mark_as_advanced(PARSEC_GPU_WITH_OPENCL) # Hide this as it is not supported yet @@ -737,6 +746,19 @@ int main(int argc, char *argv[]) { endif() set(PARSEC_HAVE_HIP ${HIP_NOT_CUDA_FOUND} CACHE BOOL "True if PaRSEC provide support for HIP") endif( PARSEC_GPU_WITH_HIP ) + if( PARSEC_GPU_WITH_LEVEL_ZERO ) + find_package(level-zero) + find_package(DPCPP) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + endif( PARSEC_GPU_WITH_LEVEL_ZERO ) find_package(AYUDAME QUIET) set(PARSEC_HAVE_AYUDAME ${AYUDAME_FOUND}) @@ -915,6 +937,13 @@ include(AddDocumentedFiles) add_subdirectory(parsec) +# +# Add dependency to Level-Zero if it is enabled +# +if(PARSEC_HAVE_LEVEL_ZERO) + target_link_libraries(parsec PRIVATE level_zero::ze_loader) +endif(PARSEC_HAVE_LEVEL_ZERO) + # # Now continue with compiling the tests. # @@ -1237,6 +1266,8 @@ install(FILES ${PROJECT_SOURCE_DIR}/cmake_modules/FindPAPI.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindOTF2.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindHWLOC.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/Findlevel-zero.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/FindDPCPP.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/ParsecCompilePTG.cmake DESTINATION ${PARSEC_INSTALL_CMAKEDIR}) diff --git a/cmake_modules/FindDPCPP.cmake b/cmake_modules/FindDPCPP.cmake new file mode 100644 index 000000000..c4073d268 --- /dev/null +++ b/cmake_modules/FindDPCPP.cmake @@ -0,0 +1,44 @@ +if(DPCPP_EXECUTABLE) + get_filename_component( _dpcpp_path ${DPCPP_EXECUTABLE} PATH ) +else(DPCPP_EXECUTABLE) + set(_dpcpp_path "") +endif(DPCPP_EXECUTABLE) +find_program( DPCPP_EXECUTABLE NAMES dpcpp HINTS ${_dpcpp_path} ENV PATH ) +mark_as_advanced( DPCPP_EXECUTABLE ) + +if( DPCPP_EXECUTABLE ) + execute_process(COMMAND ${DPCPP_EXECUTABLE} --version + RESULT_VARIABLE _res + OUTPUT_VARIABLE _out + ERROR_VARIABLE _err) + if( _res EQUAL 0 ) + string(REGEX MATCH "([^\n]+)" _ ${_out}) + message(STATUS "Found dpcpp: ${DPCPP_EXECUTABLE} version ${CMAKE_MATCH_1}") + + get_filename_component( _dpcpp_dir ${DPCPP_EXECUTABLE} DIRECTORY ) + get_filename_component( _dpcpp_lib_dir "${_dpcpp_dir}/../lib" ABSOLUTE ) + + find_library(SYCL_LIBRARY sycl HINTS "${_dpcpp_lib_dir}" "${SYCL_LIBRARY_DIR}" ENV "SYCL_LIBRARY_DIR") + if( SYCL_LIBRARY ) + message(STATUS "SYCL library: ${SYCL_LIBRARY}") + + get_filename_component( _dpcpp_inc_dir "${_dpcpp_dir}/../include" ABSOLUTE ) + find_file(_sycl_include_file "sycl.hpp" HINTS "${_dpcpp_inc_dir}" "${_dpcpp_inc_dir}/sycl/CL" "${SYCL_INCLUDE_DIR}" ENV "SYCL_INCLUDE_DIR") + if(_sycl_include_file) + get_filename_component(SYCL_INCLUDE_DIR "${_sycl_include_file}" DIRECTORY) + message(STATUS "SYCL include dir: ${SYCL_INCLUDE_DIR}") + set(PARSEC_HAVE_DPCPP "TRUE" CACHE BOOL "True if PaRSEC provide support for dpc++") + else(_sycl_include_file) + message(WARNING "Found DPC++, and SYCL library, but could not find SYCL include directory. Define SYCL_INCLUDE_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif(_sycl_include_file) + else( SYCL_LIBRARY ) + message(WARNING "Found DPC++, but did not find SYCL library. Define SYCL_LIBRARY_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif( SYCL_LIBRARY ) + else( _res EQUAL 0 ) + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + message(WARNING "${DPCPP_EXECUTABLE} does not work: 'dpcpp -v' returned ${_res}, with error ${_err}") + message(WARNING "dpc++ support is disabled, set DPCPP_EXECUTABLE to the path of dpcpp to enable DPC++ support, and/or fix your environment to run dpcpp") + endif( _res EQUAL 0 ) +endif( DPCPP_EXECUTABLE ) \ No newline at end of file diff --git a/cmake_modules/Findlevel-zero.cmake b/cmake_modules/Findlevel-zero.cmake new file mode 100644 index 000000000..51b63912e --- /dev/null +++ b/cmake_modules/Findlevel-zero.cmake @@ -0,0 +1,46 @@ +if(LEVEL_ZERO_ROOT_DIR) + message(STATUS "Trying to locate level-zero library and headers under ${LEVEL_ZERO_ROOT_DIR}") + find_library(ZE_LOADER_LIBRARY "ze_loader" HINTS "${LEVEL_ZERO_ROOT_DIR}/lib" "${LEVEL_ZERO_ROOT_DIR}/lib64" NO_DEFAULT_PATH) + find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH) + + if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY) + check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT) + + if(ZE_LOADER_HAVE_ZEINIT) + message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}") + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + set(LEVEL_ZERO_FOUND TRUE) + else(ZE_LOADER_HAVE_ZEINIT) + if(NOT ZE_LOADER_HAVE_ZEINIT) + message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search") + endif(NOT ZE_LOADER_HAVE_ZEINIT) + endif(ZE_LOADER_HAVE_ZEINIT) + else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + if(NOT ZE_LOADER_LIBRARY) + message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search") + endif(NOT ZE_LOADER_LIBRARY) + if(NOT LEVEL_ZERO_INCLUDE_DIR) + message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search") + endif(NOT LEVEL_ZERO_INCLUDE_DIR) + endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) +endif(LEVEL_ZERO_ROOT_DIR) + +if(NOT LEVEL_ZERO_FOUND) + find_package(PkgConfig QUIET) + + if(PKG_CONFIG_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if(LEVEL_ZERO_FOUND) + pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir) + pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir) + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + endif(LEVEL_ZERO_FOUND) + endif(PKG_CONFIG_FOUND) +endif(NOT LEVEL_ZERO_FOUND) diff --git a/cmake_modules/PaRSECConfig.cmake.in b/cmake_modules/PaRSECConfig.cmake.in index 60460a99b..507adf6e0 100644 --- a/cmake_modules/PaRSECConfig.cmake.in +++ b/cmake_modules/PaRSECConfig.cmake.in @@ -73,6 +73,23 @@ if(@PARSEC_HAVE_HIP@) SET(PARSEC_HAVE_HIP TRUE) endif(@PARSEC_HAVE_HIP@) +if(@PARSEC_HAVE_LEVEL_ZERO@) + if(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + set(LEVEL_ZERO_ROOT_DIR "@LEVEL_ZERO_ROOT_DIR@") + endif(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + find_package(level-zero) + find_package(DPCPP) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) +endif(@PARSEC_HAVE_LEVEL_ZERO@) + if(@PARSEC_PROF_TRACE@) # Nothing exportable here, if this test succeed then PaRSEC supports tracing endif(@PARSEC_PROF_TRACE@) diff --git a/cmake_modules/ParsecCompilePTG.cmake b/cmake_modules/ParsecCompilePTG.cmake index affdcb4dd..10a449584 100644 --- a/cmake_modules/ParsecCompilePTG.cmake +++ b/cmake_modules/ParsecCompilePTG.cmake @@ -7,7 +7,7 @@ # function(target_ptg_source_ex) set(options DEBUG LINE FORCE_PROFILE) - set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H FUNCTION_NAME DEP_MANAGEMENT) + set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H DESTINATION_DPCPP FUNCTION_NAME DEP_MANAGEMENT) set(multipleValueArgs WARNINGS IGNORE_PROPERTIES PTGPP_FLAGS) cmake_parse_arguments(PARSEC_PTGPP "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -51,6 +51,9 @@ function(target_ptg_source_ex) else() set(outname_h "${outname}.h") endif() + if(DEFINED PARSEC_PTGPP_DESTINATION_DPCPP) + set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPCPP}") + endif() if(DEFINED PARSEC_PTGPP_FUNCTION_NAME) set(fnname "${PARSEC_PTGPP_FUNCTION_NAME}") @@ -91,12 +94,21 @@ function(target_ptg_source_ex) # its cmake source_file name, yet we depend on the source_file name as it is how cmake tracks it get_property(location SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY LOCATION) - add_custom_command( + if(DEFINED outname_dpcpp) + add_custom_command( + OUTPUT ${outname_h} ${outname_c} ${outname_dpcpp} + COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -S ${outname_dpcpp} -f ${fnname} + MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} + DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c} ${outname_dpcpp}) + else(DEFINED outname_dpcpp) + add_custom_command( OUTPUT ${outname_h} ${outname_c} COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -f ${fnname} MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) - add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + endif(DEFINED outname_dpcpp) # Copy the properties to the generated files get_property(cflags SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY COMPILE_OPTIONS) @@ -110,11 +122,24 @@ function(target_ptg_source_ex) COMPILE_OPTIONS "${cflags}" INCLUDE_DIRECTORIES "${includes}" COMPILE_DEFINITIONS "${defs}") + if(DEFINED outname_dpcpp) + set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}" + TARGET_DIRECTORY ${target} + PROPERTIES + GENERATED 1 + COMPILE_OPTIONS "${cflags}" + INCLUDE_DIRECTORIES "${includes}" + COMPILE_DEFINITIONS "${defs}") + endif(DEFINED outname_dpcpp) # make sure we produce .h before we build other .c in the target add_dependencies(${target} ptgpp_${target}.${outname}) # add to the target - target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + if(DEFINED outname_dpcpp) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}") + else(DEFINED outname_dpcpp) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + endif(DEFINED outname_dpcpp) get_target_property(_includes ${target} INCLUDE_DIRECTORIES) list(FIND _includes "${CMAKE_CURRENT_BINARY_DIR}" _i1) diff --git a/parsec/data.c b/parsec/data.c index 525b04bfd..6a98bc8d5 100644 --- a/parsec/data.c +++ b/parsec/data.c @@ -94,8 +94,7 @@ static void parsec_data_destruct(parsec_data_t* obj ) #endif /* defined(PARSEC_DEBUG_PARANOID) */ assert(obj->super.obj_reference_count > 1); parsec_data_copy_detach( obj, copy, i ); - if ( !(device->type & PARSEC_DEV_CUDA) - && !(device->type & PARSEC_DEV_HIP) ) { + if ( !(device->type & PARSEC_DEV_GPU) ){ /** * GPU copies are normally stored in LRU lists, and must be * destroyed by the release list to free the memory on the device diff --git a/parsec/include/parsec/parsec_options.h.in b/parsec/include/parsec/parsec_options.h.in index 44cabc508..b10875997 100644 --- a/parsec/include/parsec/parsec_options.h.in +++ b/parsec/include/parsec/parsec_options.h.in @@ -123,6 +123,8 @@ #cmakedefine PARSEC_HAVE_HWLOC #cmakedefine PARSEC_HAVE_PAPI #cmakedefine PARSEC_HAVE_CUDA +#cmakedefine PARSEC_HAVE_LEVEL_ZERO +#cmakedefine PARSEC_HAVE_DPCPP #cmakedefine PARSEC_HAVE_HIP #cmakedefine PARSEC_HAVE_OPENCL #cmakedefine PARSEC_HAVE_MPI diff --git a/parsec/interfaces/dtd/insert_function.c b/parsec/interfaces/dtd/insert_function.c index 2d6877f02..85ca2b48f 100644 --- a/parsec/interfaces/dtd/insert_function.c +++ b/parsec/interfaces/dtd/insert_function.c @@ -41,6 +41,9 @@ #if defined(PARSEC_HAVE_CUDA) #include "parsec/mca/device/cuda/device_cuda.h" #endif /* defined(PARSEC_HAVE_CUDA) */ +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero.h" +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ #include "parsec/mca/mca_repository.h" #include "parsec/constants.h" @@ -1494,6 +1497,9 @@ parsec_dtd_startup(parsec_context_t *context, // If CUDA is enabled, let the CUDA device activated for this // taskpool. if( PARSEC_DEV_CUDA == device->type ) continue; + // If LEVEL_ZERO is enabled, let the LEVEL_ZERO device activated for this + // taskpool. + if( PARSEC_DEV_LEVEL_ZERO == device->type ) continue; if( NULL != device->taskpool_register ) if( PARSEC_SUCCESS != device->taskpool_register(device, (parsec_taskpool_t *)tp)) { @@ -2334,6 +2340,12 @@ static parsec_hook_return_t parsec_dtd_gpu_task_submit(parsec_execution_stream_t gpu_task->stage_in = parsec_default_cuda_stage_in; gpu_task->stage_out = parsec_default_cuda_stage_out; return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index); +#endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + case PARSEC_DEV_LEVEL_ZERO: + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + return parsec_level_zero_kernel_scheduler(es, gpu_task, dev_index); #endif default: parsec_fatal("DTD scheduling on device type %d: this is not a valid GPU device type in this build", device->type); @@ -2404,8 +2416,10 @@ int parsec_dtd_task_class_add_chore(parsec_taskpool_t *tp, if(PARSEC_DEV_CUDA == device_type) { incarnations[i].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; - } - else { + } else if(PARSEC_DEV_LEVEL_ZERO == device_type) { + incarnations[i].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; + } else { dtd_tc->cpu_func_ptr = function; incarnations[i].hook = parsec_dtd_cpu_task_submit; } @@ -3289,8 +3303,11 @@ __parsec_dtd_taskpool_create_task(parsec_taskpool_t *tp, /* Special case for CUDA: we need an intermediate */ (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; - } - else { + } if( device_type == PARSEC_DEV_LEVEL_ZERO ) { + /* Special case for LEVEL ZERO: we need an intermediate */ + (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; + } else { /* Default case: the user-provided function is directly the hook to call */ (*incarnations)[0].hook = fpointer; // We can directly call the CPU hook dtd_tc->cpu_func_ptr = fpointer; diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf.h b/parsec/interfaces/ptg/ptg-compiler/jdf.h index e76029ec3..29c23bbee 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf.h @@ -99,6 +99,7 @@ typedef struct jdf_compiler_global_args { char *input; char *output_c; char *output_h; + char *output_dpcpp; char *output_o; char *funcid; jdf_warning_mask_t wmask; @@ -143,10 +144,17 @@ extern int jdfdebug; * We remember the line number in the JDF file where this external code was found */ typedef struct jdf_external_entry { - struct jdf_object_t super; + struct jdf_object_t super; + struct jdf_external_entry *next; + char *language; char *external_code; } jdf_external_entry_t; +typedef struct jdf_code_string { + char *language; + char *string; +} jdf_code_string_t; + /** A global is a variable name, optionally an expression to define it, * and a line number associated with it for error printing purposes */ diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index 1cf949484..9cce36695 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -3,7 +3,6 @@ * of Tennessee Research Foundation. All rights * reserved. */ - #include "parsec/parsec_config.h" #include @@ -32,10 +31,14 @@ extern const char *yyfilename; static FILE *cfile; static int cfile_lineno; +static FILE *dpcppfile = NULL; +static int dpcppfile_lineno = 0; static FILE *hfile; static int hfile_lineno; static const char *jdf_basename; static const char *jdf_cfilename; +static char *jdf_hfilename; +static char *jdf_dpcppfilename; /* Optional declarations of local functions */ static int jdf_expr_depends_on_symbol(const char *varname, const jdf_expr_t *expr); @@ -137,6 +140,43 @@ static void coutput(const char *format, ...) } } +#if defined(__GNUC__) +static void dpcpp_output(const char *format, ...) __attribute__((format(printf,1,2))); +#endif +static void dpcpp_output(const char *format, ...) +{ + va_list ap; + char *res; + int len; + + va_start(ap, format); + len = vasprintf(&res, format, ap); + va_end(ap); + + if( len == -1 ) { + fprintf(stderr, "Unable to ouptut a string: %s\n", strerror(errno)); + } else if( 0 < len ) { +#if (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) + char *start = res, *end; + while( NULL != (end = strchr(start, '\n'))) { + if( (end != start) && (end[-1] != '\r')) { + fwrite(start, (end - start), 1, dpcppfile); + fwrite("\r\n", 2, 1, dpcppfile); + } else { + fwrite(start, (end - start) + 1, 1, dpcppfile); + } + len -= (end - start) + 1; + start = end + 1; /* skip the current \n */ + } + fwrite(start, len, 1, dpcppfile); +#else + fwrite(res, len, 1, dpcppfile); +#endif /* (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) */ + dpcppfile_lineno += nblines(res); + free(res); + } +} + #if defined(__GNUC__) static void houtput(const char *format, ...) __attribute__((format(printf,1,2))); #endif @@ -1254,7 +1294,7 @@ static jdf_expr_t *jdf_expr_lv_next(jdf_expr_t *head, jdf_expr_t *cur) return head; } -static void jdf_coutput_prettycomment(char marker, const char *format, ...) +static char *jdf_prettycomment(string_arena_t *sa, char marker, const char *format, ...) { int ls, rs, i, length, vs; va_list ap, ap2; @@ -1294,15 +1334,17 @@ static void jdf_coutput_prettycomment(char marker, const char *format, ...) ls = (80 - length) / 2; rs = 80 - length - ls; } - coutput("/*"); + string_arena_init(sa); + string_arena_add_string(sa, "/*"); for(i = 0; i < 5; i++) - coutput("%c", marker); - coutput("%s%s", indent(ls/2), v); /* indent drop two spaces */ - coutput("%s", indent(rs/2)); /* dont merge these two calls. Read the comment on the indent function */ + string_arena_add_string(sa, "%c", marker); + string_arena_add_string(sa, "%s%s", indent(ls/2), v); /* indent drop two spaces */ + string_arena_add_string(sa, "%s", indent(rs/2)); /* dont merge these two calls. Read the comment on the indent function */ for(i = 0; i < 5; i++) - coutput("%c", marker); - coutput("*/\n\n"); + string_arena_add_string(sa, "%c", marker); + string_arena_add_string(sa, "*/\n\n"); free(v); + return string_arena_get_string(sa); } /** Structure Generators **/ @@ -1511,6 +1553,9 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) "#if defined(PARSEC_HAVE_CUDA)\n" "#include \"parsec/mca/device/cuda/device_cuda.h\"\n" "#endif /* defined(PARSEC_HAVE_CUDA) */\n" + "#if defined(PARSEC_HAVE_LEVEL_ZERO)\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */\n" "#if defined(PARSEC_HAVE_HIP)\n" "#include \"parsec/mca/device/hip/device_hip.h\"\n" "#endif /* defined(PARSEC_HAVE_HIP) */\n" @@ -1530,19 +1575,15 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) jdf_generate_predeclarations(jdf); } -static void jdf_generate_structure(jdf_t *jdf) +static void jdf_dump_internal_structure(string_arena_t *sa, jdf_t *jdf) { - int nbfunctions, need_profile = 0; - string_arena_t *sa1, *sa2; + int nbfunctions = 0; jdf_function_entry_t* f; jdf_param_list_t *pl; JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); - sa1 = string_arena_new(64); - sa2 = string_arena_new(64); - - coutput("#include \"%s.h\"\n\n" + string_arena_add_string(sa, "#include \"%s.h\"\n\n" "struct __parsec_%s_internal_taskpool_s {\n" " parsec_%s_taskpool_t super;\n" " volatile int32_t sync_point;\n" @@ -1550,30 +1591,64 @@ static void jdf_generate_structure(jdf_t *jdf) " parsec_task_t* startup_queue;\n", jdf_basename, jdf_basename, jdf_basename); - coutput(" /* The ranges to compute the hash key */\n"); + string_arena_add_string(sa, " /* The ranges to compute the hash key */\n"); for(f = jdf->functions; f != NULL; f = f->next) { if( 0 == (f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { for(pl = f->parameters; pl != NULL; pl = pl->next) { - coutput(" int %s_%s_range;\n", f->fname, pl->name); + string_arena_add_string(sa, " int %s_%s_range;\n", f->fname, pl->name); } } else { - coutput(" /* nothing for %s as it gets a user-defined make_key */\n", + string_arena_add_string(sa, " /* nothing for %s as it gets a user-defined make_key */\n", f->fname); } } - coutput(" /* The list of data repositories "); + string_arena_add_string(sa, " /* The list of data repositories "); for(f = jdf->functions; NULL != f; f = f->next) { if( 0 != function_has_data_output(f) ) { - coutput(" %s ", f->fname); + string_arena_add_string(sa, " %s ", f->fname); } } - coutput("*/\n"); + string_arena_add_string(sa, "*/\n"); if(nbfunctions != 0 ) { - coutput(" data_repo_t* repositories[%d];\n", nbfunctions ); + string_arena_add_string(sa, " data_repo_t* repositories[%d];\n", nbfunctions ); + } + + string_arena_add_string(sa, "};\n\n"); +} + +static int jdf_has_dpcpp_chore(const jdf_t *jdf, const char *fname) +{ + jdf_function_entry_t *f; + jdf_body_t* body; + jdf_def_list_t *type_property; + + for(f = jdf->functions; f != NULL; f = f->next) { + if( (NULL != fname) && strcmp(f->fname, fname) ) continue; + for(body = f->bodies; body != NULL; body = body->next) { + jdf_find_property(body->properties, "type", &type_property); + if( NULL != type_property && !strcmp(type_property->expr->jdf_var, "DPCPP")) + return 1; + } + if(NULL != fname) return 0; } + return 0; +} + +static void jdf_generate_structure(jdf_t *jdf) +{ + int nbfunctions, need_profile = 0; + string_arena_t *sa1, *sa2; + jdf_function_entry_t* f; + + JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); - coutput("};\n\n"); + sa1 = string_arena_new(64); + sa2 = string_arena_new(64); + + jdf_dump_internal_structure(sa1, jdf); + coutput("%s\n", string_arena_get_string(sa1)); + string_arena_init(sa1); for( f = jdf->functions; need_profile == 0 && NULL != f; f = f->next ) { /* If the profile property is ON then enable the profiling array */ @@ -1595,6 +1670,9 @@ static void jdf_generate_structure(jdf_t *jdf) if( 1 < strlen(string_arena_get_string(sa1)) ) { coutput("/* Globals */\n%s\n", string_arena_get_string(sa1)); } + if( 1 < strlen(string_arena_get_string(sa1)) && jdf_has_dpcpp_chore(jdf, NULL) ) { + dpcpp_output("/* Globals */\n%s\n", string_arena_get_string(sa1)); + } coutput("static inline int parsec_imin(int a, int b) { return (a <= b) ? a : b; };\n\n" "static inline int parsec_imax(int a, int b) { return (a >= b) ? a : b; };\n\n"); @@ -4101,7 +4179,8 @@ static void jdf_generate_one_function( const jdf_t *jdf, jdf_function_entry_t *f has_in_in_dep |= (fl->flow_flags & JDF_FLOW_HAS_IN_DEPS); } - jdf_coutput_prettycomment('*', "%s", f->fname); + coutput("%s", jdf_prettycomment(sa, '*', "%s", f->fname)); + string_arena_init(sa); prefix = (char*)malloc(strlen(f->fname) + strlen(jdf_basename) + 32); @@ -4805,7 +4884,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e string_arena_t *sa_range_multiplier = string_arena_new(64); jdf_variable_list_t *vl; expr_info_t info = EMPTY_EXPR_INFO; - int idx; if( !(f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { coutput("static inline parsec_key_t %s(const parsec_taskpool_t *tp, const parsec_assignment_t *as)\n" @@ -4828,7 +4906,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e info.sa = sa_range_multiplier; info.assignments = "assignment"; - idx = 0; for(vl = f->locals; vl != NULL; vl = vl->next) { string_arena_init(sa_range_multiplier); @@ -4860,7 +4937,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e */ coutput(" (void)%s;\n", vl->name); } - idx++; } string_arena_init(sa_range_multiplier); @@ -6125,20 +6201,20 @@ jdf_generate_code_flow_final_writes(const jdf_t *jdf, string_arena_free(sa); } -static void jdf_generate_code_dry_run_before(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_dry_run_before(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { (void)jdf; (void)f; - coutput("\n\n#if !defined(PARSEC_PROF_DRY_BODY)\n\n"); + output_fn("\n\n#if !defined(PARSEC_PROF_DRY_BODY)\n\n"); } -static void jdf_generate_code_dry_run_after(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_dry_run_after(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { (void)jdf; (void)f; - coutput("\n\n#endif /*!defined(PARSEC_PROF_DRY_BODY)*/\n\n"); + output_fn("\n\n#endif /*!defined(PARSEC_PROF_DRY_BODY)*/\n\n"); } static void jdf_generate_code_grapher_task_done(const jdf_t *jdf, const jdf_function_entry_t *f, const char* context_name) @@ -6153,7 +6229,7 @@ static void jdf_generate_code_grapher_task_done(const jdf_t *jdf, const jdf_func jdf_property_get_string(f->properties, JDF_PROP_UD_HASH_STRUCT_NAME, NULL), context_name, context_name, context_name); } -static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { string_arena_t *sa; sa = string_arena_new(64); @@ -6163,11 +6239,11 @@ static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf dump_dataflow_varname, NULL, "", " cache_buf_referenced(es->closest_cache, ", ");\n", ""); if( strlen(string_arena_get_string(sa)) ) { - coutput(" /** Cache Awareness Accounting */\n" - "#if defined(PARSEC_CACHE_AWARENESS)\n" - "%s);\n" - "#endif /* PARSEC_CACHE_AWARENESS */\n", - string_arena_get_string(sa)); + output_fn(" /** Cache Awareness Accounting */\n" + "#if defined(PARSEC_CACHE_AWARENESS)\n" + "%s);\n" + "#endif /* PARSEC_CACHE_AWARENESS */\n", + string_arena_get_string(sa)); } string_arena_free(sa); } @@ -6581,6 +6657,9 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, int di; int profile_on; char* output; + void (*gpu_output)(const char *format, ...); + int *gpu_lineno; + const char *gpu_filename; profile_on = profile_enabled(f->properties) && profile_enabled(body->properties); @@ -6588,6 +6667,16 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, char* dev_upper = strdup_upper(type_property->expr->jdf_var); char* dev_lower = strdup_lower(type_property->expr->jdf_var); + if(0 == strcmp(dev_lower, "dpcpp")) { + gpu_output = dpcpp_output; + gpu_lineno = &dpcppfile_lineno; + gpu_filename = jdf_dpcppfilename; + } else { + gpu_output = coutput; + gpu_lineno = &cfile_lineno; + gpu_filename = jdf_cfilename; + } + /* Get the dynamic function properties */ dyld = jdf_property_get_string(body->properties, "dyld", NULL); dyldtype = jdf_property_get_string(body->properties, "dyldtype", "void*"); @@ -6609,123 +6698,158 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, dump_string, NULL, "", " (void)", ";", ";\n")); /* Generate the kernel_submit structure and function */ - coutput("struct parsec_body_%s_%s_%s_s {\n" - " uint8_t index;\n" - " %sStream_t stream;\n" - " %s dyld_fn;\n" - "};\n" - "\n" - "static int %s_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" - " parsec_gpu_task_t *gpu_task,\n" - " parsec_gpu_exec_stream_t *gpu_stream )\n" - "{\n" - " %s *this_task = (%s *)gpu_task->ec;\n" - " parsec_device_%s_module_t *%s_device = (parsec_device_%s_module_t*)gpu_device;\n" - " parsec_%s_exec_stream_t *%s_stream = (parsec_%s_exec_stream_t*)gpu_stream;\n" - " __parsec_%s_internal_taskpool_t *__parsec_tp = (__parsec_%s_internal_taskpool_t *)this_task->taskpool;\n" - " struct parsec_body_%s_%s_%s_s parsec_body = { %s_device->%s_index, %s_stream->%s_stream, NULL };\n" - "%s\n" - " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)parsec_body; (void)%s_device; (void)%s_stream;\n", - dev_lower, jdf_basename, f->fname, - dev_lower, - dyldtype, - dev_lower, jdf_basename, f->fname, - parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), - dev_lower, dev_lower, dev_lower, - dev_lower, dev_lower, dev_lower, - jdf_basename, jdf_basename, - dev_lower, jdf_basename, f->fname, dev_lower, dev_lower, dev_lower, dev_lower, - string_arena_get_string( sa3 ), - dev_lower, dev_lower); + if( 0 == strcmp(dev_lower, "dpcpp") ) { + dpcpp_output("extern \"C\" {\n" + " int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream);\n" + "}\n" + "\n" + "int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream )\n" + "{\n" + " %s *this_task = reinterpret_cast<%s *>(gpu_task->ec);\n" + " parsec_device_level_zero_module_t *level_zero_device = reinterpret_cast(gpu_device);\n" + " parsec_level_zero_exec_stream_t *level_zero_stream = reinterpret_cast(gpu_stream);\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = reinterpret_cast<__parsec_%s_internal_taskpool_t *>(this_task->taskpool);\n" + " parsec_sycl_wrapper_platform_t *parsec_sycl_platform = level_zero_device->driver->swp;\n" + " parsec_sycl_wrapper_device_t *parsec_sycl_device = level_zero_device->swd;\n" + " parsec_sycl_wrapper_queue_t *parsec_sycl_queue = level_zero_stream->swq;\n" + "\n" + "%s\n" + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)level_zero_device; (void)level_zero_stream; (void)parsec_sycl_platform; (void)parsec_sycl_device; (void)parsec_sycl_queue;\n", + jdf_basename, f->fname, + jdf_basename, f->fname, + parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), + jdf_basename, jdf_basename, + string_arena_get_string( sa3 )); + } else { + coutput("struct parsec_body_%s_%s_%s_s {\n" + " uint8_t index;\n" + " %sStream_t stream;\n" + " %s dyld_fn;\n" + "};\n" + "\n" + "static int %s_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream )\n" + "{\n" + " %s *this_task = (%s *)gpu_task->ec;\n" + " parsec_device_%s_module_t *%s_device = (parsec_device_%s_module_t*)gpu_device;\n" + " parsec_%s_exec_stream_t *%s_stream = (parsec_%s_exec_stream_t*)gpu_stream;\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = (__parsec_%s_internal_taskpool_t *)this_task->taskpool;\n" + " struct parsec_body_%s_%s_%s_s parsec_body = { %s_device->%s_index, %s_stream->%s_stream, NULL };\n" + "%s\n" + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)parsec_body; (void)%s_device; (void)%s_stream;\n", + dev_lower, jdf_basename, f->fname, + dev_lower, + dyldtype, + dev_lower, jdf_basename, f->fname, + parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), + dev_lower, dev_lower, dev_lower, + dev_lower, dev_lower, dev_lower, + jdf_basename, jdf_basename, + dev_lower, jdf_basename, f->fname, dev_lower, dev_lower, dev_lower, dev_lower, + string_arena_get_string( sa3 ), + dev_lower, dev_lower); + } ai2.sa = sa2; ai2.where = "out"; output = UTIL_DUMP_LIST(sa, f->dataflow, next, dump_data_initialization_from_data_array, &ai2, "", "", "", ""); if( 0 != strlen(output) ) { - coutput(" /** Declare the variables that will hold the data, and all the accounting for each */\n" - "%s\n", - output); + gpu_output(" /** Declare the variables that will hold the data, and all the accounting for each */\n" + "%s\n", + output); } /** * Generate code for the simulation. */ - coutput(" /** Update starting simulation date */\n" - "#if defined(PARSEC_SIM)\n" - " {\n" - " this_task->sim_exec_date = 0;\n"); + gpu_output(" /** Update starting simulation date */\n" + "#if defined(PARSEC_SIM)\n" + " {\n" + " this_task->sim_exec_date = 0;\n"); for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { if(fl->flow_flags & JDF_FLOW_TYPE_CTL) continue; /* control flow, nothing to store */ - coutput(" data_repo_entry_t *e%s = this_task->data._f_%s.source_repo_entry;\n" - " if( (NULL != e%s) && (e%s->sim_exec_date > this_task->sim_exec_date) )\n" - " this_task->sim_exec_date = e%s->sim_exec_date;\n", - fl->varname, fl->varname, - fl->varname, fl->varname, - fl->varname); - } - coutput(" if( this_task->task_class->sim_cost_fct != NULL ) {\n" - " this_task->sim_exec_date += this_task->task_class->sim_cost_fct(this_task);\n" - " }\n" - " if( es->largest_simulation_date < this_task->sim_exec_date )\n" - " es->largest_simulation_date = this_task->sim_exec_date;\n" - " }\n" - "#endif\n"); - - jdf_generate_code_cache_awareness_update(jdf, f); - - coutput("#if defined(PARSEC_DEBUG_NOISIER)\n" - " {\n" - " char tmp[MAX_TASK_STRLEN];\n" - " PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, \"GPU[%%s]:\\tEnqueue on device %%s priority %%d\", gpu_device->super.name, \n" - " parsec_task_snprintf(tmp, MAX_TASK_STRLEN, (parsec_task_t *)this_task),\n" - " this_task->priority );\n" - " }\n" - "#endif /* defined(PARSEC_DEBUG_NOISIER) */\n"); - - jdf_generate_code_dry_run_before(jdf, f); - jdf_coutput_prettycomment('-', "%s BODY", f->fname); + gpu_output(" data_repo_entry_t *e%s = this_task->data._f_%s.source_repo_entry;\n" + " if( (NULL != e%s) && (e%s->sim_exec_date > this_task->sim_exec_date) )\n" + " this_task->sim_exec_date = e%s->sim_exec_date;\n", + fl->varname, fl->varname, + fl->varname, fl->varname, + fl->varname); + } + gpu_output(" if( this_task->task_class->sim_cost_fct != NULL ) {\n" + " this_task->sim_exec_date += this_task->task_class->sim_cost_fct(this_task);\n" + " }\n" + " if( es->largest_simulation_date < this_task->sim_exec_date )\n" + " es->largest_simulation_date = this_task->sim_exec_date;\n" + " }\n" + "#endif\n"); + + jdf_generate_code_cache_awareness_update(jdf, f, gpu_output); + + gpu_output("#if defined(PARSEC_DEBUG_NOISIER)\n" + " {\n" + " char tmp[MAX_TASK_STRLEN];\n" + " PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, \"GPU[%%s]:\\tEnqueue on device %%s priority %%d\", gpu_device->super.name, \n" + " parsec_task_snprintf(tmp, MAX_TASK_STRLEN, (parsec_task_t *)this_task),\n" + " this_task->priority );\n" + " }\n" + "#endif /* defined(PARSEC_DEBUG_NOISIER) */\n"); + + jdf_generate_code_dry_run_before(jdf, f, gpu_output); + gpu_output("%s", jdf_prettycomment(sa, '-', "%s BODY", f->fname)); if( profile_on ) { - coutput("#if defined(PARSEC_PROF_TRACE)\n" - " if(gpu_stream->prof_event_track_enable) {\n" - " PARSEC_TASK_PROF_TRACE(gpu_stream->profiling,\n" - " PARSEC_PROF_FUNC_KEY_START(this_task->taskpool,\n" - " this_task->task_class->task_class_id),\n" - " (parsec_task_t*)this_task, 1);\n" - " gpu_task->prof_key_end = PARSEC_PROF_FUNC_KEY_END(this_task->taskpool,\n" - " this_task->task_class->task_class_id);\n" - " gpu_task->prof_event_id = this_task->task_class->key_functions->\n" - " key_hash(this_task->task_class->make_key(this_task->taskpool, ((parsec_task_t*)this_task)->locals), NULL);\n" - " gpu_task->prof_tp_id = this_task->taskpool->taskpool_id;\n" - " }\n" - "#endif /* PARSEC_PROF_TRACE */\n"); + gpu_output("#if defined(PARSEC_PROF_TRACE)\n" + " if(gpu_stream->prof_event_track_enable) {\n" + " PARSEC_TASK_PROF_TRACE(gpu_stream->profiling,\n" + " PARSEC_PROF_FUNC_KEY_START(this_task->taskpool,\n" + " this_task->task_class->task_class_id),\n" + " (parsec_task_t*)this_task, 1);\n" + " gpu_task->prof_key_end = PARSEC_PROF_FUNC_KEY_END(this_task->taskpool,\n" + " this_task->task_class->task_class_id);\n" + " gpu_task->prof_event_id = this_task->task_class->key_functions->\n" + " key_hash(this_task->task_class->make_key(this_task->taskpool, ((parsec_task_t*)this_task)->locals), NULL);\n" + " gpu_task->prof_tp_id = this_task->taskpool->taskpool_id;\n" + " }\n" + "#endif /* PARSEC_PROF_TRACE */\n"); } if ( NULL != dyld ) { - coutput(" /* Pointer to dynamic gpu function */\n" - " {\n" - " int chore_idx = 0;\n" - " for ( ; PARSEC_DEV_NONE != this_task->task_class->incarnations[chore_idx].type; ++chore_idx) {\n" - " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_%s) break;\n" - " }\n" - " /* The void* cast prevents the compiler from complaining about the type change */\n" - " parsec_body.dyld_fn = (%s)(void*)this_task->task_class->incarnations[chore_idx].dyld_fn;\n" - " }\n\n", - dev_upper, - dyldtype ); + gpu_output(" /* Pointer to dynamic gpu function */\n" + " {\n" + " int chore_idx = 0;\n" + " for ( ; PARSEC_DEV_NONE != this_task->task_class->incarnations[chore_idx].type; ++chore_idx) {\n" + " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_%s) break;\n" + " }\n" + " /* The void* cast prevents the compiler from complaining about the type change */\n" + " parsec_body.dyld_fn = (%s)(void*)this_task->task_class->incarnations[chore_idx].dyld_fn;\n" + " }\n\n", + dev_upper, + dyldtype ); + } + + gpu_output("%s\n", body->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { + gpu_output("#line %d \"%s\"\n", (*gpu_lineno)+1, gpu_filename); } + gpu_output("%s\n", jdf_prettycomment(sa, '-', "END OF %s BODY", f->fname)); + jdf_generate_code_dry_run_after(jdf, f, gpu_output); + gpu_output(" return PARSEC_HOOK_RETURN_DONE;\n" + "}\n\n"); - coutput("%s\n", body->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { - coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); + if(0 == strcmp(dev_lower, "dpcpp")) { + coutput("extern int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream);\n" + "\n", jdf_basename, f->fname); } - jdf_coutput_prettycomment('-', "END OF %s BODY", f->fname); - jdf_generate_code_dry_run_after(jdf, f); - coutput(" return PARSEC_HOOK_RETURN_DONE;\n" - "}\n\n"); /* Generate the hook device */ coutput("static int %s_%s(parsec_execution_stream_t *es, %s *this_task)\n" @@ -6949,10 +7073,11 @@ static void jdf_generate_code_hook(const jdf_t *jdf, } if( NULL != type_property) { type_upper = strdup_upper(type_property->expr->jdf_var); - coutput("#if defined(PARSEC_HAVE_%s)\n", type_upper); + coutput("#if defined(PARSEC_HAVE_%s) /* thomas */\n", type_upper); if (!strcasecmp(type_property->expr->jdf_var, "cuda") - || !strcasecmp(type_property->expr->jdf_var, "hip")) { + || !strcasecmp(type_property->expr->jdf_var, "hip") + || !strcasecmp(type_property->expr->jdf_var, "dpcpp")) { jdf_generate_code_hook_gpu(jdf, f, body, name); goto hook_end_block; } @@ -7020,7 +7145,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, if ((NULL == type_property) || (!strcmp(type_property->expr->jdf_var, "RECURSIVE"))) { coutput(" /** Transfer the ownership to the CPU */\n" - "#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP)\n"); + "#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO) || defined(PARSEC_HAVE_HIP)\n"); for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { /* Update the ownership of read/write data */ @@ -7038,19 +7163,22 @@ static void jdf_generate_code_hook(const jdf_t *jdf, ((fl->flow_flags & JDF_FLOW_TYPE_WRITE) ? "PARSEC_FLOW_ACCESS_RW" : "PARSEC_FLOW_ACCESS_READ") : "PARSEC_FLOW_ACCESS_WRITE"))); } } - coutput("#endif /* defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) */\n"); + coutput("#endif /* defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO) || defined(PARSEC_HAVE_HIP) */\n"); } - jdf_generate_code_cache_awareness_update(jdf, f); + jdf_generate_code_cache_awareness_update(jdf, f, coutput); - jdf_generate_code_dry_run_before(jdf, f); - jdf_coutput_prettycomment('-', "%s BODY", f->fname); + jdf_generate_code_dry_run_before(jdf, f, coutput); + coutput("%s", jdf_prettycomment(sa, '-', "%s BODY", f->fname)); + string_arena_init(sa); coutput("%s\n", body->external_code); if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); } - jdf_coutput_prettycomment('-', "END OF %s BODY", f->fname); - jdf_generate_code_dry_run_after(jdf, f); + coutput("%s", jdf_prettycomment(sa, '-', "END OF %s BODY", f->fname)); + string_arena_init(sa); + + jdf_generate_code_dry_run_after(jdf, f, coutput); coutput(" return PARSEC_HOOK_RETURN_DONE;\n" "}\n"); @@ -7058,7 +7186,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, string_arena_free(sa2); hook_end_block: - if( NULL != type_property) { + if( NULL != type_upper ) { coutput("#endif /* defined(PARSEC_HAVE_%s) */\n", type_upper); free(type_upper); } @@ -7667,7 +7795,7 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, string_arena_t *sa_tmp_type_r = string_arena_new(256); string_arena_t *sa_temp_r = string_arena_new(1024); - int depnb, last_datatype_idx; + int last_datatype_idx; assignment_info_t ai; expr_info_t info = EMPTY_EXPR_INFO; int nb_open_ldef; @@ -7717,7 +7845,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, for(fl = f->dataflow; fl != NULL; fl = fl->next) { flowempty = 1; flowtomem = 0; - depnb = 0; last_datatype_idx = -1; string_arena_init(sa_coutput); string_arena_init(sa_deps); @@ -7951,8 +8078,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, jdf_dump_context_assignment(sa1, jdf, f, fl, string_arena_get_string(sa_ontask), dl->guard->calltrue, dl, JDF_OBJECT_LINENO(dl), " ", "nc")); - depnb++; - string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -7973,7 +8098,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, "\n"); } } else { - depnb++; string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -8004,7 +8128,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, } break; } - depnb++; /* Dump the previous dependencies */ OUTPUT_PREV_DEPS((1U << dl->dep_index), sa_datatype, sa_deps); @@ -8432,14 +8555,41 @@ int jdf_force_termdet_dynamic(jdf_t* jdf) #include #endif -int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, jdf_t *jdf) +int jdf2c(const char *output_c, const char *output_h, + const char *output_dpcpp, const char *_jdf_basename, jdf_t *jdf) { int ret = 0; jdf_cfilename = output_c; + jdf_hfilename = strdup(output_h); + jdf_dpcppfilename = strdup(output_dpcpp); jdf_basename = _jdf_basename; cfile = NULL; hfile = NULL; + dpcppfile = NULL; + + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa2; + sa2 = string_arena_new(64); + dpcppfile = fopen(jdf_dpcppfilename, "w"); + if( dpcppfile == NULL ) { + fprintf(stderr, "unable to create %s: %s\n", jdf_dpcppfilename, strerror(errno)); + ret = -1; + goto err; + } + + dpcpp_output("#include \"parsec.h\"\n" + "#include \"level_zero/ze_api.h\"\n" + "#include \"sycl/ext/oneapi/backend/level_zero.hpp\"\n" + "#include \"parsec/parsec_internal.h\"\n" + "#include \"parsec/execution_stream.h\"\n" + "#include \"parsec/mca/device/device.h\"\n" + "#include \"parsec/mca/device/device_gpu.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero_dpcpp.h\"\n" + "\n"); + string_arena_free(sa2); + } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) /* When we apply indent/awk to the output of jdf2c, we need to make @@ -8527,6 +8677,7 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, cfile_lineno = 1; hfile_lineno = 1; + dpcppfile_lineno = 1; /** * Now generate the code. @@ -8543,20 +8694,38 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, /** * Dump the prologue section */ - if( NULL != jdf->prologue ) { - coutput("%s", jdf->prologue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->prologue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in prologue: code ignored", ent->language); + } + } + + /* Now we can include the header generated to define the datatypes */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa = string_arena_new(64); + jdf_dump_internal_structure(sa, jdf); + dpcpp_output("%s\n" + "typedef struct __parsec_%s_internal_taskpool_s __parsec_%s_internal_taskpool_t;\n" + "\n", + string_arena_get_string(sa), + jdf_basename, jdf_basename); + string_arena_free(sa); } /* Dump references to arenas_datatypes array */ struct jdf_name_list* g; - int datatype_index = 0; for( g = jdf->datatypes; NULL != g; g = g->next ) { coutput("#define PARSEC_%s_%s_ADT (&__parsec_tp->super.arenas_datatypes[PARSEC_%s_%s_ADT_IDX])\n", jdf_basename, g->name, jdf_basename, g->name); - datatype_index++; } jdf_generate_structure(jdf); @@ -8583,13 +8752,35 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, free_name_placeholders(); + /* Sometimes (typically because of a try {} catch block), sycl re-includes some definitions at the end of the file + * these definitions can be polluted by macros. Undefine the macros. */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa1; + sa1 = string_arena_new(64); + + UTIL_DUMP_LIST_FIELD(sa1, jdf->globals, next, name, + dump_string, NULL, + "", "#undef ", "\n", "\n"); + + dpcpp_output("%s\n", string_arena_get_string(sa1)); + string_arena_free(sa1); + } + /** * Dump all the epilogue sections */ - if( NULL != jdf->epilogue ) { - coutput("%s", jdf->epilogue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n",cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->epilogue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in epilogue: code ignored", ent->language); + } } /** @@ -8603,10 +8794,22 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, err: if( NULL != cfile ) { fclose(cfile); + cfile = NULL; } if( NULL != hfile ) { fclose(hfile); + hfile = NULL; + } + + if( NULL != dpcppfile ) { + fclose(dpcppfile); + dpcppfile = NULL; + } + + if(NULL != jdf_dpcppfilename) { + free(jdf_dpcppfilename); + jdf_dpcppfilename = NULL; } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h index 4d02c7ada..769b98ad3 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h @@ -12,6 +12,9 @@ int jdf_optimize( jdf_t* jdf ); int jdf_force_termdet_dynamic(jdf_t* jdf); -int jdf2c(const char *output_c, const char *output_h, const char *_basename, jdf_t *jdf); +int jdf_force_termdet_dynamic(jdf_t* jdf); + +int jdf2c(const char *output_c, const char *output_h, + const char *output_dpcpp, const char *_basename, jdf_t *jdf); #endif /* _jdf2c_h */ diff --git a/parsec/interfaces/ptg/ptg-compiler/main.c b/parsec/interfaces/ptg/ptg-compiler/main.c index ab705d00f..8168cd2b1 100644 --- a/parsec/interfaces/ptg/ptg-compiler/main.c +++ b/parsec/interfaces/ptg/ptg-compiler/main.c @@ -29,6 +29,7 @@ static jdf_compiler_global_args_t DEFAULTS = { .input = "-", .output_c = "a.c", .output_h = "a.h", + .output_dpcpp = "a", .output_o = "a.o", .funcid = "a", .wmask = JDF_ALL_WARNINGS, @@ -56,10 +57,11 @@ static void usage(void) " --debug|-d Enable debug output\n" " --input|-i Input File (JDF) (default '%s')\n" " --output|-o Set the BASE name for .c, .h, .o and function name (no default).\n" - " Changing this value has precendence over the defaults of\n" + " Changing this value has precedence over the defaults of\n" " --output-c, --output-h, and --function-name\n" " --output-c|-C Set the name of the .c output file (default '%s' or BASE.c)\n" " --output-h|-H Set the name of the .h output file (default '%s' or BASE.h)\n" + " --output-dpcpp|-S Set the based name of the .dpcpp.* output files (default '%s' or BASE.dpcpp)\n" " --function-name|-f Set the unique identifier of the generated function\n" " The generated function will be called PaRSEC__new\n" " (default %s)\n" @@ -96,6 +98,7 @@ static void usage(void) DEFAULTS.input, DEFAULTS.output_c, DEFAULTS.output_h, + DEFAULTS.output_dpcpp, DEFAULTS.funcid, (DEFAULTS.dep_management == DEP_MANAGEMENT_INDEX_ARRAY ? DEP_MANAGEMENT_INDEX_ARRAY_STRING : (DEFAULTS.dep_management == DEP_MANAGEMENT_DYNAMIC_HASH_TABLE ? DEP_MANAGEMENT_DYNAMIC_HASH_TABLE_STRING : @@ -171,6 +174,7 @@ static void parse_args(int argc, char *argv[]) char *O = NULL; char *h = NULL; char *o = NULL; + char *s = NULL; char *f = NULL; struct option longopts[] = { @@ -179,6 +183,7 @@ static void parse_args(int argc, char *argv[]) { "output-c", required_argument, NULL, 'C' }, { "output-h", required_argument, NULL, 'H' }, { "output-o", required_argument, NULL, 'O' }, + { "output-dpcpp", required_argument, NULL, 'S' }, { "output", required_argument, NULL, 'o' }, { "function-name", required_argument, NULL, 'f' }, { "Wmasked", no_argument, &wmasked, 1 }, @@ -204,7 +209,7 @@ static void parse_args(int argc, char *argv[]) print_jdf_line = !DEFAULTS.noline; - while( (ch = getopt_long(argc, argv, "dDi:C:H:o:f:hEsIO:M:I:", longopts, NULL)) != -1) { + while( (ch = getopt_long(argc, argv, "dDi:C:H:o:S:f:hEsIO:M:I:", longopts, NULL)) != -1) { switch(ch) { case 'd': yydebug = 1; @@ -238,6 +243,11 @@ static void parse_args(int argc, char *argv[]) free( o ); o = strdup(optarg); break; + case 'S': + if( NULL != s ) + free( s ); + s = strdup(optarg); + break; case 'f': if( NULL != f ) free( f ); @@ -319,6 +329,15 @@ static void parse_args(int argc, char *argv[]) JDF_COMPILER_GLOBAL_ARGS.output_o = DEFAULTS.output_o; } + if( NULL != s ) { + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(s); + } else { + if(NULL != o) { + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(o); + } else + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = DEFAULTS.output_dpcpp; + } + if( NULL == c) { if( NULL != o ) { JDF_COMPILER_GLOBAL_ARGS.output_c = (char*)malloc(strlen(o) + 3); @@ -361,6 +380,8 @@ static void parse_args(int argc, char *argv[]) free(h); if( NULL != o ) free(o); + if( NULL != s ) + free(s); if( print_compile_cmd ) { /* print the compilation options used to compile the preprocessed output */ @@ -433,6 +454,7 @@ int main(int argc, char *argv[]) if( jdf2c(JDF_COMPILER_GLOBAL_ARGS.output_c, JDF_COMPILER_GLOBAL_ARGS.output_h, + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp, JDF_COMPILER_GLOBAL_ARGS.funcid, ¤t_jdf) < 0 ) { return 1; diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.l b/parsec/interfaces/ptg/ptg-compiler/parsec.l index edb6a4e0c..cbf015a3e 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.l +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.l @@ -110,7 +110,8 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} /* find the end of the language */ for(lend = lstart + 1; yytext[lend] != '"'; lend++) /* nothing */; - assert( (yytext[lstart] == 'C') && ((lend-lstart) == 1) ); + YYLVAL(code).language = (char*)calloc(lend-lstart+1, 1); + memcpy(YYLVAL(code).language, &yytext[lstart], lend-lstart); /* Search for the start of the body */ for(bstart = lend + 1; yytext[bstart] != '{'; bstart++ ) /* nothing */; @@ -120,14 +121,15 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} for( lend = 0; lend <= bstart; lend++ ) if( yytext[lend] == '\n' ) skip_line++; current_lineno += skip_line; - YYLVAL(string) = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); + YYLVAL(code).string = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); ECHO; return EXTERN_DECL; } (inline_c)?{WHITE}*\%c?\{([^\%]|(\%[^\}]))+\%\} { int start; for(start = 0; yytext[start] != '{'; start++) /* nothing */ ; start++; - YYLVAL(string) = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).string = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).language = strdup("C"); ECHO; return EXTERN_DECL; } SIMCOST { ECHO; return SIMCOST; } diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.y b/parsec/interfaces/ptg/ptg-compiler/parsec.y index 92393ca41..0478bdbe8 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.y +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.y @@ -271,6 +271,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %union { int number; + jdf_code_string_t code; char* string; jdf_expr_operand_t expr_op; jdf_external_entry_t *external_code; @@ -291,7 +292,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, }; %glr-parser -%expect 5 +%expect 6 %type function %type param_list @@ -323,7 +324,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %type bodies %type VAR -%type EXTERN_DECL +%type EXTERN_DECL %type BODY_END %type ARROW %type PROPERTIES_ON @@ -372,23 +373,28 @@ jdf_file: prologue jdf epilogue current_jdf.epilogue = $3; } ; -prologue: EXTERN_DECL +prologue: EXTERN_DECL prologue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; } ; -epilogue: EXTERN_DECL +epilogue: EXTERN_DECL epilogue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; - } + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; @@ -1325,7 +1331,7 @@ expr_simple: expr_simple EQUAL expr_simple { $$ = new(jdf_expr_t); $$->op = JDF_C_CODE; - $$->jdf_c_code.code = $1; + $$->jdf_c_code.code = $1.string; $$->jdf_type = PARSEC_RETURN_TYPE_INT32; $$->local_variables = current_locally_bound_variables; $$->scope = -1; diff --git a/parsec/mca/device/CMakeLists.txt b/parsec/mca/device/CMakeLists.txt index ad676ee55..f2eb1af08 100644 --- a/parsec/mca/device/CMakeLists.txt +++ b/parsec/mca/device/CMakeLists.txt @@ -1,8 +1,8 @@ set(MCA_${COMPONENT}_SOURCES mca/device/device.c) -if(PARSEC_HAVE_CUDA OR PARSEC_HAVE_HIP) +if(PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO OR PARSEC_HAVE_HIP) list(APPEND MCA_${COMPONENT}_SOURCES mca/device/device_gpu.c mca/device/transfer_gpu.c) -endif() +endif(PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO OR PARSEC_HAVE_HIP ) set_property(TARGET parsec APPEND PROPERTY diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index 793032ca4..b7c373e5f 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -88,7 +88,7 @@ int parsec_get_best_device( parsec_task_t* this_task, double ratio ) /* Make sure data_in is not NULL */ if( NULL == this_task->data[i].data_in ) continue; /* And that we have a data (aka it is not NEW) */ - if( NULL == this_task->data[i].source_repo_entry ) continue; + if( 0 && NULL == this_task->data[i].source_repo_entry ) continue; /* Data is updated by the task, and we try to minimize the data movements */ if( (NULL != this_task->task_class->out[i]) && diff --git a/parsec/mca/device/device.h b/parsec/mca/device/device.h index 25c1fa3a1..66f34ff42 100644 --- a/parsec/mca/device/device.h +++ b/parsec/mca/device/device.h @@ -63,12 +63,15 @@ typedef struct parsec_device_base_component_2_0_0 parsec_device_base_component_t #define PARSEC_DEV_CPU ((uint8_t)(1 << 0)) #define PARSEC_DEV_RECURSIVE ((uint8_t)(1 << 1)) #define PARSEC_DEV_CUDA ((uint8_t)(1 << 2)) -#define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 3)) -#define PARSEC_DEV_OPENCL ((uint8_t)(1 << 4)) -#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 5)) -#define PARSEC_DEV_HIP ((uint8_t)(1 << 6)) -#define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x3f) -#define PARSEC_DEV_ALL ((uint8_t) 0x3f) +#define PARSEC_DEV_LEVEL_ZERO ((uint8_t)(1 << 3)) +#define PARSEC_DEV_DPCPP ((uint8_t)(1 << 3)) +#define PARSEC_DEV_HIP ((uint8_t)(1 << 4)) +#define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 5)) +#define PARSEC_DEV_OPENCL ((uint8_t)(1 << 6)) +#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 7)) +#define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x7f) +#define PARSEC_DEV_ALL ((uint8_t) 0x7f) +#define PARSEC_DEV_GPU (PARSEC_DEV_CUDA|PARSEC_DEV_LEVEL_ZERO|PARSEC_DEV_HIP) #define PARSEC_DEV_MAX_NB_TYPE (7) #define PARSEC_DEV_DATA_ADVICE_PREFETCH ((int) 0x01) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 9fb543fa4..710fb9219 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -72,9 +72,9 @@ parsec_gpu_check_space_needed(parsec_device_gpu_module_t *gpu_device, return space_needed; } -#if defined(PARSEC_PROF_TRACE) void parsec_gpu_init_profiling(void) { +#if defined(PARSEC_PROF_TRACE) if(parsec_gpu_profiling_initiated == 0) { parsec_profiling_add_dictionary_keyword("cuda", "fill:#66ff66", 0, NULL, @@ -100,8 +100,8 @@ void parsec_gpu_init_profiling(void) &parsec_gpu_use_memory_key_start, &parsec_gpu_use_memory_key_end); parsec_gpu_profiling_initiated = 1; } -} #endif +} void parsec_gpu_enable_debug(void) { diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index 129bd2b03..6c0d83aa2 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -227,8 +227,6 @@ extern int parsec_gpu_prefetch_key_start; extern int parsec_gpu_prefetch_key_end; extern int parsec_device_gpu_one_profiling_stream_per_gpu_stream; -void parsec_gpu_init_profiling(void); - typedef struct { uint64_t size; uint64_t data_key; @@ -237,6 +235,7 @@ typedef struct { #define PARSEC_DEVICE_GPU_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" #endif /* defined(PROFILING) */ +void parsec_gpu_init_profiling(void); void dump_exec_stream(parsec_gpu_exec_stream_t* exec_stream); void dump_GPU_state(parsec_device_gpu_module_t* gpu_device); diff --git a/parsec/mca/device/level_zero/ValidateModule.CMake b/parsec/mca/device/level_zero/ValidateModule.CMake new file mode 100644 index 000000000..1c2d08bd7 --- /dev/null +++ b/parsec/mca/device/level_zero/ValidateModule.CMake @@ -0,0 +1,37 @@ +# For now assume that the upper level did the CUDA search and that all +# necessary dependencies on CUDA have been correctly setup + +if( PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP ) + SET(MCA_${COMPONENT}_${MODULE} ON) + FILE(GLOB MCA_${COMPONENT}_${MODULE}_SOURCES ${MCA_BASE_DIR}/${COMPONENT}/${MODULE}/[^\\.]*.c) + ADD_CUSTOM_COMMAND(OUTPUT ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero + COMMAND ${DPCPP_EXECUTABLE} + -I${LEVEL_ZERO_INCLUDE_DIR} + -I${SYCL_INCLUDE_DIR} + -I${PROJECT_SOURCE_DIR} + -I${PROJECT_SOURCE_DIR}/parsec/include + -I${PROJECT_BINARY_DIR}/parsec/include + -I${PROJET_BINARY_DIR} + -Wall -fpic -g -c ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp) + SET_SOURCE_FILES_PROPERTIES( + ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + PROPERTIES + EXTERNAL_OBJECT true + GENERATED true) + add_library(parsec-dpcpp ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o) + target_link_libraries(parsec PRIVATE parsec-dpcpp ${SYCL_LIBRARY}) + install(TARGETS parsec-dpcpp + EXPORT parsec-targets + DESTINATION ${PARSEC_INSTALL_LIBDIR}) + SET(MCA_${COMPONENT}_${MODULE}_CONSTRUCTOR "${COMPONENT}_${MODULE}_static_component") + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_internal.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_dpcpp.h + DESTINATION include/parsec/mca/device/level_zero ) +else (PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) + MESSAGE(STATUS "Module ${MODULE} not selectable: does not have LEVEL_ZERO (${PARSEC_HAVE_LEVEL_ZERO}) or DPCPP (${PARSEC_HAVE_DPCPP})") + SET(MCA_${COMPONENT}_${MODULE} OFF) +endif(PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h new file mode 100644 index 000000000..95d791926 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED + + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/device.h" +#include "parsec/mca/device/device_gpu.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" + +#include + +BEGIN_C_DECLS + +struct parsec_level_zero_task_s; +typedef struct parsec_level_zero_task_s parsec_level_zero_task_t; + +struct parsec_level_zero_exec_stream_s; +typedef struct parsec_level_zero_exec_stream_s parsec_level_zero_exec_stream_t; + +struct parsec_device_level_zero_module_s; +typedef struct parsec_device_level_zero_module_s parsec_device_level_zero_module_t; + +struct parsec_device_level_zero_driver_s; +typedef struct parsec_device_level_zero_driver_s parsec_device_level_zero_driver_t; + +struct parsec_level_zero_workspace_s; +typedef struct parsec_level_zero_workspace_s parsec_level_zero_workspace_t; + +extern parsec_device_base_component_t parsec_device_level_zero_component; + +struct parsec_level_zero_task_s { + parsec_gpu_task_t super; +}; + +struct parsec_device_level_zero_driver_s { + ze_driver_handle_t ze_driver; + ze_context_handle_t ze_context; + uint32_t ref_count; + parsec_sycl_wrapper_platform_t *swp; +}; + +struct parsec_device_level_zero_module_s { + parsec_device_gpu_module_t super; + uint8_t level_zero_index; + parsec_device_level_zero_driver_t *driver; + ze_device_handle_t ze_device; + parsec_sycl_wrapper_device_t *swd; +}; + +PARSEC_OBJ_CLASS_DECLARATION(parsec_device_level_zero_module_t); + +struct parsec_level_zero_exec_stream_s { + parsec_gpu_exec_stream_t super; + /* There is exactly one task per active fence (max_fences being the uppoer bound). + * Upon fence completion the complete_stage function associated with the task is + * called, and this will decide what is going on next with the task. If the task + * remains in the system the function is supposed to update it. + */ + ze_fence_handle_t *fences; + ze_command_list_handle_t *command_lists; + ze_command_queue_handle_t level_zero_cq; + parsec_sycl_wrapper_queue_t *swq; +}; + + +/** +* Progress +*/ +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on GPU fences, + * such a fence indicates that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ); +parsec_hook_return_t +parsec_dpcpp_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ); + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ +/** + * Overload the default data_copy_t with a GPU specialized type + */ +typedef parsec_data_copy_t parsec_gpu_data_copy_t; + +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); +int +parsec_default_dpcpp_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); +int +parsec_default_dpcpp_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + +END_C_DECLS + +#define PARSEC_LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + parsec_warning( "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c new file mode 100644 index 000000000..51ae04d57 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" + +#include + +PARSEC_OBJ_CLASS_INSTANCE(parsec_device_level_zero_module_t, parsec_device_module_t, NULL, NULL); + +static int device_level_zero_component_open(void); +static int device_level_zero_component_close(void); +static int device_level_zero_component_query(mca_base_module_2_0_0_t **module, int *priority); +static int device_level_zero_component_register(void); + +int parsec_device_level_zero_index, parsec_device_level_zero_enabled; +int parsec_level_zero_sort_pending = 0, parsec_level_zero_max_streams = PARSEC_GPU_MAX_STREAMS; +int level_zero_mask, level_zero_nvlink_mask; +int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; + +int32_t parsec_LEVEL_ZERO_sort_pending_list = 0; + +char* level_zero_lib_path = NULL; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +parsec_device_base_component_t parsec_device_level_zero_component = { + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + PARSEC_DEVICE_BASE_VERSION_2_0_0, + + /* Component name and version */ + "level_zero", + /* Component options */ +#if defined(PARSEC_HAVE_PEER_DEVICE_MEMORY_ACCESS) + "+peer_access" +#endif + "", + PARSEC_VERSION_MAJOR, + PARSEC_VERSION_MINOR, + + /* Component open and close functions */ + device_level_zero_component_open, + device_level_zero_component_close, + device_level_zero_component_query, + /*< specific query to return the module and add it to the list of available modules */ + device_level_zero_component_register, + "", /*< no reserve */ + }, + { + /* The component has no metadata */ + MCA_BASE_METADATA_PARAM_NONE, + "", /*< no reserve */ + }, + NULL +}; + +mca_base_component_t * device_level_zero_static_component(void) +{ + return (mca_base_component_t *)&parsec_device_level_zero_component; +} + +static int device_level_zero_component_query(mca_base_module_t **module, int *priority) +{ + int i, j, rc; + ze_result_t ze_rc; + ze_device_handle_t *devices = NULL; + ze_driver_handle_t *allDrivers = NULL; + + *module = NULL; + *priority = 0; + if( 0 == parsec_device_level_zero_enabled ) { + return MCA_SUCCESS; + } + parsec_gpu_init_profiling(); + + if( parsec_device_level_zero_enabled >= 1) { + uint32_t driverCount = 0; + uint32_t totalDeviceCount = 0, maxDeviceCount = 0; + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { return MCA_ERR_NOT_AVAILABLE; } ); + + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { free(allDrivers); return MCA_ERR_NOT_AVAILABLE; } ); + + for(uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + zeDeviceGet(allDrivers[did], &deviceCount, NULL); + totalDeviceCount += deviceCount; + if(maxDeviceCount < deviceCount) + maxDeviceCount = deviceCount; + } + + parsec_device_level_zero_enabled = totalDeviceCount < (uint32_t)parsec_device_level_zero_enabled ? (int)totalDeviceCount : parsec_device_level_zero_enabled; + + if(parsec_device_level_zero_enabled > 0) { + parsec_device_level_zero_component.modules = + (parsec_device_module_t **)calloc(parsec_device_level_zero_enabled + 1, + sizeof(parsec_device_module_t *)); + devices = (ze_device_handle_t *)malloc(maxDeviceCount * sizeof(ze_device_handle_t)); + + i = j = 0; + for(uint32_t did = 0; i < parsec_device_level_zero_enabled && did < driverCount; ++did ) { + uint32_t deviceCount = maxDeviceCount; + parsec_device_level_zero_driver_t *driver = malloc(sizeof(parsec_device_level_zero_driver_t)); + + driver->ze_driver = allDrivers[did]; + driver->ref_count = 0; + driver->swp = NULL; + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_result_t ze_rc = zeContextCreate(driver->ze_driver, &ctxtDesc, &driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { free(driver); continue; } ); + + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, devices); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, { zeContextDestroy(driver->ze_context); free(driver); continue; } ); + + for(uint32_t devid = 0; i < parsec_device_level_zero_enabled && devid < deviceCount; devid++) { + ze_device_properties_t device_properties; + ze_rc = zeDeviceGetProperties(devices[devid], &device_properties); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetProperties ", ze_rc, { continue; } ); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + if( !((1 << i) & level_zero_mask) ) { i++; continue; } + rc = parsec_level_zero_module_init(i, driver, devices[devid], &device_properties, + &parsec_device_level_zero_component.modules[j]); + if( PARSEC_SUCCESS != rc ) { + assert( NULL == parsec_device_level_zero_component.modules[j] ); + continue; + } + driver->ref_count++; + parsec_device_level_zero_component.modules[j]->component = &parsec_device_level_zero_component; + j++; /* next available spot */ + parsec_device_level_zero_component.modules[j] = NULL; + i++; + } + + if( driver->ref_count == 0 ) { + zeContextDestroy(driver->ze_context); + free(driver); + } + } + parsec_device_level_zero_enabled = j; + if(0 == parsec_device_level_zero_enabled) { + free( parsec_device_level_zero_component.modules ); + } + } + } + + if(0 == parsec_device_level_zero_enabled ) + parsec_device_level_zero_component.modules = NULL; + + if(NULL != devices) + free(devices); + if(NULL != allDrivers) + free(allDrivers); + + /* SYCL wrappers must be built once all devices and queues and contexts and drivers are up, + * and they must be built in the following order: driver wrapper, then device wrappers for this + * driver, then context wrapper for this driver, then queue wrappers for each queue. + * Also, it MUST be one to one: a single device wrapper per device, a single context wrapper + * per context etc... */ + for(int did = 0; did < parsec_device_level_zero_enabled; did++) { + if(NULL == parsec_device_level_zero_component.modules[did]) + continue; + parsec_device_level_zero_module_t *module = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did]; + if(NULL != module->driver->swp) + continue; + + module->driver->swp = parsec_sycl_wrapper_platform_create(module->driver->ze_driver); + int nbdev = 1; + for(int i = 1; did + i < parsec_device_level_zero_enabled; i++) { + if( ((parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i])->driver == module->driver ) + nbdev++; + } + parsec_sycl_wrapper_device_t **devices = (parsec_sycl_wrapper_device_t **)malloc(nbdev*sizeof(parsec_sycl_wrapper_device_t*)); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + mod2->swd = parsec_sycl_wrapper_device_create(mod2->ze_device); + devices[j++] = mod2->swd; + } + } + parsec_sycl_wrapper_platform_add_context(module->driver->swp, module->driver->ze_context, devices, nbdev); + free(devices); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + j++; + for(int s = 0; s < mod2->super.max_exec_streams; s++) { + parsec_level_zero_exec_stream_t *exec_stream = (parsec_level_zero_exec_stream_t *)mod2->super.exec_stream[s]; + if(NULL != exec_stream->level_zero_cq) + exec_stream->swq = parsec_sycl_wrapper_queue_create(module->driver->swp, mod2->swd, exec_stream->level_zero_cq); + } + } + } + } + + parsec_gpu_enable_debug(); + + /* module type should be: const mca_base_module_t ** */ + void *ptr = parsec_device_level_zero_component.modules; + *priority = 10; + *module = (mca_base_module_t *)ptr; + + return MCA_SUCCESS; +} + +static int device_level_zero_component_register(void) +{ + parsec_device_level_zero_index = parsec_mca_param_reg_int_name("device_level_zero", "enabled", + "The number of LEVEL_ZERO device to enable for the next PaRSEC context (-1 for all available)", + false, false, -1, &parsec_device_level_zero_enabled); + (void)parsec_mca_param_reg_int_name("device_level_zero", "mask", + "The bitwise mask of LEVEL_ZERO devices to be enabled (default all)", + false, false, 0xffffffff, &level_zero_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "nvlink_mask", + "What devices are allowed to use NVLINK if available (default all)", + false, false, 0xffffffff, &level_zero_nvlink_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "verbose", + "Set the verbosity level of the LEVEL_ZERO device (negative value: use debug verbosity), higher is less verbose)\n", + false, false, -1, &parsec_gpu_verbosity); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_block_size", + "The LEVEL_ZERO memory page for PaRSEC internal management (in bytes).", + false, false, 512*1024, &level_zero_memory_block_size); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_use", + "The percentage of the total GPU memory to be used by this PaRSEC context", + false, false, 95, &level_zero_memory_percentage); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_number_of_blocks", + "Alternative to device_level_zero_memory_use: sets exactly the number of blocks to allocate (-1 means to use a percentage of the available memory)", + false, false, -1, &level_zero_memory_number_of_blocks); + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", + "Sets up the maximum number of blocks that can be ejected from GPU memory", + false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_LEVEL_ZERO_sort_pending_list); +#if defined(PARSEC_PROF_TRACE) + (void)parsec_mca_param_reg_int_name("device_level_zero", "one_profiling_stream_per_level_zero_stream", + "Boolean to separate the profiling of each level_zero stream into a single profiling stream", + false, false, 0, &parsec_device_gpu_one_profiling_stream_per_gpu_stream); +#endif + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_streams", + "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", + false, false, PARSEC_GPU_MAX_STREAMS, &parsec_level_zero_max_streams); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_level_zero_sort_pending); + /* If LEVEL_ZERO was not requested avoid initializing the devices */ + return (0 == parsec_device_level_zero_enabled ? MCA_ERROR : MCA_SUCCESS); +} + +/** + * Open LEVEL_ZERO and check that devices are available and ready to be used. This operation should + * only be done once during the initialization, and the devices should from there on be managed + * by PaRSEC. + */ +static int device_level_zero_component_open(void) +{ + ze_result_t ze_rc; + ze_driver_handle_t *allDrivers = NULL; + int ndevices = 0; + uint32_t driverCount = 0; + + if( 0 == parsec_device_level_zero_enabled ) { + return MCA_ERROR; /* Nothing to do around here */ + } + + ze_rc = zeInit( 0 ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, + { + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + if(driverCount > 0) { + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + + for( uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + ndevices += (int)deviceCount; + } + + free(allDrivers); + } + + + if( ndevices > parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { + ndevices = parsec_device_level_zero_enabled; + } + } else if (ndevices < parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { + parsec_warning("User requested %d LEVEL_ZERO devices, but only %d are available on %s\n." + " PaRSEC will enable all %d of them.", + parsec_device_level_zero_enabled, ndevices, parsec_hostname, ndevices); + parsec_mca_param_set_int(parsec_device_level_zero_index, ndevices); + } + } + + /* Update the number of GPU for the upper layer */ + parsec_device_level_zero_enabled = ndevices; + if( 0 == ndevices ) { + return -1; + } + + return MCA_SUCCESS; +} + +/** + * Remove all LEVEL_ZERO devices from the PaRSEC available devices, and turn them off. + * At the end of this function all LEVEL_ZERO initialization in the context of PaRSEC + * should be undone, and pending tasks either completed or transferred to another + * chore (if available), and all LEVEL_ZERO resources (fences, streams and memory) + * released. + */ +static int device_level_zero_component_close(void) +{ + parsec_device_level_zero_module_t* cdev; + int i, rc; + + if( NULL == parsec_device_level_zero_component.modules ) { /* No devices */ + return PARSEC_SUCCESS; + } + + for( i = 0; NULL != (cdev = (parsec_device_level_zero_module_t*)parsec_device_level_zero_component.modules[i]); i++ ) { + parsec_device_level_zero_component.modules[i] = NULL; + + rc = parsec_level_zero_module_fini((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to release resources on LEVEL_ZERO device\n", + cdev->level_zero_index); + } + if(0 == --cdev->driver->ref_count) { + ze_result_t ze_rc = zeContextDestroy(cdev->driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextDestroy ", ze_rc, {}); + parsec_sycl_wrapper_platform_destroy(cdev->driver->swp); + } + + /* unregister the device from PaRSEC */ + rc = parsec_mca_device_remove((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to unregister LEVEL_ZERO device %d\n", + cdev->level_zero_index, cdev->level_zero_index); + } + + free(cdev); + } + +#if defined(PARSEC_DEBUG_NOISIER) + /* Check that no LEVEL_ZERO devices are still registered with PaRSEC */ + for(i = 0; i < parsec_mca_device_enabled(); i++) { + if( NULL == (cdev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(i)) ) continue; + if(PARSEC_DEV_LEVEL_ZERO != cdev->super.super.type) continue; + + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] LEVEL_ZERO device still registered with PaRSEC at the end of LEVEL_ZERO finalize.\n" + " Please contact the developers or fill an issue.\n", + cdev->level_zero_index); + } +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + if( parsec_device_output != parsec_gpu_output_stream ) + parsec_output_close(parsec_gpu_output_stream); + parsec_gpu_output_stream = parsec_device_output; + + if ( level_zero_lib_path ) { + free(level_zero_lib_path); + } + + return PARSEC_SUCCESS; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h new file mode 100644 index 000000000..ed3a77998 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h @@ -0,0 +1,42 @@ +#ifndef PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H +#define PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H + +typedef struct parsec_sycl_wrapper_platform_s parsec_sycl_wrapper_platform_t; +typedef struct parsec_sycl_wrapper_device_s parsec_sycl_wrapper_device_t; +typedef struct parsec_sycl_wrapper_queue_s parsec_sycl_wrapper_queue_t; + +#include + +#if defined(c_plusplus) || defined(__cplusplus) +#include + +struct parsec_sycl_wrapper_platform_s { + sycl::platform platform; + sycl::context context; +}; + +struct parsec_sycl_wrapper_device_s { + sycl::device device; +}; + +struct parsec_sycl_wrapper_queue_s { + sycl::queue queue; +}; + +extern "C" { +#endif + +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device); +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device); +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp); +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd); +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp new file mode 100644 index 000000000..bafd03cca --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp @@ -0,0 +1,59 @@ +#include +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" + +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + parsec_sycl_wrapper_platform_t *res = new parsec_sycl_wrapper_platform_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device) +{ + std::vectordevices; + + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + swp->context = sycl::make_context(hContextInteropInput); +} + +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + parsec_sycl_wrapper_device_t *res = new parsec_sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); + + return res; +} + +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) +{ + parsec_sycl_wrapper_queue_t *swq = new parsec_sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} + +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp) +{ + if(nullptr != swp) + delete swp; + return 0; +} + +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd) +{ + if(nullptr != swd) + delete swd; + return 0; +} + +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq) +{ + if(nullptr != swq) + delete swq; + return 0; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_internal.h b/parsec/mca/device/level_zero/device_level_zero_internal.h new file mode 100644 index 000000000..b2bd8de42 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_internal.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED + +#include "parsec/mca/device/level_zero/device_level_zero.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) + +BEGIN_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN 0x0001 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT 0x0002 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN 0x0004 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC 0x0008 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE 0x0010 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH 0x0020 + +extern int parsec_level_zero_trackable_events; +extern int parsec_level_zero_movein_key_start; +extern int parsec_level_zero_movein_key_end; +extern int parsec_level_zero_moveout_key_start; +extern int parsec_level_zero_moveout_key_end; +extern int parsec_level_zero_own_GPU_key_start; +extern int parsec_level_zero_own_GPU_key_end; +extern int parsec_level_zero_allocate_memory_key; +extern int parsec_level_zero_free_memory_key; +extern int parsec_level_zero_use_memory_key_start; +extern int parsec_level_zero_use_memory_key_end; +extern int parsec_level_zero_prefetch_key_start; +extern int parsec_level_zero_prefetch_key_end; +extern int parsec_device_level_zero_one_profiling_stream_per_level_zero_stream; +#endif /* defined(PROFILING) */ + +#define GPU_TASK_TYPE_KERNEL 0x0000 +#define GPU_TASK_TYPE_D2HTRANSFER 0x1000 +#define GPU_TASK_TYPE_PREFETCH 0x2000 +#define GPU_TASK_TYPE_WARMUP 0x4000 +#define GPU_TASK_TYPE_D2D_COMPLETE 0x8000 + +/* From MCA parameters */ +extern int use_level_zero_index, use_level_zero; +extern int level_zero_mask, level_zero_verbosity; +extern int parsec_level_zero_sort_pending, parsec_level_zero_max_streams; +extern int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; +extern char* level_zero_lib_path; +extern int32_t parsec_LEVEL_ZERO_d2h_max_flows; +extern int32_t parsec_LEVEL_ZERO_sort_pending_list; + +PARSEC_DECLSPEC extern const parsec_device_module_t parsec_device_level_zero_module; + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ + +int parsec_level_zero_module_init( int device_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, + ze_device_properties_t *device_properties, parsec_device_module_t** module ); +int parsec_level_zero_module_fini(parsec_device_module_t* device); + +END_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +typedef struct { + uint64_t size; + uint64_t data_key; + uint64_t dc_id; +} parsec_device_level_zero_memory_prof_info_t; +#define PARSEC_DEVICE_LEVEL_ZERO_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" +#endif /* PARSEC_PROF_TRACE */ + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c new file mode 100644 index 000000000..db2e461ee --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -0,0 +1,2678 @@ +/* + * Copyright (c) 2021-2022 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" + +#include + +static int parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice); +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* gpu_device, + int memory_percentage, + int number_of_elements, + size_t eltsize ); +static int parsec_level_zero_memory_release( parsec_device_level_zero_module_t* gpu_device ); +static int parsec_level_zero_flush_lru( parsec_device_module_t *device ); + +static void* level_zero_find_incarnation(parsec_device_level_zero_module_t* level_zero_device, + const char* fname) +{ + char library_name[FILENAME_MAX], *env; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + void *fn = NULL; + char** argv = NULL; + + /** + * Prepare the list of PATH or FILE to be searched for a LEVEL_ZERO shared library. + * In any case this list might be a list of ; separated possible targets, + * where each target can be either a directory or a specific file. + */ + env = getenv("PARSEC_CUCORES_LIB"); + if( NULL != env ) { + argv = parsec_argv_split(env, ';'); + } else if( NULL != level_zero_lib_path ) { + argv = parsec_argv_split(level_zero_lib_path, ';'); + } + + fn = parsec_device_find_function(fname, library_name, (const char**)argv); + if( NULL == fn ) { /* look for the function with lesser capabilities */ + parsec_warning("No function '%s' found for LEVEL_ZERO device %s", fname, gpu_device->super.name); + } + + if( NULL != argv ) + parsec_argv_free(argv); + + return fn; +} + +/** + * Register a taskpool with a device by checking that the device + * supports the dynamic function required by the different incarnations. + * If multiple devices of the same type exists we assume thay all have + * the same capabilities. + */ +static int +parsec_level_zero_taskpool_register(parsec_device_module_t* device, + parsec_taskpool_t* tp) +{ + parsec_device_level_zero_module_t* gpu_device = (parsec_device_level_zero_module_t*)device; + int32_t rc = PARSEC_ERR_NOT_FOUND; + uint32_t i, j; + + /** + * Detect if a particular chore has a dynamic load dependency and if yes + * load the corresponding module and find the function. + */ + assert(PARSEC_DEV_LEVEL_ZERO == device->type); + assert(tp->devices_index_mask & (1 << device->device_index)); + + for( i = 0; i < tp->nb_task_classes; i++ ) { + const parsec_task_class_t* tc = tp->task_classes_array[i]; + __parsec_chore_t* chores = (__parsec_chore_t*)tc->incarnations; + for( j = 0; NULL != chores[j].hook; j++ ) { + if( chores[j].type != device->type ) + continue; + if( NULL != chores[j].dyld_fn ) { + /* the function has been set for another device of the same type */ + return PARSEC_SUCCESS; + } + if ( NULL == chores[j].dyld ) { + chores[j].dyld_fn = NULL; /* No dynamic support required for this kernel */ + rc = PARSEC_SUCCESS; + } else { + void* devf = level_zero_find_incarnation(gpu_device, chores[j].dyld); + if( NULL != devf ) { + chores[j].dyld_fn = devf; + rc = PARSEC_SUCCESS; + } + } + } + } + if( PARSEC_SUCCESS != rc ) { + tp->devices_index_mask &= ~(1 << device->device_index); /* drop support for this device */ + parsec_debug_verbose(10, parsec_gpu_output_stream, + "Device %d (%s) disabled for taskpool %p", device->device_index, device->name, tp); + } + return rc; +} + +static int +parsec_level_zero_taskpool_unregister(parsec_device_module_t* device, parsec_taskpool_t* tp) +{ + (void)device; (void)tp; + return PARSEC_SUCCESS; +} + +/** + * Attach a device to a PaRSEC context. A device can only be attached to + * a single context at the time. + */ +static int +parsec_device_level_zero_attach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + return parsec_mca_device_add(context, device); +} + +/** + * Detach a device from a context. Both the context and the device remain + * valid, they are simply disconnected. + * This function should only be called once all tasks and all data related to the + * context has been removed from the device. + */ +static int +parsec_device_level_zero_detach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + (void)context; + return parsec_mca_device_remove(device); +} + +int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, + ze_device_properties_t *prop, parsec_device_module_t** module ) +{ + int streaming_multiprocessor, len; + parsec_device_level_zero_module_t* level_zero_device; + parsec_device_gpu_module_t* gpu_device; + parsec_device_module_t* device; + ze_result_t ze_rc; + int show_caps_index, show_caps = 0, j, k; + char *szName; + float clockRate; + + show_caps_index = parsec_mca_param_find("device", NULL, "show_capabilities"); + if(0 < show_caps_index) { + parsec_mca_param_lookup_int(show_caps_index, &show_caps); + } + + *module = NULL; + + szName = prop->name; + clockRate = prop->coreClockRate/1e3f; + streaming_multiprocessor = prop->numThreadsPerEU; + + level_zero_device = (parsec_device_level_zero_module_t*)calloc(1, sizeof(parsec_device_level_zero_module_t)); + gpu_device = &level_zero_device->super; + device = &gpu_device->super; + PARSEC_OBJ_CONSTRUCT(level_zero_device, parsec_device_level_zero_module_t); + level_zero_device->level_zero_index = (uint8_t)dev_id; + level_zero_device->driver = driver; + level_zero_device->ze_device = ze_device; + + len = asprintf(&gpu_device->super.name, "%s ZE(%d)", szName, dev_id); + if(-1 == len) + gpu_device->super.name = ""; + gpu_device->data_avail_epoch = 0; + + gpu_device->max_exec_streams = parsec_level_zero_max_streams; + gpu_device->exec_stream = + (parsec_gpu_exec_stream_t**)malloc(gpu_device->max_exec_streams * sizeof(parsec_gpu_exec_stream_t*)); + gpu_device->exec_stream[0] = + (parsec_gpu_exec_stream_t*)malloc( gpu_device->max_exec_streams * sizeof(parsec_level_zero_exec_stream_t)); + for(j = 1; j < gpu_device->max_exec_streams; j++) + gpu_device->exec_stream[j] = (parsec_gpu_exec_stream_t*)( + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[0]+j); + + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, NULL); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, cmdqueueGroupProperties); + // Find a command queue type that support compute + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; + i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); + ++i ) { + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + computeQueueGroupOrdinal = i; + } + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + copyQueueGroupOrdinal = i; + } + } + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + /* CommandListImmediate would seem better for I/O, but mixing CLImmediate and CL+CQ + * seems to create some synchronization issues. */ + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandQueueDesc, &level_zero_stream->level_zero_cq); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, {goto release_device;} ); + exec_stream->workspace = NULL; + PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); + parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); + exec_stream->max_events = PARSEC_MAX_EVENTS_PER_STREAM; + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + exec_stream->name = NULL; + exec_stream->fifo_pending = (parsec_list_t*)PARSEC_OBJ_NEW(parsec_list_t); + PARSEC_OBJ_CONSTRUCT(exec_stream->fifo_pending, parsec_list_t); + exec_stream->tasks = (parsec_gpu_task_t**)malloc(exec_stream->max_events + * sizeof(parsec_gpu_task_t*)); + level_zero_stream->fences = (ze_fence_handle_t*)malloc(exec_stream->max_events * sizeof(ze_fence_handle_t)); + level_zero_stream->command_lists = (ze_command_list_handle_t*)malloc(exec_stream->max_events*sizeof(ze_command_list_handle_t)); + + /* create the fences and command lists */ + for( k = 0; k < exec_stream->max_events; k++ ) { + ze_command_list_desc_t commandListDesc = { + ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, + NULL, + computeQueueGroupOrdinal, + 0 // flags + }; + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 + }; + level_zero_stream->fences[k] = NULL; + exec_stream->tasks[k] = NULL; + ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandListDesc, &level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); + ze_rc = zeFenceCreate(level_zero_stream->level_zero_cq, &fence_desc, &(level_zero_stream->fences[k])); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, {goto release_device;} ); + } + if(j == 0) { + len = asprintf(&exec_stream->name, "h2d(%d)", j); + if(-1 == len) + exec_stream->name = "h2d"; + } else if(j == 1) { + len = asprintf(&exec_stream->name, "d2h(%d)", j); + if(-1 == len) + exec_stream->name = "d2h"; + } else { + len = asprintf(&exec_stream->name, "level_zero(%d)", j); + if(-1 == len) + exec_stream->name = "level_zero"; + } +#if defined(PARSEC_PROF_TRACE) + /* Each 'exec' stream gets its own profiling stream, except IN and OUT stream that share it. + * It's good to separate the exec streams to know what was submitted to what stream + * We don't have this issue for the IN and OUT streams because types of event discriminate + * what happens where, and separating them consumes memory and increases the number of + * events that needs to be matched between streams because we cannot differentiate some + * ends between IN or OUT, so they are all logged on the same stream. */ + if(j == 0 || (parsec_device_gpu_one_profiling_stream_per_gpu_stream == 1 && j != 1)) + exec_stream->profiling = parsec_profiling_stream_init( 2*1024*1024, PARSEC_PROFILE_STREAM_STR, dev_id, j ); + else + exec_stream->profiling = gpu_device->exec_stream[0]->profiling; + if(j == 0) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else if(j == 1) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + + device->type = PARSEC_DEV_LEVEL_ZERO; + device->executed_tasks = 0; + device->data_in_array_size = 0; // We'll let the modules_attach allocate the array of the right size for us + device->data_in_from_device = NULL; + device->data_out_to_host = 0; + device->required_data_in = 0; + device->required_data_out = 0; + + device->attach = parsec_device_level_zero_attach; + device->detach = parsec_device_level_zero_detach; + device->memory_register = NULL; // TODO: could not find a memory register equivalent in level zero + device->memory_unregister = NULL; // TODO: could not find a memory unregister equivalent in level zero + device->taskpool_register = parsec_level_zero_taskpool_register; + device->taskpool_unregister = parsec_level_zero_taskpool_unregister; + device->data_advise = parsec_level_zero_data_advise; + device->memory_release = parsec_level_zero_flush_lru; + + device->device_hweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_tweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_sweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_dweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + + /* Initialize internal lists */ + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_owned_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->pending, parsec_fifo_t); + + gpu_device->sort_starting_p = NULL; + gpu_device->peer_access_mask = 0; /* No GPU to GPU direct transfer by default */ + + if( PARSEC_SUCCESS != parsec_level_zero_memory_reserve(level_zero_device, + level_zero_memory_percentage, + level_zero_memory_number_of_blocks, + level_zero_memory_block_size) ) { + goto release_device; + } + + if( show_caps ) { + parsec_inform("LEVEL ZERO GPU Device %d: %s\n" + "\tLocation (PCI Bus/Device/Domain): %x:%x.%x\n" + "\tnumThreadsPerEU : %u\n" + "\tphysicalEUSimdWidth : %u\n" + "\tnumEUsPerSubslice : %u\n" + "\tnumSubslicesPerSlice: %u\n" + "\tnumSlices: %u\n" + "\tcoreClockRate (GHz): %2.2f\n" + "\tpeak Gflops : double %2.3f, single %2.3f tensor %2.3f half %2.3f\n", + level_zero_device->level_zero_index, device->name, + prop->deviceId, prop->subdeviceId, prop->vendorId, + streaming_multiprocessor, + prop->physicalEUSimdWidth, + prop->numEUsPerSubslice, + prop->numSubslicesPerSlice, + prop->numSlices, + clockRate*1e-3, + device->device_dweight, device->device_sweight, device->device_tweight, device->device_hweight); + } + + *module = device; + return PARSEC_SUCCESS; + + release_device: + if( NULL != gpu_device->exec_stream) { + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t *level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + if( NULL != exec_stream->fifo_pending ) { + PARSEC_OBJ_RELEASE(exec_stream->fifo_pending); + } + if( NULL != exec_stream->tasks ) { + free(exec_stream->tasks); exec_stream->tasks = NULL; + } + if( NULL != level_zero_stream->fences ) { + for( k = 0; k < exec_stream->max_events; k++ ) { + if( NULL != level_zero_stream->fences[k] ) { + (void)zeFenceDestroy(level_zero_stream->fences[k]); + } + } + free(level_zero_stream->fences); level_zero_stream->fences = NULL; + } + if( NULL != exec_stream->name ) { + free(exec_stream->name); exec_stream->name = NULL; + } +#if defined(PARSEC_PROF_TRACE) + if( NULL != exec_stream->profiling ) { + /* No function to clean the profiling stream. If one is introduced + * some day, remember that exec streams 0 and 1 always share the same + * ->profiling stream, and that all of them share the same + * ->profiling stream if parsec_device_gpu_one_profiling_stream_per_gpu_stream == 0 */ + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + } + free(gpu_device); + return PARSEC_ERROR; +} + +int +parsec_level_zero_module_fini(parsec_device_module_t* device) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t* level_zero_device = (parsec_device_level_zero_module_t*)device; + ze_result_t status; + int j, k; + + /* Release the registered memory */ + parsec_level_zero_memory_release(level_zero_device); + + /* Release pending queue */ + PARSEC_OBJ_DESTRUCT(&gpu_device->pending); + + /* Release all streams */ + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + + for( k = 0; k < exec_stream->max_events; k++ ) { + assert( NULL == exec_stream->tasks[k] ); + status = zeFenceDestroy(level_zero_stream->fences[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) zeFenceDestroy ", status, {} ); + status = zeCommandListDestroy(level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", status, {} ); + } + exec_stream->max_events = 0; + free(level_zero_stream->fences); level_zero_stream->fences = NULL; + free(exec_stream->tasks); exec_stream->tasks = NULL; + free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; + /* Deleting the sycl queue wrapper and/or the command queue conflicts with the + * cleaning procedure of the Level Zero runtime... Didn't find a way to do it + * cleanly. Don't cleanup for now... */ + //parsec_sycl_wrapper_queue_destroy(level_zero_stream->swq); + level_zero_stream->swq = NULL; + free(exec_stream->name); + + /* Release Info object array */ + PARSEC_OBJ_DESTRUCT(&exec_stream->infos); + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + level_zero_device->level_zero_index = -1; + parsec_sycl_wrapper_device_destroy(level_zero_device->swd); + level_zero_device->swd = NULL; + + /* Cleanup the GPU memory. */ + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_lru); + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_owned_lru); + + return PARSEC_SUCCESS; +} + +/** + * This function reserve the memory_percentage of the total device memory for PaRSEC. + * This memory will be managed in chuncks of size eltsize. However, multiple chuncks + * can be reserved in a single allocation. + */ +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_device, + int memory_percentage, + int number_blocks, + size_t eltsize ) +{ + ze_result_t status; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + (void)eltsize; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + uint32_t count = 0; + + size_t how_much_we_allocate; + size_t initial_free_mem, alignment = 1 << 3; + int memIndex = -1; + uint32_t mem_elem_per_gpu = 0; + + status = zeDeviceGetMemoryAccessProperties(level_zero_device->ze_device, &memAccessProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { + return PARSEC_ERROR; + }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + parsec_warning("%s:%d -- Device %s does not have memory allocation capabilities with RW access\n", + __FILE__, __LINE__, gpu_device->super.name); + return PARSEC_ERROR; + } + status = zeDeviceGetProperties(level_zero_device->ze_device, &devProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { + return PARSEC_ERROR; + }); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + return PARSEC_ERROR; + }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + free(devMemProperties); + return PARSEC_ERROR; + }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + initial_free_mem = devProperties.maxMemAllocSize < devMemProperties[memIndex].totalSize ? + devProperties.maxMemAllocSize : devMemProperties[memIndex].totalSize; + free(devMemProperties); devMemProperties = NULL; + + if( number_blocks != -1 ) { + if( number_blocks == 0 ) { + parsec_warning("LEVEL_ZERO[%d] Invalid argument: requesting 0 bytes of memory on LEVEL_ZERO device %s", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } else { + how_much_we_allocate = number_blocks * eltsize; + } + } else { + /** number_blocks == -1 means memory_percentage is used */ + how_much_we_allocate = (memory_percentage * initial_free_mem) / 100; + } + if( how_much_we_allocate > initial_free_mem ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + parsec_warning("LEVEL_ZERO[%d] Requested %zd bytes on LEVEL_ZERO device %s, but only %zd bytes are available -- reducing allocation to max available", + level_zero_device->level_zero_index, how_much_we_allocate, gpu_device->super.name, initial_free_mem); + how_much_we_allocate = initial_free_mem; + } + if( how_much_we_allocate < eltsize ) { + /** Handle another kind of jokers entirely, and cases of + * not enough memory on the device + */ + parsec_warning("LEVEL_ZERO[%d] Cannot allocate at least one element on LEVEL_ZERO device %s", + level_zero_device->level_zero_index, gpu_device->super.name); + return PARSEC_ERROR; + } + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + size_t free_mem = initial_free_mem; + /* + * We allocate a bunch of tiles that will be used + * during the computations + */ + while( (free_mem > eltsize ) + && ((total_mem - free_mem) < how_much_we_allocate) ) { + parsec_gpu_data_copy_t* gpu_elem; + void *device_ptr; + + status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, eltsize, alignment, + level_zero_device->ze_device, &device_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { break; } ); + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate LEVEL_ZERO copy %p [ref_count %d] for data [%p]", + gpu_device->super.name,gpu_elem, gpu_elem->super.obj_reference_count, NULL); + gpu_elem->device_private = (void*)(long)device_ptr; + gpu_elem->flags |= PARSEC_DATA_FLAG_PARSEC_OWNED; + gpu_elem->device_index = gpu_device->super.device_index; + mem_elem_per_gpu++; + PARSEC_OBJ_RETAIN(gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, gpu_elem, gpu_elem->super.obj_reference_count); + parsec_list_push_back( &gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem ); + free_mem -= alignment * ((size + alignment - 1) / alignment); + } + if( 0 == mem_elem_per_gpu && parsec_list_is_empty( &gpu_device->gpu_mem_lru ) ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", gpu_device->super.name, gpu_device->super.name); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu ); + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", gpu_device->super.name, mem_elem_per_gpu); +#else + if( NULL == gpu_device->memory ) { + void* base_ptr; + /* We allocate all the memory on the GPU and we use our memory management. */ + /* This computation leads to allocating more than available if we asked for more than GPU memory */ + mem_elem_per_gpu = (how_much_we_allocate + eltsize - 1 ) / eltsize; + size_t total_size = (size_t)mem_elem_per_gpu * eltsize; + + if (total_size > initial_free_mem) { + /* Mapping more than 100% of GPU memory is obviously wrong */ + /* Mapping exactly 100% of the GPU memory ends up producing errors about __global__ function call is not configured */ + /* Mapping 95% works with low-end GPUs like 1060, how much to let available for level_zero runtime, I don't know how to calculate */ + total_size = (size_t)((int)(.9*initial_free_mem / eltsize)) * eltsize; + mem_elem_per_gpu = total_size / eltsize; + } + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, total_size, alignment, + level_zero_device->ze_device, &base_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, + ({ parsec_warning("GPU[%s] Allocating %zu bytes of memory on the GPU device failed", + gpu_device->super.name, total_size); }) ); + + gpu_device->memory = zone_malloc_init( base_ptr, mem_elem_per_gpu, eltsize ); + + if( gpu_device->memory == NULL ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u segments of size %d on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu, eltsize ); + } +#endif + gpu_device->mem_block_size = eltsize; + gpu_device->mem_nb_blocks = mem_elem_per_gpu; + + return PARSEC_SUCCESS; +} + +static void parsec_level_zero_memory_release_list(parsec_device_level_zero_module_t* level_zero_device, + parsec_list_t* list) +{ + parsec_list_item_t* item; +#if defined(PARSEC_DEBUG_VERBOSE) + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; +#endif + + while(NULL != (item = parsec_list_pop_front(list)) ) { + parsec_gpu_data_copy_t* gpu_copy = (parsec_gpu_data_copy_t*)item; + parsec_data_t* original = gpu_copy->original; + + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p, in map %p", + gpu_device->super.name, gpu_copy, gpu_copy->device_private, gpu_copy->super.super + .obj_reference_count, + original, (NULL != original ? original->dc : NULL)); + assert( gpu_copy->device_index == level_zero_device->super.super.device_index ); + + if( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ) { + parsec_warning("GPU[%s] still OWNS the master memory copy for data %d and it is discarding it!", + gpu_device->super.name, original->key); + } + assert(0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + level_zeroFree( gpu_copy->device_private ); +#else + +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)gpu_copy->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)gpu_copy->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + zone_free( level_zero_device->super.memory, (void*)gpu_copy->device_private ); +#endif + gpu_copy->device_private = NULL; + + /* At this point the data copies should have no attachement to a data_t. Thus, + * before we get here (aka below parsec_fini), the destructor of the data + * collection must have been called, releasing all the copies. + */ + PARSEC_OBJ_RELEASE(gpu_copy); assert(NULL == gpu_copy); + } +} + +/** + * This function only flushes the data copies pending in LRU, and checks + * (in debug mode) that the entire allocated memory is free to use */ +static int +parsec_level_zero_flush_lru( parsec_device_module_t *device ) +{ + size_t in_use; + parsec_device_gpu_module_t *gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)device; + /* Free all memory on GPU */ + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_lru); + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_owned_lru); +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) && !defined(_NDEBUG) + if( (in_use = zone_in_use(gpu_device->memory)) != 0 ) { + parsec_warning("GPU[%s] memory leak detected: %lu bytes still allocated on GPU", + device->name, in_use); + assert(0); + } +#endif + return PARSEC_SUCCESS; +} + +/** + * This function release the LEVEL_ZERO memory reserved for this device. + * + * One has to notice that all the data available on the GPU is stored in one of + * the two used to keep track of the allocated data, either the gpu_mem_lru or + * the gpu_mem_owner_lru. Thus, going over all the elements in these two lists + * should be enough to enforce a clean release. + */ +static int +parsec_level_zero_memory_release( parsec_device_level_zero_module_t* level_zero_device ) +{ + ze_result_t status; + + parsec_level_zero_flush_lru(&level_zero_device->super.super); + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + assert( NULL != level_zero_device->super.memory ); + void* ptr = zone_malloc_fini(&level_zero_device->super.memory); + status = zeMemFree(level_zero_device->driver->ze_context, ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, + { parsec_warning("Failed to free the GPU backend memory."); } ); +#endif + + return PARSEC_SUCCESS; +} + +/** + * Try to find memory space to move all data on the GPU. We attach a device_elem to + * a memory_elem as soon as a device_elem is available. If we fail to find enough + * available elements, we push all the elements handled during this allocation + * back into the pool of available device_elem, to be picked up by another call + * (this call will remove them from the current task). + * Returns: + * PARSEC_HOOK_RETURN_DONE: All gpu_mem/mem_elem have been initialized + * PARSEC_HOOK_RETURN_AGAIN: At least one flow is marked under transfer, task cannot be scheduled yet + * PARSEC_HOOK_RETURN_NEXT: The task needs to rescheduled + */ +static inline int +parsec_gpu_data_reserve_device_space( parsec_device_level_zero_module_t* level_zero_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t* temp_loc[MAX_PARAM_COUNT], *gpu_elem, *lru_gpu_elem; + parsec_data_t* master, *oldmaster; + const parsec_flow_t *flow; + int i, j, data_avail_epoch = 0; + parsec_gpu_data_copy_t *gpu_mem_lru_cycling; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + +#if defined(PARSEC_DEBUG_NOISIER) + char task_name[MAX_TASK_STRLEN]; + parsec_task_snprintf(task_name, MAX_TASK_STRLEN, this_task); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + /** + * Parse all the input and output flows of data and ensure all have + * corresponding data on the GPU available. + */ + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + gpu_mem_lru_cycling = NULL; + flow = gpu_task->flow[i]; + assert( flow && (flow->flow_index == i) ); + + /* Skip CTL flows only */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Investigating flow %s:%d", + gpu_device->super.name, task_name, flow->name, i); + temp_loc[i] = NULL; + if (this_task->data[i].data_in == NULL) + continue; + + master = this_task->data[i].data_in->original; + parsec_atomic_lock(&master->lock); + gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + + /* There is already a copy on the device */ + if( NULL != gpu_elem ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Flow %s:%i has a copy on the device %p%s", + gpu_device->super.name, task_name, + flow->name, i, gpu_elem, + gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ? " [in transfer]" : ""); + if ( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Copy %p [ref_count %d] is still in transfer, descheduling...", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + SET_HIGHEST_PRIORITY(gpu_task->ec, parsec_execution_context_priority_comparator); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_AGAIN; + } + parsec_atomic_unlock(&master->lock); + continue; + } + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Allocate LEVEL_ZERO copy %p sz %d[ref_count %d] for data %p", + gpu_device->super.name, task_name, + gpu_elem, gpu_task->flow_nb_elts[i], gpu_elem->super.super.obj_reference_count, master); + gpu_elem->flags = PARSEC_DATA_FLAG_PARSEC_OWNED | PARSEC_DATA_FLAG_PARSEC_MANAGED; + malloc_data: + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + gpu_elem->device_private = zone_malloc(gpu_device->memory, gpu_task->flow_nb_elts[i]); + if( NULL == gpu_elem->device_private ) { +#endif + + find_another_data: + /* Look for a data_copy to free */ + lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru); + if( NULL == lru_gpu_elem ) { + /* We can't find enough room on the GPU. Insert the tiles in the begining of + * the LRU (in order to be reused asap) and return without scheduling the task. + */ +#if defined(PARSEC_DEBUG_NOISIER) + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]:%s:\tRequest space on GPU failed for flow %s index %d/%d for task %s", + gpu_device->super.name, task_name, + flow->name, i, this_task->task_class->nb_flows, task_name ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + for( j = 0; j < i; j++ ) { + /* This flow could be a control flow */ + if( NULL == temp_loc[j] ) continue; + /* This flow could be non-parsec-owned, in which case we can't reclaim it */ + if( 0 == (temp_loc[j]->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\tAdd copy %p [ref_count %d] back to the LRU list", + gpu_device->super.name, task_name, + temp_loc[j], temp_loc[j]->super.super.obj_reference_count); + /* push them at the head to reach them again at the next iteration */ + parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)temp_loc[j]); + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + PARSEC_OBJ_RELEASE(gpu_elem); +#endif + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } + + PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Evaluate LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, + lru_gpu_elem->original); + + /* If there are pending readers, let the gpu_elem loose. This is a weak coordination + * protocol between here and the parsec_gpu_data_stage_in, where the readers don't necessarily + * always remove the data from the LRU. + */ + if( 0 != lru_gpu_elem->readers ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + goto find_another_data; // TODO: add an assert of some sort to check for leaks here? + } + /* It's also possible that the ref_count of that element is bigger than 1 + * In that case, it's because some task completion did not execute yet, and + * we need to keep it in the list until it reaches 1. + */ + if( lru_gpu_elem->super.super.obj_reference_count > 1 ) { + /* It's also possible (although unlikely) that we livelock here: + * if gpu_mem_lru has *only* elements with readers == 0 but + * ref_count > 1, then we might pop/push forever, and we need + * to make progress on something else to get ref_count == 1 && readers == 0. + * So, we return that there is no more free to do. + * To detect that, we use the first one we push back: if we see it + * again, we're cycling. */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Push back LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + + /* Make sure the new GPU element is clean and ready to be used */ + assert( master != lru_gpu_elem->original ); + if ( NULL != lru_gpu_elem->original ) { + /* Let's check we're not trying to steal one of our own data */ + oldmaster = lru_gpu_elem->original; + if( !parsec_atomic_trylock( &oldmaster->lock ) ) { + /* Even if we have the lock on oldmaster, any other thread + * might be adding/removing other elements to the list, so we + * need to protect all accesses to gpu_mem_lru with the locked version */ + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + for( j = 0; j < i; j++ ) { + if( NULL == this_task->data[j].data_in ) continue; + if( this_task->data[j].data_in->original == oldmaster ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] already in use by same task %d:%d original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, i, j, lru_gpu_elem->original); + /* If we are the owner of this tile we need to make sure it remains available for + * other tasks or we run in deadlock situations. + */ + if( temp_loc[j] != lru_gpu_elem ) + temp_loc[j] = lru_gpu_elem; +#if defined(PARSEC_DEBUG_NOISIER) + /* Make sure the data copy is indeed referenced from the current task */ + for( j = 0; j < i; j++ ) { + if( lru_gpu_elem == temp_loc[j] ) break; + } + assert( j < i ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + } + if( lru_gpu_elem->readers != 0 ) { + /* Damn, another thread started to use this data. */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + int do_unlock = oldmaster->super.obj_reference_count != 1; + parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index); + if( do_unlock ) + parsec_atomic_unlock( &oldmaster->lock ); + assert(lru_gpu_elem->readers == 0); + /* The data is not used, it's not one of ours, and it has been detached from the device + * so no other device can use it as a source for their copy : we can free it or reuse it */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d repurpose copy %p [ref_count %d] to data %p instead of %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem, + lru_gpu_elem->super.super.obj_reference_count, master, oldmaster); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d found detached memory from previously destructed data %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem); + oldmaster = NULL; + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + /* Let's free this space, and try again to malloc some space */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p", + gpu_device->super.name, + lru_gpu_elem, lru_gpu_elem->device_private, lru_gpu_elem->super.super.obj_reference_count, + oldmaster); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + zone_free( gpu_device->memory, (void*)(lru_gpu_elem->device_private) ); + lru_gpu_elem->device_private = NULL; + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:%s: Release LRU-retrieved LEVEL_ZERO copy %p [ref_count %d: must be 1]", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count); + PARSEC_OBJ_RELEASE(lru_gpu_elem); + assert( NULL == lru_gpu_elem ); + goto malloc_data; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Succeeded Allocating LEVEL_ZERO copy %p at real address %p [ref_count %d] for data %p", + gpu_device->super.name, + gpu_elem, gpu_elem->device_private, gpu_elem->super.super.obj_reference_count, master); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_allocate_memory_key, (int64_t)gpu_elem->device_private, + gpu_device->super.device_index, + &gpu_task->flow_nb_elts[i], PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif +#else + gpu_elem = lru_gpu_elem; +#endif + assert( 0 == gpu_elem->readers ); + gpu_elem->coherency_state = PARSEC_DATA_COHERENCY_INVALID; + gpu_elem->version = 0; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets created with version 0 at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + parsec_data_copy_attach(master, gpu_elem, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + /* set the new datacopy type to the correct one */ + this_task->data[i].data_out->dtt = this_task->data[i].data_in->dtt; + temp_loc[i] = gpu_elem; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem); + parsec_atomic_unlock(&master->lock); + } + if( data_avail_epoch ) { + gpu_device->data_avail_epoch++; + } + return PARSEC_HOOK_RETURN_DONE; +} + +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_in; + copy_out = task->data[i].data_out; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? + copy_in->original->nb_elts : copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +int +parsec_default_dpcpp_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + return parsec_default_level_zero_stage_in(gtask, flow_mask, gpu_stream); +} + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_out; + copy_out = copy_in->original->device_copies[0]; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? copy_in->original->nb_elts : + copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +int +parsec_default_dpcpp_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + return parsec_default_level_zero_stage_out(gtask, flow_mask, gpu_stream); +} + +/** + * If the most current version of the data is not yet available on the GPU memory + * schedule a transfer. + * Returns: + * 0: The most recent version of the data is already available on the GPU + * 1: A copy has been scheduled on the corresponding stream + * -1: A copy cannot be issued due to LEVEL_ZERO. + */ +static inline int +parsec_gpu_data_stage_in( parsec_device_level_zero_module_t* level_zero_device, + const parsec_flow_t *flow, + parsec_data_pair_t* task_data, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream ) +{ + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + int32_t type = flow->flow_flags; + parsec_data_copy_t* in_elem = task_data->data_in; + parsec_data_copy_t* release_after_data_in_is_attached = NULL; + parsec_data_t* original = in_elem->original; + parsec_gpu_data_copy_t* gpu_elem = task_data->data_out; + uint32_t nb_elts = gpu_task->flow_nb_elts[flow->flow_index]; + int transfer_from = -1; + int undo_readers_inc_if_no_transfer = 0; + + if( gpu_task->task_type == GPU_TASK_TYPE_PREFETCH ) { + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: Prefetch task %p is staging in", + gpu_device->super.name, gpu_task); + } + + parsec_atomic_lock( &original->lock ); + + /** + * If the data will be accessed in write mode, remove it from any GPU data management + * lists until the task is completed. + */ + if( PARSEC_FLOW_ACCESS_WRITE & type ) { + if (gpu_elem->readers > 0 ) { + if( !((1 == gpu_elem->readers) && (PARSEC_FLOW_ACCESS_READ & type)) ) { + parsec_warning("GPU[%s]:\tWrite access to data copy %p [ref_count %d] with existing readers [%d] " + "(possible anti-dependency,\n" + "or concurrent accesses), please prevent that with CTL dependencies\n", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->readers); + parsec_atomic_unlock( &original->lock ); + return -1; + } + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tDetach writable LEVEL_ZERO copy %p [ref_count %d] from any lists", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count); + /* make sure the element is not in any tracking lists */ + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_elem); + PARSEC_LIST_ITEM_SINGLETON(gpu_elem); + } + + /* Detect if we can do a device to device copy. + * Current limitations: only for read-only data used read-only on the hosting GPU. */ + parsec_device_level_zero_module_t *in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get( in_elem->device_index ); + if( (PARSEC_FLOW_ACCESS_READ & type) && !(PARSEC_FLOW_ACCESS_WRITE & type) ) { + int potential_alt_src = 0; + if( PARSEC_DEV_LEVEL_ZERO == in_elem_dev->super.super.type ) { + if( gpu_device->peer_access_mask & (1 << in_elem_dev->level_zero_index) ) { + /* We can directly do D2D, so let's skip the selection */ + goto src_selected; + } + } + + /* If gpu_elem is not invalid, then it is already there and the right version, + * and we're not going to transfer from another source, skip the selection */ + if( gpu_elem->coherency_state != PARSEC_DATA_COHERENCY_INVALID ) + goto src_selected; + + for(int t = 1; t < (int)parsec_nb_devices; t++) { + parsec_device_level_zero_module_t *target = (parsec_device_level_zero_module_t*)parsec_mca_device_get(t); + if( PARSEC_DEV_LEVEL_ZERO != target->super.super.type ) continue; + if(gpu_device->peer_access_mask & (1 << target->level_zero_index)) { + parsec_data_copy_t *candidate = original->device_copies[t]; + if( NULL != candidate && candidate->version == in_elem->version ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is a potential alternative source for in_elem %p on data %p", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, in_elem, original); + if(PARSEC_DATA_COHERENCY_INVALID == candidate->coherency_state) { + /* We're already pulling this data on candidate... + * If there is another candidate that already has it, we'll use + * that one; otherwise, we'll fall back on the CPU version. */ + potential_alt_src = 1; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is invalid, continuing to look for alternatives", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index); + continue; + } + /* candidate is the best candidate to do D2D. Let's register as a reader for this + * data copy, and we can unlock and schedule the D2D. */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is the best candidate to to Device to Device copy, increasing its readers to %d", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, candidate->readers+1); + parsec_atomic_fetch_inc_int32( &candidate->readers ); + undo_readers_inc_if_no_transfer = 1; + /* We swap data_in with candidate, so we update the reference counters */ + PARSEC_OBJ_RETAIN(candidate); + release_after_data_in_is_attached = task_data->data_in; + task_data->data_in = candidate; + in_elem = candidate; + in_elem_dev = target; + goto src_selected; + } + } + } + if( potential_alt_src ) { + /* We found a potential alternative source, but it's not ready now, + * we delay the scheduling of this task. */ + /** TODO: when considering RW acccesses, don't forget to unchop gpu_elem + * from its queue... */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tThere is a potential alternative source for in_elem %p [ref_count %d] in original %p to go in copy %p [ref_count %d], but it is not ready, falling back on CPU source", + gpu_device->super.name, in_elem, in_elem->super.super.obj_reference_count, original, gpu_elem, gpu_elem->super.super.obj_reference_count); + //return PARSEC_HOOK_RETURN_NEXT; + } + + /* We fall back on the CPU copy */ + assert( original->device_copies[0] != NULL && in_elem->version == original->device_copies[0]->version ); + in_elem = original->device_copies[0]; + assert(task_data->data_in == in_elem); + assert(in_elem->device_index == 0); + in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(in_elem->device_index); + } + + src_selected: + transfer_from = parsec_data_start_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + if( PARSEC_FLOW_ACCESS_WRITE & type && gpu_task->task_type != GPU_TASK_TYPE_PREFETCH ) { + gpu_elem->version++; /* on to the next version */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] increments version to %d at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, + __FILE__, __LINE__); + } + + gpu_device->super.required_data_in += nb_elts; + if( -1 != transfer_from ) { + /* If it is already under transfer, don't schedule the transfer again. + * This happens if the task refers twice (or more) to the same input flow */ + if( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p): data copy is already under transfer, ignoring double request", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*) + in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*) + gpu_elem->device_private); + } else { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p)", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*)in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*)gpu_elem->device_private); + + assert((gpu_elem->version < in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + parsec_profile_data_collection_info_t info; + + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + assert( GPU_TASK_TYPE_PREFETCH != gpu_task->task_type ); + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = -1; + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH) ) { + gpu_task->prof_key_end = parsec_gpu_prefetch_key_end; + gpu_task->prof_event_id = (int64_t)gpu_elem->device_private; + gpu_task->prof_tp_id = level_zero_device->level_zero_index; + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_prefetch_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } + if(GPU_TASK_TYPE_PREFETCH != gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) ) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_start, + (int64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, + &info); + } + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) { + parsec_device_level_zero_memory_prof_info_t _info; + _info.size = (uint64_t)nb_elts; + _info.data_key = gpu_elem->original->key; + _info.dc_id = (uint64_t)(gpu_elem->original->dc); + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_gpu_use_memory_key_start, (uint64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, &_info, PARSEC_PROFILING_EVENT_HAS_INFO); + } + } +#endif + /* Push data into the GPU from the source device */ + if(PARSEC_SUCCESS != gpu_task->stage_in(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_in"); + if( in_elem_dev->super.super.type != PARSEC_DEV_LEVEL_ZERO ) + parsec_warning("<<%p>> -> <<%p on LEVEL_ZERO device %d>> [%d, H2D]", + in_elem->device_private, gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + else + parsec_warning("<<%p on LEVEL_ZERO device %d>> -> <<%p on LEVEL_ZERO device %d>> [%d, D2D]", + in_elem->device_private, in_elem_dev->level_zero_index, + gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + assert(0); + return -1; + } + assert(in_elem_dev->super.super.device_index < gpu_device->super.data_in_array_size); + gpu_device->super.data_in_from_device[in_elem_dev->super.super.device_index] += nb_elts; + if( PARSEC_GPU_TASK_TYPE_KERNEL == gpu_task->task_type ) + gpu_device->super.nb_data_faults += nb_elts; + + /* update the data version in GPU immediately, and mark the data under transfer */ + assert((gpu_elem->version != in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + gpu_elem->version = in_elem->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets the same version %d as copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, in_elem, in_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + + gpu_elem->data_transfer_status = PARSEC_DATA_STATUS_UNDER_TRANSFER; + } + gpu_elem->push_task = gpu_task->ec; /* only the task who does the transfer can modify the data status later. */ + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + return 1; + } + if( undo_readers_inc_if_no_transfer ) + parsec_atomic_fetch_dec_int32( &in_elem->readers ); + assert( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER ); + + parsec_data_end_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tNO Move %s for data copy %p [ref_count %d, key %x] of %d bytes (host v:%d / device v:%d)", + gpu_device->super.name, + (NULL == in_elem_dev) ? "h2d" : (in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D"), + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem->version, gpu_elem->version); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + /* TODO: data keeps the same coherence flags as before */ + return 0; +} + +static parsec_flow_t parsec_level_zero_data_prefetch_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_data_prefetch_tc = { + .name = "LEVEL_ZERO PREFETCH", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_data_prefetch_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static int +parsec_level_zero_destroy_task(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_task_t** out_task) +{ +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + parsec_gpu_task_t *gpu_task = *out_task; + (void)gpu_device; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: Destroying task %s (%p with ec %p)", + gpu_device->super.name, parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task, gpu_task->ec); + assert( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type || GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ); + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type) PARSEC_DATA_COPY_RELEASE( gpu_task->ec->data[0].data_in); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, "GPU[%s]: gpu_task %p freed at %s:%d\n", + gpu_device->super.name, gpu_task, __FILE__, __LINE__); + free( gpu_task->ec ); + gpu_task->ec = NULL; + free( *out_task ); + *out_task = NULL; + return 0; +} + +#if defined(PARSEC_DEBUG_NOISIER) +static char *parsec_level_zero_debug_advice_to_string(int advice) +{ + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + return "Prefetch"; + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + return "Set Preferred Device"; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return "Mark data as recently used"; + default: + assert(0); + return "Undefined advice"; + } +} +#endif + +static int +parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)dev; +#if defined(PARSEC_DEBUG_NOISIER) + char buffer[64]; + if(NULL != data->dc) { + data->dc->key_to_string(data->dc, data->key, buffer, 64); + } else { + snprintf(buffer, 64, "unbound data"); + } +#endif + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: User provides advice %s of %s (%p)", + gpu_device->super.name, + parsec_level_zero_debug_advice_to_string(advice), + buffer, + data); + + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + data->preferred_device = dev->device_index; + /* We continue on to the next case, as we want to also + * prefetch the data on the target device, if it is the + * preferred device */ + break; //__attribute__ ((fallthrough)); + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + { + if( parsec_type_contiguous(data->device_copies[ data->owner_device ]->dtt) != PARSEC_SUCCESS){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + " PARSEC_DEV_DATA_ADVICE_PREFETCH cannot be applied to non contiguous types "); + return PARSEC_ERROR; + } + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_PREFETCH; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_data_prefetch_tc; + gpu_task->flow[0] = &parsec_level_zero_data_prefetch_flow; + gpu_task->flow_nb_elts[0] = data->device_copies[ data->owner_device ]->original->nb_elts; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + PARSEC_DEBUG_VERBOSE(20, parsec_debug_output, "Retain data copy %p [ref_count %d] at %s:%d", + data->device_copies[ data->owner_device ], + data->device_copies[ data->owner_device ]->super.super.obj_reference_count, + __FILE__, __LINE__); + PARSEC_OBJ_RETAIN(data->device_copies[ data->owner_device ]); + gpu_task->ec->data[0].data_in = data->device_copies[ data->owner_device ]; + gpu_task->ec->data[0].data_out = NULL; + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] linked to prefetch gpu task %p on GPU copy %p [ref_count %d]", + gpu_device->super.name, gpu_task->ec->data[0].data_in, gpu_task->ec->data[0].data_in->super.super.obj_reference_count, + gpu_task, gpu_task->ec->data[0].data_out, gpu_task->ec->data[0].data_out->super.super.obj_reference_count); + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_SUCCESS; + } + break; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return PARSEC_ERR_NOT_IMPLEMENTED; + break; + default: + assert(0); + return PARSEC_ERR_NOT_FOUND; + } + return PARSEC_SUCCESS; +} + +#if PARSEC_GPU_USE_PRIORITIES + +static inline parsec_list_item_t* parsec_push_task_ordered( parsec_list_t* list, + parsec_list_item_t* elem ) +{ + parsec_list_push_sorted(list, elem, parsec_execution_context_priority_comparator); + return elem; +} +#define PARSEC_PUSH_TASK parsec_push_task_ordered +#else +#define PARSEC_PUSH_TASK parsec_list_push_back +#endif + +static parsec_flow_t parsec_level_zero_d2d_complete_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_d2d_complete_tc = { + .name = "D2D TRANSFER COMPLETE", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_d2d_complete_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static void +parsec_gpu_send_transfercomplete_cmd_to_device(parsec_data_copy_t *copy, + parsec_device_module_t *current_dev, + parsec_device_module_t *dst_dev) +{ + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_D2D_COMPLETE; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_d2d_complete_tc; + gpu_task->flow[0] = &parsec_level_zero_d2d_complete_flow; + gpu_task->flow_nb_elts[0] = copy->original->nb_elts; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + gpu_task->ec->data[0].data_in = copy; /* We need to set not-null in data_in, so that the fake flow is + * not ignored when poping the data from the fake task */ + gpu_task->ec->data[0].data_out = copy; /* We "free" data[i].data_out if its readers reaches 0 */ + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* D2D complete tasks are pure internal management, we do not trace them */ +#endif + (void)current_dev; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] D2D transfer is complete, sending order to count it " + "to LEVEL_ZERO Device %s", + current_dev->name, gpu_task->ec->data[0].data_out, + gpu_task->ec->data[0].data_out->super.super.obj_reference_count, + dst_dev->name); + parsec_fifo_push( &(((parsec_device_gpu_module_t*)dst_dev)->pending), (parsec_list_item_t*)gpu_task ); +} + +static int +parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t **gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + (void)gpu_stream; + + parsec_gpu_task_t *gtask = *gpu_task; + parsec_task_t *task; + int32_t i; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; + char task_str2[MAX_TASK_STRLEN]; +#endif + const parsec_flow_t *flow; + /** + * Even though level_zero fence returns success, the PUSH may not be + * completed if no PUSH is required by this task and the PUSH is + * actually done by another task, so we need to check if the data is + * actually ready to use + */ + assert(gpu_stream == gpu_device->exec_stream[0]); + task = gtask->ec; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: parsec_gpu_callback_complete_push, PUSH of %s", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task)); + + for( i = 0; i < task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->data[i].data_in ) continue; + /* We also don't push back non-parsec-owned copies */ + if(NULL != task->data[i].data_out && + 0 == (task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED)) continue; + + flow = gtask->flow[i]; + assert( flow ); + assert( flow->flow_index == i ); + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if(task->data[i].data_out->push_task == task ) { /* only the task who did this PUSH can modify the status */ + parsec_atomic_lock(&task->data[i].data_out->original->lock); + task->data[i].data_out->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER; + parsec_data_end_transfer_ownership_to_copy(task->data[i].data_out->original, + gpu_device->super.device_index, + flow->flow_flags); +#if defined(PARSEC_PROF_TRACE) + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_end, + (int64_t)(int64_t)task->data[i].data_out->device_private, + level_zero_device->level_zero_index, + NULL); + } +#endif + task->data[i].data_out->push_task = NULL; + parsec_atomic_unlock(&task->data[i].data_out->original->lock); + parsec_device_gpu_module_t *src_device = + (parsec_device_gpu_module_t*)parsec_mca_device_get( task->data[i].data_in->device_index ); + if( PARSEC_DEV_LEVEL_ZERO == src_device->super.type ) { + int om; + while(1) { + /* There are two ways out: + * either we exit with om = 0, and then nobody was managing src_device, + * and nobody can start managing src_device until we make it change from -1 to 0 + * (but anybody who has work to do will wait until that happens), or + * we exit with om > 0, then there is a manager for that thread, and we have + * increased mutex to warn the manager that there is another task for it to do. + */ + om = src_device->mutex; + if(om == 0) { + /* Nobody at the door, let's try to lock the door */ + if( parsec_atomic_cas_int32(&src_device->mutex, 0, -1) ) + break; + continue; + } + if(om < 0 ) { + /* Damn, another thread is also trying to do an atomic operation on src_device, + * we give it some time and try again */ + struct timespec delay; + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + continue; + } + /* There is a manager, let's try to reserve another task to do. + * If that fails, the manager may have leaved, try a gain. */ + if( parsec_atomic_cas_int32(&src_device->mutex, om, om+1) ) + break; + } + if( 0 == om ) { + int rc; + /* Nobody is at the door to handle that event on the source of that data... + * we do the command directly */ + parsec_atomic_lock( &task->data[i].data_in->original->lock ); + task->data[i].data_in->readers--; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tExecuting D2D transfer complete for copy %p [ref_count %d] for " + "device %s -- readers now %d", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, src_device->super.name, + task->data[i].data_in->readers); + assert(task->data[i].data_in->readers >= 0); + if(0 == task->data[i].data_in->readers) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count); + parsec_list_item_ring_chop((parsec_list_item_t*)task->data[i].data_in); + PARSEC_LIST_ITEM_SINGLETON(task->data[i].data_in); + parsec_list_push_back(&src_device->gpu_mem_lru, (parsec_list_item_t*)task->data[i].data_in); + src_device->data_avail_epoch++; + } + parsec_atomic_unlock( &task->data[i].data_in->original->lock ); + /* Notify any waiting thread that we're done messing with that device structure */ + rc = parsec_atomic_cas_int32(&src_device->mutex, -1, 0); (void)rc; + assert(rc); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tSending D2D transfer complete command to %s for copy %p " + "[ref_count %d] -- readers is still %d", + gpu_device->super.name, src_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, task->data[i].data_in->readers); + parsec_gpu_send_transfercomplete_cmd_to_device(task->data[i].data_in, + (parsec_device_module_t*)gpu_device, + (parsec_device_module_t*)src_device); + } + } + continue; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tparsec_gpu_callback_complete_push, PUSH of %s: task->data[%d].data_out = %p [ref_count = %d], and push_task is %s, %s because transfer_status is %d", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task), + i, task->data[i].data_out, task->data[i].data_out->super.super.obj_reference_count, + (NULL != task->data[i].data_out->push_task) ? parsec_task_snprintf(task_str2, MAX_TASK_STRLEN, task->data[i].data_out->push_task) : "(null)", + (task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER) ? "all is good" : "Assertion", + task->data[i].data_out->data_transfer_status); + assert(task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + if( task->data[i].data_out->data_transfer_status != PARSEC_DATA_STATUS_COMPLETE_TRANSFER ) { /* data is not ready */ + /** + * As long as we have only one stream to push the data on the GPU we should never + * end up in this case. Remove previous assert if changed. + */ + return -1; + } + } + gtask->complete_stage = NULL; + + if( GPU_TASK_TYPE_PREFETCH == gtask->task_type ) { + parsec_data_copy_t *gpu_copy = task->data[0].data_out; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + assert(NULL != gpu_copy); + if( NULL != gpu_copy->original->dc ) + gpu_copy->original->dc->key_to_string(gpu_copy->original->dc, gpu_copy->original->key, tmp, MAX_TASK_STRLEN); + else + snprintf(tmp, MAX_TASK_STRLEN, "unbound data"); +#endif + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tPrefetch for data copy %p [ref_count %d] (%s) done. readers = %d, device_index = %d, version = %d, flags = %d, state = %d, data_transfer_status = %d", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, + tmp, + gpu_copy->readers, gpu_copy->device_index, gpu_copy->version, + gpu_copy->flags, gpu_copy->coherency_state, gpu_copy->data_transfer_status); + gpu_copy->readers--; + if( 0 == gpu_copy->readers ) { + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tMake copy %p [ref_count %d] available after prefetch from gpu_task %p, ec %p", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, gtask, gtask->ec); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + return parsec_level_zero_destroy_task(gpu_device, gpu_task); + } + + return 0; +} + +/** + * This function tries to progress a stream, by picking up a ready task + * and applying the progress function. The task to be progresses is + * always the highest priority in the waiting queue, even when a task + * has been specified as an input argument. + * The progress function is either specified by the caller via the + * upstream_progress_fct input argument or by the next task to be progresses + * via the submit function associated with the task. In any case, this + * function progresses a single task, which is then returned as the + * out_task parameter. + * + * Beware: this function does not generate errors by itself, instead + * it propagates upward the return code of the progress function. + * However, by convention the error code follows the parsec_hook_return_e + * enum. + */ +static inline int +progress_stream( parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* stream, + parsec_advance_task_function_t upstream_progress_fct, + parsec_gpu_task_t* task, + parsec_gpu_task_t** out_task ) +{ + parsec_advance_task_function_t progress_fct; + int saved_rc = 0, rc; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; +#endif + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)stream; + + /* We always handle the tasks in order. Thus if we got a new task, add it to the + * local list (possibly by reordering the list). Also, as we can return a single + * task first try to see if anything completed. */ + if( NULL != task ) { + PARSEC_PUSH_TASK(stream->fifo_pending, (parsec_list_item_t*)task); + task = NULL; + } + *out_task = NULL; + progress_fct = upstream_progress_fct; + + if( NULL != stream->tasks[stream->end] ) { + rc = zeFenceQueryStatus(level_zero_stream->fences[stream->end]); + if( ZE_RESULT_SUCCESS == rc ) { + /* Save the task for the next step */ + task = *out_task = stream->tasks[stream->end]; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: Completed %s priority %d on stream %s{%p}", + gpu_device->super.name, + parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task->ec), + task->ec->priority, stream->name, (void*)stream); + stream->tasks[stream->end] = NULL; + rc = zeCommandListReset(level_zero_stream->command_lists[stream->end]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", rc, {} ); + stream->end = (stream->end + 1) % stream->max_events; + +#if defined(PARSEC_PROF_TRACE) + if( stream->prof_event_track_enable ) { + if( task->prof_key_end != -1 ) { + PARSEC_PROFILING_TRACE(stream->profiling, task->prof_key_end, task->prof_event_id, task->prof_tp_id, NULL); + } + } +#endif /* (PARSEC_PROF_TRACE) */ + + rc = PARSEC_HOOK_RETURN_DONE; + if (task->complete_stage) + rc = task->complete_stage(gpu_device, out_task, stream); + /* the task can be withdrawn by the system */ + return rc; + } + if( ZE_RESULT_NOT_READY != rc ) { + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", rc, + {return PARSEC_HOOK_RETURN_AGAIN;} ); + } + } + + grab_a_task: + if( NULL == stream->tasks[stream->start] ) { /* there is room on the stream */ + task = (parsec_gpu_task_t*)parsec_list_pop_front(stream->fifo_pending); /* get the best task */ + } + if( NULL == task ) { /* No tasks, we're done */ + return saved_rc; + } + PARSEC_LIST_ITEM_SINGLETON((parsec_list_item_t*)task); + + assert( NULL == stream->tasks[stream->start] ); + /** + * In case the task is succesfully progressed, the corresponding profiling + * event is triggered. + */ + if ( NULL == upstream_progress_fct ) { + /* Grab the submit function */ + progress_fct = task->submit; +#if defined(PARSEC_DEBUG_PARANOID) + int i; + const parsec_flow_t *flow; + for( i = 0; i < task->ec->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->ec->data[i].data_in ) continue; + + flow = task->flow[i]; + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if( 0 == (task->ec->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + assert(task->ec->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + } +#endif /* defined(PARSEC_DEBUG_PARANOID) */ + } + + /* Prepare the command list and the Fence before progressing the function since it may + * enqueue stuff to the command list */ + ze_result_t ze_rc = zeCommandListReset(level_zero_stream->command_lists[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); + ze_rc = zeFenceReset(level_zero_stream->fences[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { } ); + + rc = progress_fct( gpu_device, task, stream ); + if( 0 > rc ) { + if( PARSEC_HOOK_RETURN_AGAIN != rc && + PARSEC_HOOK_RETURN_ASYNC != rc ) { + *out_task = task; + return rc; + } + + if( PARSEC_HOOK_RETURN_ASYNC == rc ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU task %p has been removed by the progress function", + gpu_device->super.name, (void*)task); + } else { + parsec_fifo_push(stream->fifo_pending, (parsec_list_item_t*)task); + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Reschedule task %p: no room available on the GPU for data", + gpu_device->super.name, (void*)task->ec); + } + *out_task = NULL; + return PARSEC_HOOK_RETURN_DONE; + } + /** + * Do not skip the level_zero fence generation. The problem is that some of the inputs + * might be in the pipe of being transferred to the GPU. If we activate this task + * too early, it might get executed before the data is available on the GPU. + * Obviously, this lead to incorrect results. + */ + ze_rc = zeCommandListClose(level_zero_stream->command_lists[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[stream->start], level_zero_stream->fences[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); + + stream->tasks[stream->start] = task; + stream->start = (stream->start + 1) % stream->max_events; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]: Submitted %s(task %p) priority %d on stream %s{%p}", + gpu_device->super.name, + task->ec->task_class->name, (void*)task->ec, task->ec->priority, + stream->name, (void*)stream); + + task = NULL; + goto grab_a_task; +} + +/** + * This function schedule the move of all the data required for a + * specific task from the main memory into the GPU memory. + * + * Returns: + * a positive number: the number of data to be moved. + * -1: data cannot be moved into the GPU. + * -2: No more room on the GPU to move this data. + */ +static int +parsec_gpu_kernel_push( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + parsec_task_t *this_task = gpu_task->ec; + const parsec_flow_t *flow; + int i, ret = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + +#if 0 + if( gpu_task->last_data_check_epoch == gpu_device->data_avail_epoch ) + return PARSEC_HOOK_RETURN_AGAIN; +#endif + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Push %s", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task) ); + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type ) { + if( NULL == gpu_task->ec->data[0].data_in->original ) { + /* The PREFETCH order comes after the copy was detached and released, ignore it */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s has been released already, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task)); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + if( NULL != gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index] && + gpu_task->ec->data[0].data_in->original->owner_device == gpu_device->super.device_index ) { + /* There is already a copy of this data in the GPU */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s data_copy at index %d is %p, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_device->super.device_index, + gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index]); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + } + + /* Do we have enough available memory on the GPU to hold the input and output data ? */ + ret = parsec_gpu_data_reserve_device_space( level_zero_device, gpu_task ); + if( ret < 0 ) { + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch; + return ret; + } + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + + flow = gpu_task->flow[i]; + /* Skip CTL flows */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* If there is already a GPU data copy (set by reserve_device_space), and this copy + * is not parsec-owned, don't stage in */ + if( NULL != this_task->data[i].data_out && + (0 == (this_task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) ) continue; + + assert( NULL != parsec_data_copy_get_ptr(this_task->data[i].data_in) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\t\tIN Data of %s <%x> on GPU", + gpu_device->super.name, flow->name, + this_task->data[i].data_out->original->key); + ret = parsec_gpu_data_stage_in( level_zero_device, flow, + &(this_task->data[i]), gpu_task, gpu_stream ); + if( ret < 0 ) { + return ret; + } + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Push task %s DONE", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + gpu_task->complete_stage = parsec_gpu_callback_complete_push; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* We do not log that event as the completion of this task */ +#endif + return ret; +} + +/** + * This function schedule the move of all the modified data for a + * specific task from the GPU memory into the main memory. + * + * Returns: negative number if any error occured. + * positive: the number of data to be moved. + */ +static int +parsec_gpu_kernel_pop( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy; + parsec_data_t *original; + uint32_t nb_elts; + const parsec_flow_t *flow; + int return_code = 0, how_many = 0, i, update_data_epoch = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + for( i = 0; i < this_task->locals[0].value; i++ ) { + gpu_copy = this_task->data[i].data_out; + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + original = gpu_copy->original; + if(PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << i), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + goto release_and_return_error; + } + } + return return_code; + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Pop %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* We need to manage all data that has been used as input, even if they were read only */ + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + flow = gpu_task->flow[i]; + if( PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags) ) continue; /* control flow */ + + gpu_copy = this_task->data[i].data_out; + + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + original = gpu_copy->original; + nb_elts = gpu_task->flow_nb_elts[i]; + + assert( this_task->data[i].data_in == NULL || original == this_task->data[i].data_in->original ); + + if( !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Do not propagate GPU copies to successors (temporary solution) */ + this_task->data[i].data_out = original->device_copies[0]; + } + parsec_atomic_lock(&original->lock); + if( flow->flow_flags & PARSEC_FLOW_ACCESS_READ ) { + gpu_copy->readers--; + if( gpu_copy->readers < 0 ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: While trying to Pop %s, gpu_copy %p [ref_count %d] on flow %d with original %p had already 0 readers", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), + gpu_copy, gpu_copy->super.super.obj_reference_count, + i, original); + + } + assert(gpu_copy->readers >= 0); + if( (0 == gpu_copy->readers) && + !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available on flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); /* TODO: singleton instead? */ + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + update_data_epoch = 1; + parsec_atomic_unlock(&original->lock); + continue; /* done with this element, go for the next one */ + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tread copy %p [ref_count %d] on flow %s has readers (%i)", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name, gpu_copy->readers); + } + if( flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE ) { + assert( gpu_copy == parsec_data_get_copy(gpu_copy->original, gpu_device->super.device_index) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tOUT Data copy %p [ref_count %d] for flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + + /* Stage the transfer of the data back to main memory */ + gpu_device->super.required_data_out += nb_elts; + assert( ((parsec_list_item_t*)gpu_copy)->list_next == (parsec_list_item_t*)gpu_copy ); + assert( ((parsec_list_item_t*)gpu_copy)->list_prev == (parsec_list_item_t*)gpu_copy ); + + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + if( gpu_task->pushout & (1 << i) ) { + /* TODO: make sure no readers are working on the CPU version */ + original = gpu_copy->original; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tMove D2H data <%s:%x> copy %p [ref_count %d] -- D:%p -> H:%p requested", + gpu_device->super.name, flow->name, original->key, gpu_copy, gpu_copy->super.super.obj_reference_count, + (void*)gpu_copy->device_private, original->device_copies[0]->device_private); +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT) { + parsec_profile_data_collection_info_t info; + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = parsec_gpu_moveout_key_end; + gpu_task->prof_tp_id = this_task->taskpool->taskpool_id; + gpu_task->prof_event_id = this_task->task_class->key_functions->key_hash(this_task->task_class->make_key(this_task->taskpool, this_task->locals), NULL); + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_moveout_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } else { + gpu_task->prof_key_end = -1; + } + } +#endif + /* Move the data back into main memory */ + if( PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + parsec_atomic_unlock(&original->lock); + goto release_and_return_error; + } + gpu_device->super.data_out_to_host += nb_elts; /* TODO: not hardcoded, use datatype size */ + how_many++; + } else { + assert( 0 == gpu_copy->readers ); + } + } + parsec_atomic_unlock(&original->lock); + } + + release_and_return_error: + if( update_data_epoch ) { + gpu_device->data_avail_epoch++; + } + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Pop %s DONE (return %d data epoch %"PRIu64")", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), return_code, gpu_device->data_avail_epoch ); + + return (return_code < 0 ? return_code : how_many); +} + +/** + * Make sure all data on the device is correctly put back into the queues. + */ +static int +parsec_gpu_kernel_epilog( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Epilog of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + + + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + cpu_copy = original->device_copies[0]; + + /* If it is a copy managed by the user, don't bother either */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + /** + * There might be a race condition here. We can't assume the first CPU + * version is the corresponding CPU copy, as a new CPU-bound data + * might have been created meanwhile. + * + * WARNING: For now we always forward the cpu_copy to the next task, to + * do that, we lie to the engine by updating the CPU copy to the same + * status than the GPU copy without updating the data itself. Thus, the + * cpu copy is really invalid. this is related to Issue #88, and the + * fact that: + * - we don't forward the gpu copy as output + * - we always take a cpu copy as input, so it has to be in the + * same state as the GPU to prevent an extra data movement. + */ + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + + /** + * The cpu_copy will be updated in the completion, and at that moment + * the two versions will be identical. + */ + cpu_copy->version = gpu_copy->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: CPU copy %p [ref_count %d] gets the same version %d as GPU copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + cpu_copy, cpu_copy->super.super.obj_reference_count, cpu_copy->version, gpu_copy, gpu_copy->super.super.obj_reference_count, + __FILE__, __LINE__); + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + + assert( 0 == gpu_copy->readers ); + + if( gpu_task->pushout & (1 << i) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the owned LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_push_back(&gpu_device->gpu_mem_owned_lru, (parsec_list_item_t*)gpu_copy); + } + } + return 0; +} + +/** @brief Release the LEVEL_ZERO copies of the data used in WRITE mode. + * + * @details This function can be used when the LEVEL_ZERO task didn't run + * to completion on the device (either due to an error, or + * simply because the body requested a reexecution on a + * different location). It releases the LEVEL_ZERO copies of the + * output data, allowing them to be reused by the runtime. + * This function has the drawback of kicking in too late, + * after all data transfers have been completed toward the + * device. + * + * @param [IN] gpu_device, the GPU device the the task has been + * supposed to execute. + * @param [IN] gpu_task, the task that has been cancelled, and which + * needs it's data returned to the runtime. + * @return Currently only success. + */ +static int +parsec_gpu_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i, data_avail_epoch = 0; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Cleanup of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + parsec_atomic_lock(&original->lock); + assert(gpu_copy->super.super.obj_reference_count > 1); + /* Issue #134 */ + parsec_data_copy_detach(original, gpu_copy, gpu_device->super.device_index); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy = original->device_copies[0]; + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + if( 0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + parsec_atomic_unlock(&original->lock); + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s\n", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + } + if( data_avail_epoch ) /* Update data availability epoch */ + gpu_device->data_avail_epoch++; + return 0; +} + +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on LEVEL_ZERO events, + * such an event indicate that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ) +{ + parsec_device_gpu_module_t* gpu_device; + int rc, exec_stream = 0; + parsec_gpu_task_t *progress_task, *out_task_submit = NULL, *out_task_pop = NULL; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + int pop_null = 0; + + gpu_device = (parsec_device_gpu_module_t*)parsec_mca_device_get(which_gpu); + +#if defined(PARSEC_PROF_TRACE) + PARSEC_PROFILING_TRACE_FLAGS( es->es_profile, + PARSEC_PROF_FUNC_KEY_END(gpu_task->ec->taskpool, + gpu_task->ec->task_class->task_class_id), + gpu_task->ec->task_class->key_functions->key_hash(gpu_task->ec->task_class->make_key(gpu_task->ec->taskpool, gpu_task->ec->locals), NULL), + gpu_task->ec->taskpool->taskpool_id, NULL, + PARSEC_PROFILING_EVENT_RESCHEDULED ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + /* Check the GPU status -- three kinds of values for rc: + * - rc < 0: somebody is doing a short atomic operation while there is no manager, + * so wait. + * - rc == 0: there is no manager, and at the exit of the while, this thread + * made rc go from 0 to 1, so it is the new manager of the GPU and + * needs to deal with gpu_task + * - rc > 0: there is a manager, and at the exit of the while, this thread has + * committed new work that the manager will need to do, but the work is + * not in the queue yet. + */ + while(1) { + rc = gpu_device->mutex; + struct timespec delay; + if( rc >= 0 ) { + if( parsec_atomic_cas_int32( &gpu_device->mutex, rc, rc+1 ) ) { + break; + } + } else { + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + } + } + if( 0 < rc ) { + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_HOOK_RETURN_ASYNC; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Entering GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_start, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + check_in_deps: + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tUpload data (if any) for %s priority %d", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[0], + parsec_gpu_kernel_push, + gpu_task, &progress_task ); + if( rc < 0 ) { /* In case of error progress_task is the task that raised it */ + if( -1 == rc ) + goto disable_gpu; + /* We are in the early stages, and if there no room on the GPU for a task we need to + * delay all retries for the same task for a little while. Meanwhile, put the task back + * trigger a device flush, and keep executing tasks that have their data on the device. + */ + if( NULL != progress_task ) { + PARSEC_PUSH_TASK(gpu_device->exec_stream[0]->fifo_pending, (parsec_list_item_t*)progress_task); + progress_task = NULL; + } + /* If we can extract data go for it, otherwise try to drain the pending tasks */ + gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); + if( NULL != gpu_task ) + goto get_data_out_of_device; + } + gpu_task = progress_task; + + /* Stage-in completed for this task: it is ready to be executed */ + exec_stream = (exec_stream + 1) % (gpu_device->max_exec_streams - 2); /* Choose an exec_stream */ + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tExecute %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[2+exec_stream], + NULL, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( PARSEC_HOOK_RETURN_DISABLE == rc ) + goto disable_gpu; + if( PARSEC_HOOK_RETURN_ASYNC != rc ) { + /* Reschedule the task. As the chore_id has been modified, + another incarnation of the task will be executed. */ + if( NULL != progress_task ) { + parsec_gpu_kernel_cleanout(gpu_device, progress_task); + __parsec_reschedule(es, progress_task->ec); + gpu_task = progress_task; + progress_task = NULL; + goto remove_gpu_task; + } + gpu_task = NULL; + goto fetch_task_from_shared_queue; + } + progress_task = NULL; + } + gpu_task = progress_task; + out_task_submit = progress_task; + + get_data_out_of_device: + if( NULL != gpu_task ) { /* This task has completed its execution */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tRetrieve data (if any) for %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + /* Task is ready to move the data back to main memory */ + rc = progress_stream( gpu_device, + gpu_device->exec_stream[1], + parsec_gpu_kernel_pop, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( -1 == rc ) + goto disable_gpu; + } + if( NULL != progress_task ) { + /* We have a succesfully completed task. However, it is not gpu_task, as + * it was just submitted into the data retrieval system. Instead, the task + * ready to move into the next level is the progress_task. + */ + gpu_task = progress_task; + progress_task = NULL; + goto complete_task; + } + gpu_task = progress_task; + out_task_pop = progress_task; + + fetch_task_from_shared_queue: + assert( NULL == gpu_task ); + if (1 == parsec_level_zero_sort_pending && out_task_submit == NULL && out_task_pop == NULL) { + parsec_gpu_sort_pending_list(gpu_device); + } + gpu_task = (parsec_gpu_task_t*)parsec_fifo_try_pop( &(gpu_device->pending) ); + if( NULL != gpu_task ) { + pop_null = 0; + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch - 1; /* force at least one tour */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tGet from shared queue %s priority %d", gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority); + if( GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ) { + goto get_data_out_of_device; + } + } else { + pop_null++; + if( pop_null % 1024 == 1023 ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, "GPU[%s]:\tStill waiting for %d tasks to execute, but poped NULL the last %d times I tried to pop something...", + gpu_device->super.name, gpu_device->mutex, pop_null); + } + } + goto check_in_deps; + + complete_task: + assert( NULL != gpu_task ); + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tComplete %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec)); + /* Everything went fine so far, the result is correct and back in the main memory */ + PARSEC_LIST_ITEM_SINGLETON(gpu_task); + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + parsec_gpu_complete_w2r_task(gpu_device, gpu_task, es); + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + } + if (gpu_task->task_type == GPU_TASK_TYPE_D2D_COMPLETE) { + free( gpu_task->ec ); + gpu_task->ec = NULL; + goto remove_gpu_task; + } + parsec_gpu_kernel_epilog( gpu_device, gpu_task ); + __parsec_complete_execution( es, gpu_task->ec ); + gpu_device->super.executed_tasks++; + remove_gpu_task: + // Load problem: was parsec_device_load[gpu_device->super.device_index] -= gpu_task->load; + parsec_device_load[gpu_device->super.device_index] -= parsec_device_sweight[gpu_device->super.device_index]; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream,"GPU[%s]: gpu_task %p freed at %s:%d", gpu_device->super.name, + gpu_task, __FILE__, __LINE__); + free( gpu_task ); + rc = parsec_atomic_fetch_dec_int32( &(gpu_device->mutex) ); + if( 1 == rc ) { /* I was the last one */ +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_end, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Leaving GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + + return PARSEC_HOOK_RETURN_ASYNC; + } + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + + disable_gpu: + /* Something wrong happened. Push all the pending tasks back on the + * cores, and disable the gpu. + */ + parsec_warning("Critical issue related to the GPU discovered. Giving up\n"); + return PARSEC_HOOK_RETURN_DISABLE; +} + +parsec_hook_return_t +parsec_dpcpp_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ) +{ + return parsec_level_zero_kernel_scheduler(es, gpu_task, which_gpu); +} +#endif /* PARSEC_HAVE_LEVEL_ZERO */ diff --git a/parsec/mca/device/parsec_gpu.c b/parsec/mca/device/parsec_gpu.c new file mode 100644 index 000000000..84fcb3c25 --- /dev/null +++ b/parsec/mca/device/parsec_gpu.c @@ -0,0 +1,97 @@ +/* + * + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/mca/device/device.h" +#include "parsec/mca/device/device_gpu.h" +#include "parsec/utils/mca_param.h" +#include "parsec/mca/mca_repository.h" +#include "parsec/constants.h" +#include "parsec/utils/debug.h" +#include "parsec/execution_stream.h" +#include "parsec/utils/argv.h" +#include "parsec/parsec_internal.h" + +void* parsec_gpu_pop_workspace(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* gpu_stream, size_t size) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; (void)gpu_stream; (void)size; + void *work = NULL; + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + if (gpu_stream->workspace == NULL) { + gpu_stream->workspace = (parsec_gpu_workspace_t *)malloc(sizeof(parsec_gpu_workspace_t)); + gpu_stream->workspace->total_workspace = PARSEC_GPU_MAX_WORKSPACE; + gpu_stream->workspace->stack_head = PARSEC_GPU_MAX_WORKSPACE - 1; + + for( int i = 0; i < PARSEC_GPU_MAX_WORKSPACE; i++ ) { + gpu_stream->workspace->workspace[i] = zone_malloc( gpu_device->memory, size); + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Succeeded Allocating workspace %d (device_ptr %p)", + gpu_device->super.name, + i, gpu_stream->workspace->workspace[i]); +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + &size, PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif + } + } + assert (gpu_stream->workspace->stack_head >= 0); + work = gpu_stream->workspace->workspace[gpu_stream->workspace->stack_head]; + gpu_stream->workspace->stack_head --; +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return work; +} + +int parsec_gpu_push_workspace(parsec_device_gpu_module_t* gpu_device, parsec_gpu_exec_stream_t* gpu_stream) +{ + (void)gpu_device; (void)gpu_stream; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_stream->workspace->stack_head ++; + assert (gpu_stream->workspace->stack_head < PARSEC_GPU_MAX_WORKSPACE); +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} + +int parsec_gpu_free_workspace(parsec_device_gpu_module_t * gpu_device) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + int i, j; + for( i = 0; i < gpu_device->max_exec_streams; i++ ) { + parsec_gpu_exec_stream_t *gpu_stream = &(level_zero_device->exec_stream[i].super); + if (gpu_stream->workspace != NULL) { + for (j = 0; j < gpu_stream->workspace->total_workspace; j++) { +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + } +#endif + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Release workspace %d (device_ptr %p)", + gpu_device->super.name, + j, gpu_stream->workspace->workspace[j]); + zone_free( gpu_device->memory, gpu_stream->workspace->workspace[j] ); + } + free(gpu_stream->workspace); + gpu_stream->workspace = NULL; + } + } +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index b3a28db19..7ddcec22b 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -18,9 +18,9 @@ #include "parsec/utils/output.h" #include "parsec/scheduling.h" -#if !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) -#error This file should not be included in a non-CUDA/HIP build -#endif /* !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) */ +#if !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) && !defined(PARSEC_HAVE_LEVEL_ZERO) +#error This file should not be included in a non-GPU (CUDA/HIP/LEVEL_ZERO) build +#endif /* !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) && !defined(PARSEC_HAVE_LEVEL_ZERO) */ /** * Entirely local tasks that should only be used to move data between a device and the main memory. Such @@ -133,6 +133,11 @@ static const __parsec_chore_t __gpu_d2h_task_chores[] = { .evaluate = NULL, .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, #endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + {.type = PARSEC_DEV_LEVEL_ZERO, + .evaluate = NULL, + .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, +#endif #if defined(PARSEC_HAVE_HIP) {.type = PARSEC_DEV_HIP, .evaluate = NULL, diff --git a/parsec/parsec_internal.h b/parsec/parsec_internal.h index 2bf4b7382..19f75d75f 100644 --- a/parsec/parsec_internal.h +++ b/parsec/parsec_internal.h @@ -339,6 +339,10 @@ typedef int (parsec_data_ref_fn_t)(parsec_task_t *task, #define PARSEC_TASK_CLASS_TYPE_PTG 0x01 #define PARSEC_TASK_CLASS_TYPE_DTD 0x02 +#if defined(__cplusplus) +#define restrict +#endif + /** * Find the dependency corresponding to a given execution context. */ diff --git a/parsec/remote_dep_mpi.c b/parsec/remote_dep_mpi.c index a55081966..a8f184c2d 100644 --- a/parsec/remote_dep_mpi.c +++ b/parsec/remote_dep_mpi.c @@ -1680,7 +1680,7 @@ remote_dep_mpi_put_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(dataptr, PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, // TODO JS: this interface is so broken, fix it! + -1, (parsec_datatype_t)NULL, // TODO JS: this interface is so broken, fix it! dtt_size, // TODO JS: what about nbdtt? Is it ok to ignore it?! &source_memory_handle, &source_memory_handle_size); @@ -2025,7 +2025,7 @@ static void remote_dep_mpi_get_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(PARSEC_DATA_COPY_GET_PTR(deps->output[k].data.data), PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, + -1, (parsec_datatype_t)NULL, dtt_size, &receiver_memory_handle, &receiver_memory_handle_size); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b489f82e4..023f52607 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -78,6 +78,7 @@ target_link_libraries(tests_common PRIVATE parsec) add_subdirectory(class) add_subdirectory(api) +add_subdirectory(subsystem) if( TARGET parsec-ptgpp ) add_subdirectory(runtime) add_subdirectory(dsl/ptg) diff --git a/tests/dsl/dtd/CMakeLists.txt b/tests/dsl/dtd/CMakeLists.txt index 50abef063..d642c342a 100644 --- a/tests/dsl/dtd/CMakeLists.txt +++ b/tests/dsl/dtd/CMakeLists.txt @@ -24,7 +24,25 @@ parsec_addtest_executable(C dtd_test_interleave_actions SOURCES dtd_test_interle parsec_addtest_executable(C dtd_test_ce SOURCES dtd_test_ce.c) parsec_addtest_executable(C dtd_test_new_tile SOURCES dtd_test_new_tile.c) -if( PARSEC_HAVE_CUDA ) +if( PARSEC_HAVE_LEVEL_ZERO ) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C dtd_test_simple_gemm_lz SOURCES dtd_test_simple_gemm_lz.c dtd_test_simple_gemm_lz_kernel.dpcpp.C) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader m) + find_package(BLAS) + if(BLAS_FOUND) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE ${BLAS_LIBRARIES}) + target_include_directories(dtd_test_simple_gemm_lz PRIVATE ${BLAS_INCLUDE_DIRS}) + target_compile_definitions(dtd_test_simple_gemm_lz PRIVATE HAVE_BLAS=1) + endif(BLAS_FOUND) + endif(TARGET MKL::MKL_DPCPP) +elseif( PARSEC_HAVE_CUDA ) parsec_addtest_executable(C dtd_test_cuda_task_insert SOURCES dtd_test_cuda_task_insert.c) if( TARGET CUDA::cublas ) parsec_addtest_executable(C dtd_test_simple_gemm SOURCES dtd_test_simple_gemm.c) @@ -37,4 +55,4 @@ if( PARSEC_HAVE_CUDA ) endif(BLAS_FOUND) target_sources(dtd_test_new_tile PUBLIC dtd_test_new_tile_cuda_kernels.cu) endif( TARGET CUDA::cublas ) -endif() +endif( PARSEC_HAVE_LEVEL_ZERO ) diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz.c b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c new file mode 100644 index 000000000..769fbf12c --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c @@ -0,0 +1,646 @@ +#include +#include "parsec.h" +#include "parsec/arena.h" +#include "parsec/data_dist/matrix/matrix.h" +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/interfaces/dtd/insert_function_internal.h" + +// The file is not compiled if LEVEL_ZERO is not present or cannot be compiled +#include "parsec/mca/device/cuda/device_cuda.h" + +#if defined(HAVE_BLAS) +// If our CMake finds a BLAS library, it defines HAVE_BLAS +// BLAS does not guarantee there is a cblas.h, we define our own prototype +typedef enum CBLAS_LAYOUT {CblasRowMajor=101, CblasColMajor=102} CBLAS_LAYOUT; +typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113} CBLAS_TRANSPOSE; +typedef enum CBLAS_UPLO {CblasUpper=121, CblasLower=122} CBLAS_UPLO; +typedef enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG; +typedef enum CBLAS_SIDE {CblasLeft=141, CblasRight=142} CBLAS_SIDE; +#define CBLAS_INDEX int + +extern void cblas_dgemm(const CBLAS_LAYOUT layout, const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, const CBLAS_INDEX M, const CBLAS_INDEX N, + const CBLAS_INDEX K, const double alpha, const double *A, + const CBLAS_INDEX lda, const double *B, const CBLAS_INDEX ldb, + const double beta, double *C, const CBLAS_INDEX ldc); +#endif + +#if defined(PARSEC_HAVE_MPI) +#include +#endif /* defined(PARSEC_HAVE_MPI) */ + +#include +#include + +static int TILE_FULL = -1; +int gemm_lz_verbose = 0; +static int device = PARSEC_DEV_LEVEL_ZERO; +static int P = -1; +static int Q = -1; + +#define Rnd64_A 6364136223846793005ULL +#define Rnd64_C 1ULL +#define RndF_Mul 5.4210108624275222e-20f +#define RndD_Mul 5.4210108624275222e-20 +#define NBELEM 1 + +#define EPSILON 1e-10 + +static unsigned long long int Rnd64_jump(unsigned long long int n, unsigned long long int seed) +{ + unsigned long long int a_k, c_k, ran; + int i; + + a_k = Rnd64_A; + c_k = Rnd64_C; + + ran = seed; + for( i = 0; n; n >>= 1, ++i ) { + if( n & 1 ) + ran = a_k * ran + c_k; + c_k *= (a_k + 1); + a_k *= a_k; + } + + return ran; +} + +int initialize_tile(parsec_execution_stream_t *es, parsec_task_t *this_task) +{ + (void)es; + double *data; + int i, j, mb, nb, m, n, M, ld; + unsigned int seed; + unsigned long long jump, ran; + + parsec_dtd_unpack_args(this_task, &data, &m, &n, &mb, &nb, &M, &ld, &seed); + + jump = (unsigned long long int)m + (unsigned long long int)n * (unsigned long long int)M; + + for( j = 0; j < nb; j++ ) { + ran = Rnd64_jump(NBELEM * jump, seed); + for( i = 0; i < mb; i++ ) { + *data = 0.5f - ran * RndF_Mul; + ran = Rnd64_A * ran + Rnd64_C; + data++; + } + data += ld - i; + jump += M; + } + return PARSEC_HOOK_RETURN_DONE; +} + +int initialize_matrix(parsec_context_t *parsec_context, int rank, parsec_matrix_block_cyclic_t *mat, unsigned int seed, + const char *name, int *gpu_device_index, int nb_gpus) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t key; + int perr; + + parsec_task_class_t *init_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + init_tc = parsec_dtd_create_task_class(tp, "init", + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* M */ + sizeof(int), PARSEC_VALUE, /* ld */ + sizeof(unsigned int), PARSEC_VALUE, /* seed */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, init_tc, PARSEC_DEV_CPU, initialize_tile); + + int g = 0; + for( int i = 0; i < mat->super.mt; i++ ) { + for( int j = 0; j < mat->super.nt; j++ ) { + key = mat->super.super.data_key(&mat->super.super, i, j); + parsec_dtd_insert_task_with_task_class(tp, init_tc, 1, PARSEC_DEV_CPU, + PARSEC_PUSHOUT, PARSEC_DTD_TILE_OF_KEY(&mat->super.super, key), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.nb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.m, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &seed, + PARSEC_DTD_ARG_END); + if(PARSEC_DEV_LEVEL_ZERO == device && + (int)mat->super.super.rank_of_key(&mat->super.super, key) == rank ) { + if( gemm_lz_verbose ) { + fprintf(stderr, "Advice %s(%d, %d) to prefer GPU device %d (parsec device %d) of rank %d\n", + name, i, j, g, gpu_device_index[g], (int)mat->super.super.rank_of_key(&mat->super.super, key)); + } + parsec_advise_data_on_device(mat->super.super.data_of_key(&mat->super.super, key), + gpu_device_index[g], + PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE); + g = (g + 1) % nb_gpus; + } + } + } + parsec_dtd_data_flush_all(tp, &mat->super.super); + + // Wait for task completion + perr = parsec_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, init_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); + +#if defined(HAVE_BLAS) +int gemm_kernel_cpu(parsec_execution_stream_t *es, + parsec_task_t *this_task) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + double alpha = 1.0; + double beta = 1.0; + double delta; + struct timeval start, end, diff; + + (void)es; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + gettimeofday(&start, NULL); + cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mb, nb, kb, alpha, A, mb, B, kb, beta, C, mb); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if( gemm_lz_verbose ) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d, %dx%d, %dx%d on node %d, on core %d: %g s\n", + m, n, k, mb, kb, kb, nb, mb, kb, + this_task->taskpool->context->my_rank, + es->core_id, + delta); + + return PARSEC_HOOK_RETURN_DONE; +} +#endif + +int simple_gemm(parsec_context_t *parsec_context, parsec_matrix_block_cyclic_t *A, parsec_matrix_block_cyclic_t *B, parsec_matrix_block_cyclic_t *C) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t keyA, keyB, keyC; + int perr; + + parsec_task_class_t *gemm_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + gemm_tc = parsec_dtd_create_task_class(tp, "GEMM", + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* A */ + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* B */ + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, /* C */ + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* k */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* kb */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_LEVEL_ZERO, gemm_kernel_lz); +#if defined(HAVE_BLAS) + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_CPU, gemm_kernel_cpu); +#endif + + for( int i = 0; i < C->super.mt; i++ ) { + for( int j = 0; j < C->super.nt; j++ ) { + keyC = C->super.super.data_key(&C->super.super, i, j); + for( int k = 0; k < A->super.nt; k++ ) { + keyA = A->super.super.data_key(&A->super.super, i, k); + keyB = B->super.super.data_key(&B->super.super, k, j); + parsec_dtd_insert_task_with_task_class(tp, gemm_tc, C->super.mt*C->super.nt*A->super.nt - i*C->super.nt + j, device, + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&A->super.super, keyA), + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&B->super.super, keyB), + k == A->super.nt - 1 ? (PARSEC_INOUT | PARSEC_PUSHOUT) : PARSEC_INOUT, + PARSEC_DTD_TILE_OF_KEY(&C->super.super, keyC), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &k, + PARSEC_DTD_EMPTY_FLAG, &C->super.mb, + PARSEC_DTD_EMPTY_FLAG, &C->super.nb, + PARSEC_DTD_EMPTY_FLAG, &B->super.mb, + PARSEC_DTD_ARG_END); + } + } + } + parsec_dtd_data_flush_all(tp, &A->super.super); + parsec_dtd_data_flush_all(tp, &B->super.super); + parsec_dtd_data_flush_all(tp, &C->super.super); + + // Wait for task completion + perr = parsec_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, gemm_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int get_nb_gpu_devices() +{ + int nb = 0; + + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + nb++; + } + } + + return nb; +} + +int *get_gpu_device_index() +{ + int *dev_index = NULL; + + dev_index = (int *)malloc(parsec_nb_devices * sizeof(int)); + int i = 0; + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + dev_index[i++] = device->device_index; + } + } + + return dev_index; +} + +static parsec_matrix_block_cyclic_t *create_initialize_matrix(parsec_context_t *parsec_context, int rank, unsigned int seed, const char *name, int mb, int nb, int M, int N, int *gpu_device_index, int nbgpus) +{ + parsec_matrix_block_cyclic_t *dc; + dc = calloc(1, sizeof(parsec_matrix_block_cyclic_t)); + parsec_matrix_block_cyclic_init(dc, PARSEC_MATRIX_DOUBLE, PARSEC_MATRIX_TILE, rank, + mb, nb, + M, N, + 0, 0, + M, N, + P, Q, + 1, 1, + 0, 0); + parsec_data_collection_t *A = &dc->super.super; + parsec_data_collection_set_key(A, name); + dc->mat = parsec_data_allocate((size_t)dc->super.nb_local_tiles * + (size_t)dc->super.bsiz * + (size_t)parsec_datadist_getsizeoftype(dc->super.mtype)); + parsec_dtd_data_collection_init(A); + initialize_matrix(parsec_context, rank, dc, seed, name, gpu_device_index, nbgpus); + + return dc; +} + +static void destroy_matrix(parsec_matrix_block_cyclic_t *dc) +{ + parsec_data_collection_t *A = &dc->super.super; + parsec_dtd_data_collection_fini(A); + if( NULL != dc->mat ) { + parsec_data_free(dc->mat); + } + parsec_tiled_matrix_destroy_data(&dc->super); + parsec_data_collection_destroy(&dc->super.super); + + free(dc); +} + +static void print_matrix(parsec_matrix_block_cyclic_t *dc, parsec_context_t *parsec_context, const char *info) +{ + for( int i = 0; i < dc->super.mt; i++ ) { + for( int j = 0; j < dc->super.nt; j++ ) { + if( (int)dc->super.super.rank_of(&dc->super.super, i, j) == parsec_context->my_rank ) { + fprintf(stderr, "%-5s(%2d, %2d): ", info, i, j); + parsec_data_t *tile = dc->super.super.data_of(&dc->super.super, i, j); + double *mat = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(tile, 0)); + for( int ii = 0; ii < dc->super.mb; ii++) { + for(int jj = 0; jj < dc->super.nb; jj++) { + fprintf(stderr, "%5.2g ", mat[ii*dc->super.nb + jj]); + } + fprintf(stderr, "\n "); + } + } + } + } +} + +int main(int argc, char **argv) +{ + int ret = 0, rc, nbgpus = 0; + parsec_context_t *parsec_context = NULL; + int rank, world; + int mb = 1024, nb = 1024, kb = 1024; + int M = 16 * mb, N = 16 * nb, K = 16 * kb; + double min_perf=0.0; + int runs = 5; + int debug=-1; + int check = 0; + +#if defined(PARSEC_HAVE_MPI) + { + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided); + } + MPI_Comm_size(MPI_COMM_WORLD, &world); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); +#else + world = 1; + rank = 0; +#endif + + while( 1 ) { + int option_index = 0; + static struct option long_options[] = { + {"M", required_argument, 0, 'M'}, + {"N", required_argument, 0, 'N'}, + {"K", required_argument, 0, 'K'}, + {"mb", required_argument, 0, 'm'}, + {"nb", required_argument, 0, 'n'}, + {"kb", required_argument, 0, 'k'}, + {"P", required_argument, 0, 'P'}, + {"Q", required_argument, 0, 'Q'}, + {"device", required_argument, 0, 'd'}, + {"nruns", required_argument, 0, 't'}, + {"verbose", no_argument, 0, 'v'}, + {"Debug", required_argument, 0, 'D'}, + {"Alarm", required_argument, 0, 'A'}, + {"Check", no_argument, 0, 'x'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int c = getopt_long(argc, argv, "M:N:K:m:n:k:P:Q:t:d:D:A:xvh", + long_options, &option_index); + if( c == -1 ) + break; + + switch( c ) { + case 'M': + M = atoi(optarg); + break; + case 'N': + N = atoi(optarg); + break; + case 'K': + K = atoi(optarg); + break; + case 'm': + mb = atoi(optarg); + break; + case 'n': + nb = atoi(optarg); + break; + case 'k': + kb = atoi(optarg); + break; + case 'P': + P = atoi(optarg); + break; + case 'Q': + Q = atoi(optarg); + break; + case 't': + runs = atoi(optarg); + break; + case 'v': + gemm_lz_verbose = !gemm_lz_verbose; + break; + case 'd': + if(strcmp(optarg, "GPU") == 0) { + device=PARSEC_DEV_LEVEL_ZERO; + } else if(strcmp(optarg, "CPU") == 0) { +#if defined(HAVE_BLAS) + device=PARSEC_DEV_CPU; +#else + fprintf(stderr, "Error: requested to run on CPU (--device=CPU), but no BLAS library has been found at configure time\n"); + exit(1); +#endif + } else { + fprintf(stderr, "Error: device parameter should either be 'GPU' or 'CPU' (got '%s')\n", optarg); + exit(1); + } + break; + case 'D': + debug = atoi(optarg); + break; + case 'A': + min_perf = strtod(optarg, NULL); + break; + case 'x': +#if defined(HAVE_BLAS) + check = 1; + runs=1; +#else + fprintf(stderr, "Error: requested to run with checks, but CPU BLAS not available\n"); +#endif + break; + case 'h': + case '?': + fprintf(stderr, + "Usage %s [flags] [-- ]\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n" + " Compute pdgemm on a process grid of PxQ, using all available GPUs on each\n" + " node (modulo parsec options), using DTD. Compute C += AxB, where A is MxK\n" + " tiled in mb x kb, B is KxN tiled in kb x nb, and C is MxN tiled in mb x nb\n" + " Executes nruns iterations of the GEMM operation.\n" + " flags:\n" + " --M|-M / --K|-K / --N|-N: set M, K and N (resp.)\n" + " --mb|-m / --kb/-k / --nb|-n: set mb, kb and nb (resp.)\n" + " --nruns|-t: set the number of runs to do\n" + " --device|-d: which device to use (CPU or GPU)\n" + " --verbose|-v: display which GEMM runs on which GPU\n" + " as execution is unfolding\n" + " --help|-h|-?: display this help\n" + " --debug|-D: blocks the process passed as parameter and\n" + " waits for gdb to connect to it\n" + " --Alarm|-A: sets the expected minimum performance for a\n" + " single GPU (kills the process if it takes longer\n" + " than the time corresponding to the expected\n" + " performance to complete the product)\n" + " --Check|-x: perform numerical check of the solution by comparing\n" + " the output of the GPU computation with the output of the\n" + " CPU BLAS computation\n" + "\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n", + argv[0]); + break; + } + } + int pargc = argc - optind + 1; + char **pargv = (char **)malloc((pargc + 1) * sizeof(char *)); + pargv[0] = argv[0]; + for( int i = 0; i < argc - optind; i++ ) + pargv[i + 1] = argv[optind + i]; + pargv[pargc] = NULL; + + if( -1 == P ) + P = (int)sqrt(world); + if( -1 == Q ) + Q = world / P; + while( P * Q != world ) { + P--; + Q = world / P; + } + + if(debug == rank) { + int loop=1; + char hostname[64]; + gethostname(hostname, 64); + fprintf(stderr, "ssh -t %s gdb -p %d\n", hostname, getpid()); + while(loop) { sleep(1); } + } + + // Number of CPU cores involved + int ncores = -1; // Use all available cores + parsec_context = parsec_init(ncores, &pargc, &pargv); + + int *gpu_device_index = NULL; + if( PARSEC_DEV_LEVEL_ZERO == device ) { + nbgpus = get_nb_gpu_devices(); + rc = !(nbgpus >= 1); + if( rc != 0 ) { + fprintf(stderr, "Rank %d doesn't have CUDA accelerators\n", rank); + MPI_Abort(MPI_COMM_WORLD, 0); + return -1; + } + gpu_device_index = get_gpu_device_index(); + } + + // Create datatypes + parsec_arena_datatype_t *adt = parsec_dtd_create_arena_datatype(parsec_context, &TILE_FULL); + parsec_add2arena_rect(adt, parsec_datatype_double_t, mb, nb, mb); + + if(check) { + fprintf(stderr, "Computing target GEMM\n"); + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcC, parsec_context, "C0"); + simple_gemm(parsec_context, dcA, dcB, dcC); + print_matrix(dcC, parsec_context, "C1"); + + device = PARSEC_DEV_CPU; + fprintf(stderr, "Computing CPU BLAS GEMM\n"); + parsec_matrix_block_cyclic_t *dcAcheck = create_initialize_matrix(parsec_context, rank, 1789, "Acheck", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcBcheck = create_initialize_matrix(parsec_context, rank, 1805, "Bcheck", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcCcheck = create_initialize_matrix(parsec_context, rank, 1901, "Ccheck", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcCcheck, parsec_context, "C'0"); + simple_gemm(parsec_context, dcAcheck, dcBcheck, dcCcheck); + print_matrix(dcCcheck, parsec_context, "C'1"); + + for( int i = 0; i < 0*dcC->super.mt; i++ ) { + for( int j = 0; j < dcC->super.nt; j++ ) { + if( (int)dcC->super.super.rank_of(&dcC->super.super, i, j) == parsec_context->my_rank ) { + parsec_data_t *CTile = dcC->super.super.data_of(&dcC->super.super, i, j); + parsec_data_t *DTile = dcCcheck->super.super.data_of(&dcCcheck->super.super, i, j); + double *c = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(CTile, 0)); + double *d = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(DTile, 0)); + for( int x = 0; x < mb*nb; x++) { + if( fabs(c[x] - d[x]) > EPSILON ) { + fprintf(stderr, "Tile (%d, %d), double %d is %g/%g -- differ by %g > %g\n", i, j, x, c[x], d[x], fabs(c[x]-d[x]), EPSILON ); + } + } + } + } + } + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + destroy_matrix(dcAcheck); + destroy_matrix(dcBcheck); + destroy_matrix(dcCcheck); + } else { + // Create and initialize the data + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + + for( int r = 0; r < runs + 1; r++ ) { + double gflop = 2.0 * M * N * K / 1e9; + double maxtime = 0.0; + if(min_perf > 0.0) + maxtime = gflop/world/nbgpus/min_perf; + struct timeval start, end, diff; + if(maxtime > 0.0 && maxtime < 60.0) maxtime=60.0; + if(rank == 0 && maxtime > 0.0) fprintf(stderr, "watchdog: %d seconds\n", (int)maxtime); + if(maxtime > 0.0) alarm((int)maxtime); + gettimeofday(&start, NULL); + simple_gemm(parsec_context, dcA, dcB, dcC); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + double t = (double)diff.tv_sec + (double)diff.tv_usec / 1e6; + double gflops = gflop / t; + (void)t; + (void)gflops; + if( 0 == rank && r > 0 ) { + fprintf(stderr, "DTD_GEMM PxQxg: %d %d %d M: %d N: %d K: %d mb: %d nb: %d kb: %d -- %g s, %g GFLop => %g GFLop/s\n", + P, Q, nbgpus, M, N, K, mb, nb, kb, t, gflop, gflops); + } + } + // deactivate the alarm if it was set + alarm(0); + + parsec_type_free(&adt->opaque_dtt); + PARSEC_OBJ_RELEASE(adt->arena); + parsec_dtd_destroy_arena_datatype(parsec_context, TILE_FULL); + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + } + + parsec_fini(&parsec_context); + +#if defined(PARSEC_HAVE_MPI) + MPI_Finalize(); +#endif + + return ret; +} diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C new file mode 100644 index 000000000..03d162262 --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C @@ -0,0 +1,77 @@ +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/mca/device/level_zero/device_level_zero.h" +#include "parsec/interfaces/dtd/insert_function.h" +#include "parsec/runtime.h" +#include "parsec/execution_stream.h" + +#include +#include + +extern "C" { + extern int gemm_lz_verbose; + int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + parsec_task_t *this_task = gpu_task->ec; + struct timeval start, end, diff; + double delta; + double *a_gpu, *b_gpu, *c_gpu; + parsec_level_zero_exec_stream_t *lz_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + cl::sycl::event gemm_event; + + (void)gpu_device; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + a_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 0)); + b_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 1)); + c_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 2)); + + gettimeofday(&start, NULL); + + double alpha=0.0; + double beta=1.0; + try { + gemm_event = oneapi::mkl::blas::gemm(lz_stream->swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, kb, nb, + alpha, static_cast(a_gpu), mb, + static_cast(b_gpu), kb, + beta, static_cast(c_gpu), nb); + gemm_event.wait(); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if(gemm_lz_verbose) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d [%p], %dx%d [%p], %dx%d [%p] on node %d, Stream %s of GPU %s submitted in %g s\n", + m, n, k, mb, kb, a_gpu, kb, nb, b_gpu, mb, kb, c_gpu, + this_task->taskpool->context->my_rank, + gpu_device->super.name, gpu_stream->name, delta); + + return PARSEC_HOOK_RETURN_DONE; +} diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index d4d73b014..69a493a9c 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -1,4 +1,22 @@ -if(PARSEC_HAVE_CUDA) +if(PARSEC_HAVE_LEVEL_ZERO) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + include(ParsecCompilePTG) + + parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) + target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") + target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) + else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink_lz test with DPCPP is disabled") + endif(TARGET MKL::MKL_DPCPP) +elseif(PARSEC_HAVE_CUDA) if( NOT TARGET CUDA::cublas ) message(WARNING "CUBLAS library not automatically detected when CUDA was found -- not enabling nvlink test") else( NOT TARGET CUDA::cublas ) @@ -27,4 +45,5 @@ if(PARSEC_HAVE_CUDA) target_ptg_sources(testing_get_best_device PRIVATE "get_best_device_check.jdf") parsec_addtest_cmd(get_best_device_shm ${SHM_TEST_CMD_LIST} ./testing_get_best_device -N 400 -t 20 -g 1) -endif(PARSEC_HAVE_CUDA) +endif(PARSEC_HAVE_LEVEL_ZERO) + diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index 2402b5cfe..5b473e20b 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -38,6 +38,13 @@ typedef cublasStatus_t (*cublas_dgemm_v2_t) ( cublasHandle_t handle, %} +extern "DPCPP" %{ + +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include + +%} + %option no_taskpool_instance = true /* can be anything */ /* @@ -150,6 +157,35 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } +} +END + BODY { fprintf(stderr, "Kernel GEMM(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", @@ -195,6 +231,36 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception that is also badly formatted..."); + } +} +END + + BODY { fprintf(stderr, "Kernel GEMM2(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 06a34079a..5b10b71e9 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -6,7 +6,6 @@ */ #include "parsec.h" -#include "parsec/mca/device/cuda/device_cuda_internal.h" #include "parsec/data_distribution.h" #include "parsec/data_dist/matrix/matrix.h" #include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" @@ -14,12 +13,20 @@ #include "parsec/class/info.h" #if defined(PARSEC_HAVE_CUDA) +#include "parsec/mca/device/cuda/device_cuda_internal.h" #include +#define TARGET_DEVICE_TYPE PARSEC_DEV_CUDA +#elif defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#define TARGET_DEVICE_TYPE PARSEC_DEV_LEVEL_ZERO +#else +#define TARGET_DEVICE_TYPE PARSEC_DEV_CPU #endif #include "nvlink.h" #if defined(PARSEC_HAVE_CUDA) +/* Only CUDA/CUBLAS requires to define handles on each GPU device */ static void destruct_cublas_handle(void *p) { cublasHandle_t handle = (cublasHandle_t)p; @@ -45,17 +52,15 @@ static void *create_cublas_handle(void *obj, void *p) (void)status; return (void*)handle; } -#endif static void destroy_cublas_handle(void *_h, void *_n) { -#if defined(PARSEC_HAVE_CUDA) cublasHandle_t cublas_handle = (cublasHandle_t)_h; cublasDestroy_v2(cublas_handle); -#endif (void)_n; (void)_h; } +#endif static void __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) @@ -65,22 +70,33 @@ __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) parsec_matrix_block_cyclic_t *dcA; parsec_del2arena( & nvlink_taskpool->arenas_datatypes[PARSEC_nvlink_DEFAULT_ADT_IDX] ); parsec_data_free(nvlink_taskpool->_g_descA->mat); +#if defined(PARSEC_HAVE_CUDA) parsec_info_unregister(&parsec_per_stream_infos, nvlink_taskpool->_g_CuHI, NULL); +#endif dcA = nvlink_taskpool->_g_descA; parsec_tiled_matrix_destroy( (parsec_tiled_matrix_t*)nvlink_taskpool->_g_descA ); userM = nvlink_taskpool->_g_userM; for(g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, userM->super.super.myrank); - parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, cuda_device->super.super.device_index); + parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, device->device_index); +#if defined(PARSEC_HAVE_CUDA) + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {} ); status = (cudaError_t)cudaFree( gpu_copy->device_private ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaFree ", status, {} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_result_t status = zeMemFree(level_zero_device->driver->ze_context, gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, {} ); +#else + free(gpu_copy->device_private); +#endif gpu_copy->device_private = NULL; - parsec_data_copy_detach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_detach(dta, gpu_copy, device->device_index); PARSEC_OBJ_RELEASE(gpu_copy); g++; } @@ -101,18 +117,18 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb parsec_matrix_block_cyclic_t *dcA; parsec_matrix_block_cyclic_t *userM; - /** Find all CUDA devices */ + /** Find all GPU devices */ nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { nb++; } } if(nb == 0) { char hostname[256]; gethostname(hostname, 256); - fprintf(stderr, "This test requires at least one CUDA device per node -- no CUDA device found on rank %d on %s\n", + fprintf(stderr, "This test requires at least one GPU device per node -- no GPU device found on rank %d on %s\n", ctx->my_rank, hostname); return NULL; } @@ -120,7 +136,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { dev_index[nb++] = device->device_index; } } @@ -179,31 +195,81 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb /* Now, we create a GPU version of each tile. As these tiles will be accessed RW * in the JDF, this also pins the task on the GPU that we chose to host the tile */ for(int g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { /* We get the data from the data collection */ parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, ctx->my_rank); /* The corresponding data copy on CPU RAM */ parsec_data_copy_t *cpu_copy = parsec_data_get_copy(dta, 0); /* And we create a new data copy on GPU */ parsec_data_copy_t *gpu_copy = PARSEC_OBJ_NEW(parsec_data_copy_t); +#if defined(PARSEC_HAVE_CUDA) /* We chose the GPU */ + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {return NULL;} ); /* Allocate memory on it, for one tile */ status = (cudaError_t)cudaMalloc( &gpu_copy->device_private, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMalloc ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_device_memory_properties_t devMemProperties; + uint32_t count = 1; + /* Safety: we check that there is one memory segment available on this device (ordinal 0), and + * that this segment is big enough to store what we need to allocate */ + ze_result_t status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { return NULL; }); + assert(count >= 1); + assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= (int)devMemProperties.totalSize); + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = 0 + }; + /* Allocate memory on it, for one tile */ + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + level_zero_device->ze_device, &gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); +#else + gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); +#endif /* Attach this copy to the data, on the corresponding device */ - parsec_data_copy_attach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_attach(dta, gpu_copy, device->device_index); /* We also need to tell PaRSEC that the owner of this data is the GPU, or the * GPU might not be selected to work on that data */ - parsec_data_transfer_ownership_to_copy(dta, cuda_device->super.super.device_index, PARSEC_FLOW_ACCESS_RW); + parsec_data_transfer_ownership_to_copy(dta, device->device_index, PARSEC_FLOW_ACCESS_RW); +#if defined(PARSEC_HAVE_CUDA) /* And copy the tile from CPU to GPU */ status = (cudaError_t)cudaMemcpy( gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, cudaMemcpyHostToDevice ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; + ze_fence_handle_t copySignalFence = level_zero_stream->fences[0]; + status = zeFenceReset(copySignalFence); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", status, { return NULL; } ); + status = zeCommandListReset(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); + status = zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[0], gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, NULL, 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + status = zeCommandListClose(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); + status = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[0], copySignalFence); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); + while(1) { + status = zeFenceQueryStatus(copySignalFence); + if(status == ZE_RESULT_SUCCESS) + break; + if(status != ZE_RESULT_NOT_READY) + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceQueryStatus ", status, { break; } ); + usleep(1000); + } +#else + memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); +#endif g++; } } diff --git a/tests/subsystem/CMakeLists.txt b/tests/subsystem/CMakeLists.txt new file mode 100644 index 000000000..38a7fed60 --- /dev/null +++ b/tests/subsystem/CMakeLists.txt @@ -0,0 +1,5 @@ +if(PARSEC_HAVE_LEVEL_ZERO) + add_subdirectory(level_zero) +else(PARSEC_HAVE_LEVEL_ZERO) + message(STATUS "PaRSEC doesn't have Level Zero support") +endif(PARSEC_HAVE_LEVEL_ZERO) diff --git a/tests/subsystem/level_zero/CMakeLists.txt b/tests/subsystem/level_zero/CMakeLists.txt new file mode 100644 index 000000000..de11e7887 --- /dev/null +++ b/tests/subsystem/level_zero/CMakeLists.txt @@ -0,0 +1,14 @@ +if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) +endif(NOT TARGET MKL::MKL_DPCPP) +if(TARGET MKL::MKL_DPCPP) +message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C lz SOURCES main.c kernel.dpcpp.C interface.dpcpp.C) + target_include_directories(lz PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_link_libraries(lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) +else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, Level Zero test with DPCPP is disabled") +endif(TARGET MKL::MKL_DPCPP) diff --git a/tests/subsystem/level_zero/interface.dpcpp.C b/tests/subsystem/level_zero/interface.dpcpp.C new file mode 100644 index 000000000..8abe7c5e8 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.C @@ -0,0 +1,56 @@ +#include "level_zero/ze_api.h" +#include "interface.dpcpp.h" + +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + sycl_wrapper_driver_t *res = new sycl_wrapper_driver_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device) +{ + std::vectordevices; + + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + swp->context = sycl::make_context(hContextInteropInput); +} + +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + sycl_wrapper_device_t *res = new sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); + + return res; +} + +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) +{ + sycl_wrapper_queue_t *swq = new sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} + +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp) +{ + delete swp; + return 0; +} + +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd) +{ + delete swd; + return 0; +} + +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq) +{ + delete swq; + return 0; +} + diff --git a/tests/subsystem/level_zero/interface.dpcpp.h b/tests/subsystem/level_zero/interface.dpcpp.h new file mode 100644 index 000000000..04aa2f380 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.h @@ -0,0 +1,40 @@ +#ifndef INTERFACE_DPCPP_H +#define INTERFACE_DPCPP_H + +typedef struct sycl_wrapper_driver_s sycl_wrapper_driver_t; +typedef struct sycl_wrapper_device_s sycl_wrapper_device_t; +typedef struct sycl_wrapper_queue_s sycl_wrapper_queue_t; + +#if defined(c_plusplus) || defined(__cplusplus) +#include "sycl/ext/oneapi/backend/level_zero.hpp" + +struct sycl_wrapper_driver_s { + sycl::platform platform; + sycl::context context; +}; + +struct sycl_wrapper_device_s { + sycl::device device; +}; + +struct sycl_wrapper_queue_s { + sycl::queue queue; +}; + +extern "C" { +#endif + +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device); +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device); +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp); +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd); +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //INTERFACE_DPCPP_H diff --git a/tests/subsystem/level_zero/kernel.dpcpp.C b/tests/subsystem/level_zero/kernel.dpcpp.C new file mode 100644 index 000000000..0ab12e76a --- /dev/null +++ b/tests/subsystem/level_zero/kernel.dpcpp.C @@ -0,0 +1,64 @@ +#include +#include "sycl/ext/oneapi/backend/level_zero.hpp" +#include "interface.dpcpp.h" + +extern "C" { + int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); +} + +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb) +{ + double alpha=1.0; + double beta=1.0; +/* sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bA = sycl::make_buffer(hBufferInteropInputA, swp->context); + auto bbA = bA.reinterpret(sycl::range<1>(mb*mb)); + sycl::backend_input_t> hBufferInteropInputC = { (void*)C, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bC = sycl::make_buffer(hBufferInteropInputC, swp->context); + auto bbC = bC.reinterpret(sycl::range<1>(mb*mb));*/ + try { + oneapi::mkl::blas::gemm(swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, mb, mb, + alpha, A, mb, + A, mb, + beta, C, mb); + } catch (const oneapi::mkl::invalid_argument &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception\n"); + } catch (const oneapi::mkl::unsupported_device &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unsuported device exception\n"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws host bad allocation exception\n"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws device bad allocation exception\n"); + } catch (const oneapi::mkl::unimplemented &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unimplemented exception\n"); + } catch (const std::exception& e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception '%s'\n", e.what()); + } catch (...) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + fprintf(stderr, "kernel has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n"); + + return 0; +} +/* +void *sycl_malloc(sycl_wrapper_t *sw, size_t size) +{ + return sycl::malloc_device(size, sw->queue); +} + +void sycl_free(sycl_wrapper_t *sw, void *ptr) +{ + sycl::free(ptr, sw->queue); +} +*/ diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c new file mode 100644 index 000000000..d6cce9635 --- /dev/null +++ b/tests/subsystem/level_zero/main.c @@ -0,0 +1,581 @@ +#include +#include +#include +#include +#include + +#include +#include "interface.dpcpp.h" + +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); + +struct driver_s; +struct device_s; +struct stream_s; + +#define NB_STREAMS 4 +#define MAX_FENCES 2 + +typedef struct stream_s { + int immediate; + sycl_wrapper_queue_t *swq; + ze_command_queue_handle_t cq; + ze_command_list_handle_t cl; + struct device_s *device; + ze_fence_handle_t fences[MAX_FENCES]; +} stream_t; + +typedef struct device_s { + ze_device_handle_t device; + struct driver_s *driver; + sycl_wrapper_device_t *swd; + stream_t streams[NB_STREAMS]; +} device_t; + +typedef struct driver_s { + ze_driver_handle_t driver; + ze_context_handle_t context; + sycl_wrapper_driver_t *swp; + int nb_devices; + device_t *devices; +} driver_t; + +#define LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + fprintf(stderr, "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +static int init_device(device_t *device, ze_device_handle_t gpuDevice) +{ + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + ze_result_t ze_rc; + + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (count) ", ze_rc, { return -1; } ); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, cmdqueueGroupProperties); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (populate) ", ze_rc, { return -1; } ); + + // Find a command queue type that support compute + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); ++i ) { + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as compute queue\n", i); + if(cmdqueueGroupCount == computeQueueGroupOrdinal) + computeQueueGroupOrdinal = i; + } + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as copy queue\n", i); + if(cmdqueueGroupCount == copyQueueGroupOrdinal) + copyQueueGroupOrdinal = i; + } + } + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + fprintf(stderr, "INFO: computeQueueGroupOrdinal = %d, copyQueueGroupOrdinal = %d\n", computeQueueGroupOrdinal, copyQueueGroupOrdinal); + + for(int j = 0; j < NB_STREAMS; j++ ) { + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + device->streams[j].device = device; + if( 0 && j < 2 ) { + device->streams[j].immediate = 1; + commandQueueDesc.ordinal = copyQueueGroupOrdinal; + ze_rc = zeCommandListCreateImmediate(device->driver->context, gpuDevice, + &commandQueueDesc, + &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, { return -1;} ); + } else { + device->streams[j].immediate = 0; + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(device->driver->context, gpuDevice, + &commandQueueDesc, &device->streams[j].cq); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, { return -1;} ); + ze_command_list_desc_t commandListDesc = { + 0 // flags + }; + ze_rc = zeCommandListCreate(device->driver->context, gpuDevice, + &commandListDesc, &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, { return -1;} ); + } + + for(int k = 0; k < MAX_FENCES; k++ ) { + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 + }; + device->streams[j].fences[k] = NULL; + ze_rc = zeFenceCreate(device->streams[j].cq, &fence_desc, &(device->streams[j].fences[k])); + LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, { continue; } ); + } + } + device->device = gpuDevice; + return 0; +} + +static int init_driver(driver_t *driver, int maxDevices) +{ + uint32_t deviceCount = 0; + ze_device_handle_t *allDevices; + ze_device_handle_t *gpuDevices; + ze_result_t ze_rc; + + ze_rc = zeDeviceGet(driver->driver, &deviceCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (count) ", ze_rc, { return -1; } ); + if(deviceCount == 0) + return 0; + + allDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + gpuDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + ze_rc = zeDeviceGet(driver->driver, &deviceCount, allDevices); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (populate) ", ze_rc, { return -1; } ); + + int deviceId = 0; + for(int did = 0; did < (int)deviceCount; did++) { + ze_device_properties_t device_properties; + zeDeviceGetProperties(allDevices[did], &device_properties); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + gpuDevices[deviceId++] = allDevices[did]; + if( deviceId > maxDevices ) { + break; + } + } + free(allDevices); + allDevices = NULL; + + if( deviceId == 0) { + free(gpuDevices); + return 0; + } + deviceCount = deviceId; + + driver->devices = (device_t*)malloc(deviceCount * sizeof(device_t)); + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_rc = zeContextCreate(driver->driver, &ctxtDesc, &driver->context); + LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { continue; } ); + + int dpos = 0; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[dpos].driver = driver; + if( init_device(&driver->devices[dpos], gpuDevices[did]) < 0 ) { + continue; + } + dpos++; + } + deviceCount = dpos; + + driver->nb_devices = deviceCount; + if(deviceCount == 0) { + free(driver->devices); + driver->devices = NULL; + return 0; + } + free(gpuDevices); + + driver->swp = sycl_wrapper_platform_create(driver->driver); + sycl_wrapper_device_t *swd[deviceCount]; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[did].swd = sycl_wrapper_device_create(driver->devices[did].device); + swd[did] = driver->devices[did].swd; + } + sycl_wrapper_platform_add_context(driver->swp, driver->context, swd, deviceCount); + for(int did = 0; did < (int)deviceCount; did++) { + for(int sid = 0; sid < NB_STREAMS; sid++) { + driver->devices[did].streams[sid].swq = sycl_wrapper_queue_create(driver->swp, driver->devices[did].swd, driver->devices[did].streams[sid].cq); + } + } + + return deviceCount; +} + +static void *allocate_workspace(device_t *device, size_t size) +{ + ze_result_t status; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + void *device_ptr; + uint32_t count = 0; + int memIndex = -1; + + status = zeDeviceGetMemoryAccessProperties(device->device, &memAccessProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { return NULL; }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + fprintf(stderr, "Device does not have memory allocation capabilities with RW access\n"); + return NULL; + } + status = zeDeviceGetProperties(device->device, &devProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { return NULL; }); + status = zeDeviceGetMemoryProperties(device->device, &count, NULL); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (count) ", status, { return NULL; }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(device->device, &count, devMemProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (populate) ", status, { free(devMemProperties); return NULL; }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + + if( size > devMemProperties[memIndex].totalSize ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + fprintf(stderr, "Requested %zd bytes on LEVEL_ZERO device, but only %zd bytes are available -- Returning NULL\n", + size, devMemProperties[memIndex].totalSize); + return NULL; + } + free(devMemProperties); devMemProperties = NULL; + + /*device_ptr = sycl_malloc(device->streams[2].sw, size); + assert(NULL != device_ptr); */ + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + + status = zeMemAllocDevice(device->driver->context, &memAllocDesc, size, 128, + device->device, &device_ptr); + LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); + return device_ptr; +} + +typedef struct { + struct timespec start; + struct timespec enter_wait; + struct timespec end; +} run_timing_t; + +typedef struct { + struct timespec start; + struct timespec start_alloc; + struct timespec end_alloc; + struct timespec start_cpu2gpu; + struct timespec enter_wait_cpu2gpu; + struct timespec end_cpu2gpu; + struct timespec start_gpu2cpu; + struct timespec enter_wait_gpu2cpu; + struct timespec end_gpu2cpu; + run_timing_t *runs; +} device_timing_t; + +double diff_timespec(const struct timespec *time1, const struct timespec *time0) { + return (time1->tv_sec - time0->tv_sec) + + (time1->tv_nsec - time0->tv_nsec) / 1000000000.0; +} + +static void usage(const char *name) { + fprintf(stderr, "USAGE: %s [] [Problem size] [Number of iterations]\n", name); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + ze_result_t ze_rc; + uint32_t driverCount = 0; + driver_t *drivers; + ze_driver_handle_t *allDrivers; + int max_devices = 1024*1024, nb_devices = 0; + void **device_workspaceA, **device_workspaceC; + int errors = 0; + int N = 4096; + int NRUNS = 3; + device_timing_t *timings; + + if(argc > 1) { + max_devices = atoi(argv[1]); + if(max_devices <= 0) { + usage(argv[0]); + } + + if(argc > 2) { + N = atoi(argv[2]); + if(N <= 0) { + usage(argv[0]); + } + + if(argc >3) { + NRUNS = atoi(argv[3]); + if(NRUNS <= 0) { + usage(argv[0]); + } + } + } + } + + ze_rc = zeInit( 0 ); + LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, { return -1; }); + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (count) ", ze_rc, { return 1; } ); + fprintf(stderr, "STATUS: found %d drivers\n", driverCount); + drivers = malloc(driverCount * sizeof(driver_t)); + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (populate) ", ze_rc, { return 1; } ); + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + int nb; + drivers[driverId].driver = allDrivers[driverId]; + if( (nb = init_driver(&drivers[driverId], max_devices)) <= 0 ) { + fprintf(stderr, "%d device found in driver %d... Bailing out\n", nb_devices, driverId); + return 1; + } else { + fprintf(stderr, "STATUS: there are %d GPU devices in driver %d\n", nb, driverId); + nb_devices += nb; + nb_devices = nb_devices <= max_devices ? nb_devices : max_devices; + } + } + free(allDrivers); + fprintf(stderr, "%d devices found and initialized\n", nb_devices); + + timings = (device_timing_t*)calloc(sizeof(device_timing_t), nb_devices); + for(int i = 0; i < nb_devices; i++) { + clock_gettime(CLOCK_REALTIME, &timings[i].start); + timings[i].runs = (run_timing_t*)calloc(sizeof(run_timing_t), NRUNS); + } + + //Allocate GPU memory for each device + device_workspaceA = (void**)malloc(sizeof(void*)*nb_devices); + device_workspaceC = (void**)malloc(sizeof(void*)*nb_devices); + int did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + clock_gettime(CLOCK_REALTIME, &timings[did].start_alloc); + device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N*2); + device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + clock_gettime(CLOCK_REALTIME, &timings[did].end_alloc); + did++; + } + } + + //Initialize the GPU memory for each device + double *cpuA = (double*)calloc(sizeof(double), N*N); + for(int i = 0; i < N; i++) { + cpuA[i*N+i] = 1.0; + } + double *cpuC = (double*)calloc(sizeof(double), N*N); + did=0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Initialize A and C on device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)cpuA; + void *copy_out = (void*)device_workspaceA[did]; + size_t count = N*N*sizeof(double); + clock_gettime(CLOCK_REALTIME, &timings[did].start_cpu2gpu); + ze_rc = zeCommandListReset(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(A->GPU) ", ze_rc, { continue; } ); + copy_in = (void*)cpuC; + copy_out = (void*)device_workspaceC[did]; + count = N*N*sizeof(double); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(C->GPU) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[0].cq, 1, &device->streams[0].cl, device->streams[0].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_cpu2gpu); + do { + ze_rc = zeFenceQueryStatus(device->streams[0].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: copies ended on device %d of driver %d\n", deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].end_cpu2gpu); + fprintf(stderr, "STATUS: CPU->GPU copies on device %d of driver %d completed!\n", deviceId, driverId); + } + did++; + } + } + + for(int run = 0; run < NRUNS; run++) { + //Do a GEMM (blocking) on each device, and wait for its completion -- yes, memory is not initialized. + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Ready to submit GEMM[%d] on device %d of driver %d\n", run, deviceId, driverId); + fprintf(stderr, "STATUS: Context of driver %d is %s\n", driverId, zeContextGetStatus(drivers[driverId].context) == ZE_RESULT_SUCCESS ? "Fine" : "Broken"); + dpcpp_kernel_GEMM(device->driver->swp, device->swd, device->streams[2].swq, (double*)device_workspaceA[did], (double*)device_workspaceC[did], N); + fprintf(stderr, "STATUS: GEMM[%d] submitted on device %d of driver %d\n", run, deviceId, driverId); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].start); + ze_rc = zeCommandListReset(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = zeCommandListClose(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + + ze_rc = zeFenceReset(device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[2].cq, 1, &device->streams[2].cl, device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].enter_wait); + do { + ze_rc = zeFenceQueryStatus(device->streams[2].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: GEMM[%d] ended on device %d of driver %d\n", run, deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].end); + fprintf(stderr, "STATUS: GEMM[%d] on device %d of driver %d completed!\n", run, deviceId, driverId); + } + did++; + } + } + } + + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Get C back from device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)device_workspaceC[did]; + void *copy_out = (void*)cpuC; + size_t count = N*N*sizeof(double); + + for(int i = 0; i < N*N; i++) cpuC[i] = -1.0; + + clock_gettime(CLOCK_REALTIME, &timings[did].start_gpu2cpu); + ze_rc = zeCommandListReset(device->streams[1].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[1].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(GPU->A) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[1].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeFenceReset(device->streams[1].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[1].cq, 1, &device->streams[1].cl, device->streams[1].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_gpu2cpu); + do { + ze_rc = zeFenceQueryStatus(device->streams[1].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: copies ended from device %d of driver %d\n", deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].end_gpu2cpu); + fprintf(stderr, "STATUS: GPU->CPU copies on device %d of driver %d completed!\n", deviceId, driverId); + + int error0 = 0; + int error1 = 0; + for(int i = 0; i < N; i++) { + for(int j = 0; j < N; j++) { + if(i != j) { + if(cpuC[i*N+j] != 0.0) { + if(0 == error0) + fprintf(stderr, "ERROR during checks of driver %d: C[%d,%d] = %g expected 0.0\n", did, i, j, cpuC[i*N+j]); + error0++; + errors++; + } + } else { + if(cpuC[i*N+j] != (double)NRUNS) { + if(0 == error1) + fprintf(stderr, "ERROR during checks of driver %d: C[%d, %d] = %g, expected %g\n", did, i, j, cpuC[i*N+j], (double)NRUNS); + error1++; + errors++; + } + } + } + } + if(error0 > 0) fprintf(stderr, " %d errors of type 0 during checks of driver %d\n", error0, did); + if(error1 > 0) fprintf(stderr, " %d errors of type 1 during checks of driver %d\n", error1, did); + } + did++; + } + } + + for(did = 0; did < nb_devices; did++) { + printf("TIME: Device %d\n", did); + printf("TIME: START: %ld.%09ld s\n", timings[did].start.tv_sec, timings[did].start.tv_nsec); + printf("TIME: START_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].start_cpu2gpu, &timings[did].start)); + printf("TIME: START_ENTER_WAIT_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_cpu2gpu, &timings[did].start_cpu2gpu)); + printf("TIME: END_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].end_cpu2gpu, &timings[did].enter_wait_cpu2gpu)); + for(int run = 0; run < NRUNS; run++) { + printf("TIME: START_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].start, + run == 0 ? &timings[did].end_cpu2gpu : &timings[did].runs[run-1].end)); + printf("TIME: ENTER_WAIT_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].enter_wait, &timings[did].runs[run].start)); + printf("TIME: END_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].enter_wait)); + double tot_kernel = diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].start); + double gflops = 2.0*N*N*N/1e9; + printf("PERF: device %d run %d %11.9g s for %d x %d x %d DGEMM (%g . 10^9 flops): %g GFLOP/s\n", did, run, tot_kernel, N, N, N, gflops, gflops/tot_kernel); + } + printf("TIME: START_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].start_gpu2cpu, &timings[did].runs[NRUNS-1].end)); + printf("TIME: ENTER_WAIT_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_gpu2cpu, &timings[did].start_gpu2cpu)); + printf("TIME: END_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].end_gpu2cpu, &timings[did].enter_wait_gpu2cpu)); + printf("\n"); + } + + if(errors == 0) + return EXIT_SUCCESS; + return EXIT_FAILURE; +} From 60ed5a1b2dca809e9743569007f9338c111fca56 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Wed, 8 Feb 2023 17:14:50 -0500 Subject: [PATCH 3/3] Updated version of parsec.l and parsec.y --- contrib/pregen_flex_bison.tar | Bin 393728 -> 382464 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/contrib/pregen_flex_bison.tar b/contrib/pregen_flex_bison.tar index 0c7a4baa5fbfccb6142db6624d4abd30ae45c66f..a0322692f27a5889c2933e7de789371eaa5ec9b7 100644 GIT binary patch delta 18855 zcmd6Pd3aPsw)m^6&O#t0Aqga5$xXtNge3IsES<22q=85_vLIoxX+i^$q!YS3ViXME z?uddBjyj9M8AnG)1X1sE#sy}<4Fwz(9hF5L7gQD*mXg$=d74(@qh zMZum?B*Ex;fXoVu7kH83>fhS6S^*q*h-83e7a0%MT_iKXdU>I)TeZnkG^ew^vAVXd z##shwk06TPT_lFU&PT|Yn8tcXrXx$6kgd7fI{aNQ*dQrTv74k??AQ{v>?RtVeuPxQ z=10j)_;EM69u_@D>LK&LNgS+u9Fe{K7}>%|A$Xo3lSu(Q_yjpk?N;NLe;{Eb-qYE( z)a}(Wm!qJ%y13nC!SO$kczEKEWMDV;+SJroUGA9KP%}r%&DH9=+LpGrYN4(C?W^3L zEYu&DUt8uzm2&$umj?y9%G18a!?jL8oUc^@yqb+`7WULoS6W?PS=!*p#O@r`x$yTt zl3#3llvrTHlO#QL8dB)@x;(yCm*2gxscHUVjf=#@V?;hhwhrhn&cP1zi{!&zB)*E} zqr@t{N=o>nz?v@}Hu)$h7WRSVc3EJTP$jy6DiIh8t>UXt7l;aNHeOd~vx`T*sOv5) zG%*zx$@(IJsj#GggB4ljLl=#T@&%@%{6bMzAQ}`E$htzSBB{tOtBXYsMa8mOQdT6< z+I2BtyDrCXm!q;vn(T<4i$T-u1pCoN(A~6!EfDatCuSsL2zdG3lA=}Kp-ORF3U>zk_J)$#N< zWuwtErRQkr!u9c&(Za8x9~H&(O6z!GX?=agtm-O9bp!u$%x`eiRXfWaWtHhjH@uWX zt?*+G9ZK;dF_#t$#1`l+FXpvILoWS0h3MyrMs&z~o{XZJ29?jF@o#>fOzqAUJV)-} zpinnC&vW%c{uTP*l>#JJs5`2?Y@n3)JYdOd)Lw}lRlFa398q_0Lm>lIph*<{e`!+x zwL(FXP3ohj9(L0VG|?>*At{zRrzq3&xlZKj-N8deeHT_?!r+(G-k<{H!~f{(Um)t{ zTns^Ij@aZ~5g+3PTr%6(gv|t3b9^OY82QD5J?fzs3bzvU;3RP?kQMoY6Iy=0;8wn5 zX09&2n-vZ^*`$z&!ay4n`nyl08g<|{xn0$%JM4|Yi*|`>(g_>Xp z_9Zk{HPwp6;50b~4IQ^2$!udHDh|>^O)EyUIDSof#6Vl&z))+rWNZbdCk&Ahwvjp| z97JC)qENSrrlAecote6%3!6}ccdOw@geE3Uc$~hkAag{QXIGdM3i&XB&2FunA*#@< z(}cS-zjQQ+C_erRz~sD#zV$*B=wvxxB-JLSd|h$LEYOf`2*i8f4>5NI>WX=x`Q<`) zEn%eP;q;*_1#$L)nV2L(Y#m%D95SB@`U*LE?w%1BY&b6)@%iSqRU5OQK=Z z0TM~_;no9Wc)XH#z-wr>9C+#g$;1%o=m9b`PARvH2{g@`1K|fr@c>n<+hO%Vk_Iae zlGLre)DPbuB%{nW2+WyNsjWvd zBT1tagqS9gOd$e3LQZs%4*FLZ*xiPJ=3w(n_+V4urN5D}v8r)_kRQ$mvTEe3B!kZ| z(_bYQb%%(_CUqjjN#rW&%kTqNg?%DK<5N?c<#WHf9(Od{ct z*GK}+b%L{~V4T64v?LiDDu08{Sk%cdkJF8Cy5f#aMJe2(1Qare<58NaT&oPCB(Hdw zBrpiu=*Ca!nhF9|CgU{SDy`2Zt7U+y%PC$LffjZtsMe(#>B{B;v8yJ7WKCi~{}s%S zhSC|8j)hH7W}%62_;oTCzJHxKVCrGg)-APCAup0&njD^VHM-K@e91LAJ>g6(hK|9Z zQbSEDuM7-y8HK9&K$k%$H$%=YiCqQ&+HmFlbri6~q)Xq$^}qHANoMH1;Ph8yc$jdz z+$7d2QC&|SEDOgQPM})B`Bw2{FmPxQQ2SuqOUl@m$o1ZUj@7P8JBIIfX7d8S!#C4jEQ>Q{C;#^cg$wo0RqQNN| zC7(=uoKBo3Lfr_iZxkl0au>3#j3rD~%K$_xQRGSxxn9sDUv!+O6hq{Mu4ZyRU-Geu zm>c_-jKSrvD3$uJ<|18T3a}RAegUP%u9}82OBxsk^8!-2Cb85&F`6XlQs?yi5*en+ z<&V;wP;Tjkig&7O@uj1GQCw2Nmm;-Db}Iu@T?Waza9ucNDO4GM>2lemrRAHUC<>92 zY{s_IXX8Q#?mmXT#&eWJh*{?N`y_dQYFuPHPn#eb!SWTw()sHuqQ-tWb{U_UbF&AV>me^`H(XzMY8uEM27w57)j+^8^T95 zcPC`wEy9fdL^u^uHcaPKh^`;%knlN;C1=~PAu@c|k!MhRdKXz)NTh6_N2BOEv= zn5}&+;DfU`tAt7*A%Iy(PTC?Mx050K2i#Z-FgiK_$R{o6eV&Yjx8Ec2k#fe-3bvgg zL&MQ3^uj_&`xb*Q?R^ryDBy3*m9f%FxpL6dB&uxCjID%wFe5FsXhA~=b%E&q<37SL z>lg21r%wCXvdGy+5Vy!q0xpshz+IVgcxutDyj5`lq&Pa(0s*mD3HE{DUM4k;g!2`$)zSueXZV;BXksFX9c^0g`iQoK>a z#cAI}B%7)*QAQei0c<%#hQZVm=#j>sM1S()84}I7(*E!v3i0E2NJ6oSB~=8W$vNH( z3zY{|@ucaP#podBBpCsF&XA-;zQ4w0EfpL$3^-A+{shTOm-4{9F(FZN!zfrDFz914 z&0IRDcvvsyS~eQGKSsiz;neT^7!@4<h9RXn{(b$oeWbl4UK85g4 zajf=JBnh-naPRl;pP~c3{|xSSjra`9_IyOL;oLjeEc#RYLel%|v$xPhpE& zPM}W+={Y>Kkh67uGl6hIejtW7T55 z!QQaat!1`g^2Y0N1@_t7m-FOImgZ`1##{y)&Nq0xWo>vff1ffx|E{@&<`*4H-htw2UZrJXUYi3#TsBP@n4<;KoN#j)wMPWi4uv<%=v9fl z!yIj%tF;p!$W_5^_*pqkF^pvD4$D@Fl+a`^@>dDAtIOx_z+67h><(~H2UIz*>2e26 zBtuLYs3(FLH_fDfB?J1XS}>c|^nDtS&Zc+Js{WF+NI9MY;B3RUEM(-mfQ*%pxkpYr zKh2Xp0a8-w*hu7=-(B6=2D4LX!MIsoS6iFQtL!|F=gPUnc>0rGHnSj1*JsjK$;_Y5U!iShYo(en3(VQO@eDwR*X_eY144irIx&%fh2xzpprjw)u@)7o67g8(yXW}kwUNA-|6)r$k>K+jxC^XVf`N$(wpRCG<0mIL&-$gyM*RK_%ur#_4wh;CDb+u zC6qf=os@wMn`jZ#-bGV}aNgvF98!VO=`1l!L)-DgWG5|y&3~c8(&D8!<2~l0SIhUO@$uDC`Ww`vyPD~Z+!XPd(PK@tK<{s85=`Gu z6X4hhs>AwboThxR*G+HX$i~CY<@6jD9NI+FVET2`0{gF{V?7b5(r_7jZ`58cVQwq(Ld7rh99AMw&h@qyS=xwa(Ywx&|2r<&;~ zXulLU6JmGKXd~56zbCMF4b6blscaxTeHBjpcXZQyp%D_rNU#oMh$0`U3I$xThUOVq z7S0)x`Z3OegRjI{@YLnhsRqTB1=e-Z6$r4a2h$g*FVV7>VOQX!{u7Ng=)Dpnj@-5M zQi&ZE|9v|6Ut`PTl0<~*To5+Fw`if#9p3g1w--;x^xjSz<(TkPj2iAGvOo^fqT44% z!H(^8INZ`rFUG2WbklLTE4zIW4(^x-gXhKT>1Kt4p`Yy8{%U+xwl=F>YsYgx~K4<4A~=eM{?^IDgfrkmT_Ixv9oYHM6x4;rs8k7MfD`UxhKcG0MT z*&6T9iI%NxTSpQtOWT)qL8~9vDjz&S;{g?8P^1X~d-tN>5I)`P+r7-WvY+(! zmSOr&0Ci#4$Oe8yufL|rW>W++UvsB=@X#P1J}hz5p>f)BuiJfo>qK4`54WtR;Sj%p zj+%l&Ljy{T57U(M@=i}P4?49LAJWk7MKQYYJQSygpN+$LjcbC_QC;3McYmV9W#2#- zqROEM8O6PCm#L3Jgx4SVcd4m0t>FixT?>CHAhLSa$%qYWGfoA>DJKdxz8C3wF6if%)4HYz3Y1VGvy6> z$6@g$H_#;rH}`GyL89Q8ESP>{z!i$(k8Y$+*%RlVJempXAE)8Qs+;J3?s4olQ{^b% zyoJt!ft%=@3rPEMGb-KHn`pHhz&`1d5&m&m@E1N+ZP`p;k)NaC`YqI@$PK%Vy0I8X zlwoYTjgIDhq7y5Su6D<-=o~D#^>(_PmtT+GS8(!&+i+KT{5G5l_@pYU#GIYTY;vlx zfeFJAU`w;wE}V41*3E3LA_3jykkKI;9;Cdc!PQq(@q?!E($(9FC&FVuXL47V3C=s{ z*VqZ_PAdHNBN1vyM|~Gv)aRou@Uy$P&o)lng<%Fhb3UOFM2@HCeYzbnY&lLRuzaha zFcZ|^zx@?w7NF?V{BgtGbUqdwyPMiMmu!FqgL)d?94dkqDuWsuoa+^i`1llb3phU| z!GqiBr83^}-b4R_1Qh+64&a1L`wh*9`)o0v4!##nZPah+Vl24w zxAeXNLNk}SJF-A;LB*_n2bHY(KKdrsPrjd4^sGKQ9IheEYRNA`N3UOttQSIyeh6ol zZz&51E$2OS_&m^Jd`37Au>1jf28}JG)Q4THe@KiTb!50~TXKHeexMR91y&O*FmXki z12N~U^$*iz*t3YnK<>lB;ITjtq2cVqsF^79EI4*02{(SRlQPaY)ZR4Mvx~L`y(hzp zM}!Y};t_fkmfLsJ8}RYWZrY0v=cDxZ@{wq0kJ0D37(RZSIw%$jZ0O|@;H*Jo4c`;A zDbPt@aoitiMWFbhgSaN%{3QKvY(Do-G#(Z{MXxa5KYEI*9h5$g!fJk+Zjt4Hk3lce z2k;U1GRooH3Ni?u{xj_W6Im?${yAJ#PunYu577>XJ$wIM^Uu!5ZXosqPAER(q4)sC zLW0IjSagss=R}Kswqhw#h=E!Np@QhmFVMAE-XFERy9;6Z?b8H{<{dwR?F6IJ4vCom z!9#RCieUUJc;t_xe3d>aAHyKwHTsPCe&RLyM|{8cb-{+B!#w_6gA0f|rdwh_dkdE? zm%KrLi4f>{|6<0UZ@x(%3=_HryI!Y5Wu#YtE-|Dj;uRsP=WQARKfWbY2eBMQWY%|t zvY>acia0;C$Cr*!?|E&Dp?9Hhi-;fxuHDoZGK(2OSYz+o^fGwymV65L?6| zV8Rcm%on0S{eo3z=|+5fc9wP{|69-Dq||TwF%-ohP*}gG!{A&x`g^%vLzZ=#lS94$ z>Dl>en2|-x%fmAhSZAN)Hn1WT0ZbJKJA9GF;T>#jzjytG7=yu-@4(Ze$bIHAB}>#$DnC#Nh%};2x3O4{nS^RrNE& z`)R(BhnAxC(oy}izDCx4r?Kz)__2|iii>I;&jp8Jt&qy}Sk}bsidiskbU(b<$Q_dy z&bhJuR-2r)2}Y~*f2kUJ-*A96QxmBQ*}@Xx${k(;c#Q}3ts23BnHYDz)>^@>^d zoh=g6If`O5B%ej^rz}Sl{QNA&TLDcT$zvKsjnS5{OOt0o%AO3MFz4voPcm*DZzH`??sm_d%^Vl*rW=2l&0~uY&?;(827#^*$aJ}xPPukvxwavRQkPoH~zGkT^hWdZg;XBX5D8_ zwuem&?!@C>Q-A&@H1_QY*_GzxB9Jx(roGG(4Y~|<+C(|f%`5}@cB&!mc9sIY-?6dK z+rkFH*_YU0!|!2t(LNX>c{O8UDZyl{5)3uD0|#zq!-C)!Tqb- zYcVl;(IFPQO~<9|(Y5R`5++m4CEyXD6X3qwV9O7<56KcN< zZFuGb$e6l~>>&cS1vDWAb2LJ$yeoa27lI*NJ+rdls_U2yvaZ2>z8BwNBaFstm`+8b zu;nhiKnK6Oj*T09VVf(U(7|GiLD!?L_A!N%FeNs4a2@SF53KgEG}v$x!__6cBOfI{ zW9KYduBPLPolS-hk}N|qbEj&fM{84XlPK7N#Pv)*Xz_xZ=UK4oX2v%ktee=lnBdM@ z@a#@pB};^Ln^FzZe>3NB|zXz4;vFS2-LEF4DG`!Y|^Q8 zG;Rq?fLrclgW>6~SX5A>rmbu^!^?oi+O4dbL`aLufem*e{GP3B2>kL6HqOwX-%0~3 zUkS${wcm}V?gOnCqq}?VX3IwKf#aSN+K7HZ8pA-aJHjws^w2X*gR(o2$-lghB^jsg zWf75(?_hDpz#XU};W7aOf8WU-33{JlyuOPqAVWgZGg9paJ<5{k;w*UTB31}Iq7 z6%bhranIs9ZRc&+S>2;-GKp30`tpFk?tP5?GYTE)?RywlaB$=Q$_|S~xw9SjE@5@F zB_2lZL!a)0OZTzA_V0}6KFh`+z-7-ee|$h0_=`Z|1uKTKQ0vB}e{~NU*{J7`6dx>j zjx7`0Kaz#UzUNp3g`+(fI_AE}tkBWSA`|`YY~*hTCP%xvR*S!0)T1;HCV5V_Bjnx} z*+lsB7|Sr&`4z^`W&2qk`1Yd@`uTpAI|N&D!C*3Mndrnj$vC#3JxR|qajthk+VbK7 zRvT-&axPu&our+-bTb=hTzrsy&&)~1`6_!WQEY{3nMflh*Os@oH2Zy88r*V-Wx&m^ zF|7}`VZ6r1u^~L2-3K`y^GXocNroGryuo%b$%1Q7Fy~O7_DkdWHlCLcT&{(=U$C?U zd4*Qfc#bYGB^`Pf_3Zn1QR7aWU==AL?VP1`v+DUh*ovygDwy;Md(z4`j8%RgX?J;7 z;L(MD-3I5&!ADpDjQEs|2{J)*ir~irjEeWz>tS&sKbVDtN%Hu8gClI3lYjS_u54!& zf)N=+FJ4^5Y%kw6#(TQl(S&Hf_~6N2j)sp}TD^L~&S5jIyN37MN;ZOZM_4x4i*Ygc z@DVl(K0m^y48cA67N5c?F~gLjEFW5rvRv4Dl$F9O{O$XrY<3*4mpS3f6~@YAtQgiG zV|fGR;R%5Vo;rru_pibA`R08rd~9=jCnl0*<}_H6Ari^ryZ6nr5@%?rFfj8ID)pCL7XiGc!C3m0Odl@FW z+uPeTyzFeskHg|&t2PA3y?S{?nd84;_~xmCk->MraZg_zD?pTOE`JuBJ;jE_pv;$Z z%?uiT5}Y}KbH}OEEGk7DI^mrJ#1t5N#u-*^MrvJW*r@^B#Jaj##*a6y_#6Y+5%PYj zJU$j;cz6oIxcdv7NtK^5R)5J}vIIGjitjKY3aiGfbcb=^dp3oHO9h0Kvuv7rH&&cw zcagy=fOd&IhL~m3jHy2&^I>y59Vsqk$J%I~`4tYQh$ScR71hkM+jt0O)kN43PfOt7 zqqw>EtIyE+7E((RhqFd27}wHvuU~;_zsJ)}-NSGXW+}Cd>^bmmJO>?y-G#%o6t}oG z#^a*s=|mb8%Ohp%fuD!#@VB~__UwH(jROxAzy=4tHl7ypG!g7cq>*u|K_@DKQ(Vn8 zTO>k9JZ^c7V3xsfLp*LiO)o`b&t{g9f=;MneIv8Lff=|xi(%lc@xl7RqauDZXYOq{ zUC*{yTBwDe34xdRZpedkSWJY9R-tW}B?H=yuxKMW!ctFzriAhE;4sTV_;G+`LO4&3 zS+gK6%`z6WNK3eJYn0_2>ACIgAuu7@auEL3%Z3TO+oK>W$a+TJQ~8y0*xHDY{alG zJHs;B!VgJ7+)nDAFOY8`bA z6^?qR>;SK%!p;&)%0y9JUfNLVtae;18Y+Ti8P8&K^onN3+{RK_fTx7lPc(T`>8P(4 zV2Z#>$6Wr2*Fy#Rs+u}MB;L=F->O<2yDzOS7to5Qc+`@&RlJ<9J{48fHG*My*jr$_ zsJnco6HiKt7K(|rm5qWJydEZdtEg55in@l1Du;kDFI<;b%;N~mv;0*x<&Bj!qC`A}m-3)~S}CrChpfxs#uD z;Aa;wN$&LXw&L`I@z_+0j%xeAYGiX^MVaMJqjz*zF4dH7Z#QlTfw4G zX>oC|G*nV(1wtW|{hJjy(8}5;55)Rg`%rg#G}hVG*1A8KJ_;or9s7eBchs=YJuV-( zn~x1;4-WQ6_VjP=BIy3D)`}I@;>yK2)}lpLc6L{!r#(-<$)PvF94lv;%skAe%iein z{2hCre4ib7=a0z;Ql3e@WB=2=_CwQwp_eD#apakX%Vdg3y6NCK>=PPNTvS>bC~-BU zNHwG+_zyHh)@)|4in9EWT(X62XF+*;D@%|6#}*dlEEZ20m-GUcZ@04LtE%d)y80$- zV|8tdRbOYVsb1C8ys^5`YC2<8leKDNwR&4sU)j9Ax~{3Z%IfT~`gcaG?$(}x)~?)S z%aY5tv!yevXrDDW7|yq>rk#;NtFXGf%+!22TI z(F-E&sIsIWIk{koRns2n>F?|qv{rYCptY;B-Rh6-iu4Rad)BIr z058lvaQN8FA%N4f59_MN#+tQt)Kh>g%C-FeP|F44t6aUQsd{5wxT?Cc7PX6q)kb9p zWHw-Y)hp}QuV00VC0ayT`q>9m-%wo_Zcq&=rFKuFhB$>j;L6(i#%cpyHjEw1Le&I_ zukPyls+MqL)2fXmiSpr-C$sNWumNV6Ns;BW7Ec(y0uhMcm5*4 znle9R&HBpvM&d81bYR*-0yTv`*vk4^3{F5&_dhv%!|-ZCpq2 zrCuR}lo!NTzhNVEW@A%Lbz@kI#it&qPas1Hm`^~khDKq~t0M-1T^}Gm1XZh=R)y=T zsj;C`Xh_wntp;Yqy3MsT%tAqL%W1@tpcQ%o@zt%~(7a03gUUb=wgOfc`QTvF9niwz zB#4Fs;;UL)-PlN+73qOIM?DXN*+3x0xw3Vhm<5mxen!{V!*>A0n1ea$*@tC4ssW@_ zW0_5lK8RIn42H@yL zD^_rsfYU`jxT>172nxccHqN6*AH@3ls^;2y0#b%|dKQt?Wj@HpGf2S-jm&4$<1h&2 ziOMy#xs7KGljz1(b!(~nivo&ol9n#=VXAK+GeCgaT55No%6yP5EvqUks~ePK3yrMM z3tHRSBE9{7ZqIA)2$P>CDyJ9<4#<#?YHwe(E4pJq)g8f{(lByBd{ra8olaGc9W_oF zRUHtg>SP^Pu!g%jWBr7}V>%Oc!J7v0QQsbIC(6Ypma^3Ia2VaRVyzd8ffjI7=7a1Q=xOVx zAP^WaG|zoJb;vI9LH4%xwRYP~7Z2A2;yeN3tK1cBYwZeO(AwA8x?QPKu~!{X*#;Rh zFrSElx~x;%9IzYEY`}!hn{au4r2d31$eGxdIwn!|7hqh=B z)@FM8V`|l*GiL1uEklZy8@h%%cSX7e2@h5t%yI)U5DF0G))+Xcwszt-aCUdbuq+W9 z=8+>V`URm7|c0Dowi6^jf-tS03yLRc#x z9Lsn1_xHvs3JP|H0>Sd){N1~Ehw^v!M)KRD-34cb3U);Ld)qtmTVuU@3W`>(D#|M@ zDJ`t7J3Akxr|E?fBl>W7XYarsNA}GKPQ$g;s~VfiFsIsN%IiC<#EW2wd81|RdZVf! zm@h`PVJ~$x*pcGGVtM1$EHl31D)tAy9DJx=Z)s_0+E`VyW=*&W)jK;3hIT6)c3xmM z84lCJna18OPhQTZoo>@|VHj2kD2Icgfi>%!*J~`1E*DLkq80(TQ7rcM_dz8)u-F4^ zq*7f2?GbBvtiK)n$Se1=CF_;aj!3M@@>DnSN$_l#v0S9((SvMyp(FIC~9yGdq7=TA2vCh2_-EfCDlbo;`HYpF zspUaqgO-FUo5HK=8f!2#yr9$$43=zp^|fq)%&JdHTS&DNm0NpZp{0f{^#{vskT`kmGlKgMD?} zkc#9BOZ2Z&74;RCsflu7nF`_x%T+*8Sgt^d^paaqky_^lfg)YCNQWIoWeP)4xeAbq z%GFbGK!u{kSlvT~V!d=#j71kuk zsishh^(5deD^i@7=}AgVT3lAHI4ch-$Z}NBh{~~`fvV+2id?y#`j%rNS5Kvy z!*U(emg`O^FDEes18N-^zl@aBUsR4|Hu?qW(Jv*lV4%26j{81apos>9Si8bYY-p)p zYUUXXs>nVVRB{Lg%R;i{`)p33Vlx=hpdk$!QcVknR8Is$WhCZcNcAaDm&o!v*#Z`j z=iUj8z4%VHMm~C+Es}5F$x?^Z(s^DGb9J=O!sdeod#j!jg-R5(SmC^*UbmkVLpE@* zRQ0<(D@i~p?9@Cj_91u05ir@W?FD2#s}Gr3g_${-IgFX>7TaA(aX82tmV&ASywsu9 znqM1c*ppU@Q@(bgA6C`-?i7fI5)z?tK>S)t$$X3&4npNd;r*}mm@h0-U1fd+Aqiq) zq+f;TVq$3sU5*7${8gxBnio_k^e-tBjgzGw4F3tejnLX%l5>2GzoQ|_wV<~)jL-xp1X3oazqY_Y5 zOv4>aHn7cVfC!mF`yH*iI)&&GLgWTXruO5X5+4~U-+R0YY7_3!O01-lupBDbOSJV{ zu^LqSO3rKLrR*mdJ5&W)BhVJ5!gKoj|i@q@Ah-Xad`EFDn~O zwSuK`)mw<#?z@-GliPmH`sIcnvAI*|l|c;IP>p>*Vr8n>Hdf`T_%RgEAz(8wLbiW) zs>>-Ku<0`NV>VL}3>C{~o@A-A(x(KCj$=WWcq%EB z4?e_FleL&EgiZ3~Lu_S_c{VPqRP(F{Xb{mVF{Lai{?PBV)v_Im*;@uv29Y-UdAdFJ z{;@HnMJ4j^NeuAY9%mDVs3CS8EkzaBsiqKBWz_Lo3r2^ZK^^u4jqB6=(FUSHWSVU3 zml=FeGT^g!By^Z;6%Xmwn8-MV&&mB54;c*TL1DWG1!G%!fErA?;}PoWPEoC*ekVkFbRqCnPuK>fLak&v z!Ul+~A=>W_;k|*@3gwYjP)|=REv=wd-k=s%5E^AzGL0_Qh|GS4o^u5C#$C{MDY{X5 zB0`(FbHO3yQW$yb04ivPL2*c)_Df83=N)Cqa?%rQl3e>Fn~`FxvK~<`ZAy{XJjo_5 zupKtgtM(hUGR@Q+9DJV56d-!=2^jNjC)p_NbMjthV@88zypcnXvc*+)+J>fOl-_o8 zcJIO^5+%ofIlW6|!E(9bQI?UeF=%H><#r5~7z`P5@BJ+K;9G3=3QsTCc#X~)>Io1d zJ3SRfU&;MXu<2M#VE3_1dFP98WwroksHU=F`QEQtDtAdPgBO-i;R0FtOPFsPe!fim z08%HZ$p1LWCex5jOEL8HL=>H<38{+LJOW4T)R|~@Z81(spbby3+0#iPI`X0JCqV=Y z<>6n$uX-puoWk>`p)Iz-P>ibZ!vn4~W44_80T}ggt5re-iwpl5F6GK!vUEoZV`z(m z{@Hu$M)49V;|OP*EgUm&{2Dx7{^{3j3RYEAgyZ2fjnvUgc)UF8*KCoepX8SwLDcny z_n@^@d$@!%lE^{H{r8~TPkS0ao=4m_SxZ`L4#{Hg(`?ojW3-*v)t=TE=VY9v8Eqz8 z&eXB3zt($`9t4dvf+`Hk!!IIYOL>tcaSQRxr(U1z6OLfccjXzQ(3i884%j6VN8jris;Tq`DAKMgZC++W}uuvCMoH z3z~m>7UMPgJe#ft=y_CxZW|-JUt$v%nV`-XS8`k=K)pI;yCO^tfGG9gF&4`*{N{O7 z$9~c#eeW;X%E{CZC^n1*W$&|WA(5TR^MZ2j^H80I&#^`7wN^FR{2W^;PkSDWS+C%) z%U@*4N_|fr#V|z88S?2DVI~(nkH6l1o=uV8eU8mn5JN9wN&$#T;|#f?fb4w^B!F$U zf_UdyEJOv4!QAYFFjUcNzr?C7Oz;@_BxrJU=AoC^ocPzCV@sKQ;zd@ZaAA1jw>(w- z3ts}>9%Ad(W3V?zUc@vBWkp`L)Kl~rn(@F3nD=BHhrvKICdtPE>LxS!CA4ksakj+6 zX2zL`c|l`SedA`Nnl@GE)itl*YUSlop0uS?kyVN^WfN0kHRUkHVw9d?UaFl#`QUMO zt~~k_nhnb~QxQj?PHn7znJpe?TZS@u?$az)G5+H?W4$j_w8EwpYmQ8{t1ZDUY z#0wDTBzgQ8ntk9EHc!PFW{{RT5vS2st57z)&bp-aDw{SMK{wHCe$`AJ&5RPUD;3{l z7u&8)#bxr>cCkH%6bTu44Ge}}2ZI;A1_sq%592R2D>ggxdKxIlpI~KbwrTd<5vWIh zNt50a=*LG+u!%{E{*5Qt@}+7XwlTPQV{>gn*sfe~ThLwD``d7sO-r%>DWAZSXT1C* zyNsu31R)23eCl$I{Ia^Petq?NBoX9=tc&EzBpw(?I|e<`?nrk$n#5GH82bGeS-ee_%;#Sm8^+doKeh(t&*t+h$0zp7n&Rgy;MmRD zwvczSauWF~YqN6Y z%STv%Y)Y1$)J|1&(mLKQxSbV^r?)DVJ1aNdb~}5CwJKJ;?2A%6bqtIEV8K2TB zcV}V4i=e4bRFk(-pXmKRd1#?nJ<%f>`PM>_yX2EPxGxA)eFI$))w}U8j^e-J^8B&< z{vkUdprxg@zH*gHKyN^{eiJkRc>#&k1z`6BU_6q3M7oh4(%Lsjt-c`A(}^sth_$;9 zyR4)EJvik8-H2f{TD_4dW#ZV|q%iHtXOeTZC|f4lHqec;E=V}R;TB|{bY7r`fsd?7 zgfWR9sH(9+z2##Qc*de(d}gmfo@=hPB)c}+ruMTfE7BETH;#9UA<}a<3U3Q7Xk3^w z7*%>9IO^#S>vfR-?`Bh*Evw>RZ@2vM1irY$-)W|;i#%R>AfWFJbw)N&K9N_B(>kk+)AmipKcK{L7MmhE4qh zq&4#G$$W|4@}8B#7Zmu$zd{E}ij=q6iW7N%3ZE~pNa2e#j>(x~mV7#eFH2enM&gsE z@F-J%%1fs52V`T7m?0l-7Nu?*)B7(|`KN1V%*6IA>R3e>goz>bkfWd^Jqy|L}VV)=7o6ghQ z?c^s2yT1=-0Wq@7(6mdk8*?a0#mQE2r;ey979cQ0O`Zd}AeE~`^?TcT2b1T^>t^w4 z`PwX=9{5-4)H*~4Gehn!V z{exAfQ*5AMyTovOX^4+an&AX2D$A~=MTInS*CirD)|K#?No2a^juL)m3aUDCqaKq- zN_fdAmG&opRl+yXakHS&Vx!t_rs`Ql-6_*V{`$lxx^ZgN>WTJSyptNCOp zPUi*Eew-lJMfg^Eeht#f8eZY^2xNks`aWf#y&=ZReLMN&)FQ=ng1|y_Q2*daxWBNP zFU=@c;C5sFZ}~a$0Z*CX!~cow=^M~sALizXA{7W*o3hn<^CI1}# zb)Wog75`@Znri+fCh>Frfo;DSEgxv-Q{+%PPnRbu`8s*`T7Ff?gP#taCW1+wY;7`% zAwP2@#X~$vZm#2R$U_(MS=0@mll#x(i)6-`e47k*gP|jj^QrOgoyk8ZGKNw5B$OX} zu}zovZss%Nfd+o9nC(Gi&KMb7u5MLLT-L!OjR ziw_07RDN+2&&_tx9js7Cfk*BYMdS4Q_)PiKg_!?!Z|2X*@K(eQU*C+_;oU7r!i;X= zo8>c~!2q!?;FIL}=kOhJ|338Ak+XS*oV1ze=MJZKYCZTzDWdb|I9&7)Q`fo3=AX;+ z^IT-@!$iuW=#xnau`A))(wrvWy^7b$$Is=hBv(X9n-Cyv5A&vkn6Yf;6SG2*K(3H| z6z1)6+cv&fwV#eDRp?{Cn62Vmae}_*FtB{LgBQtrxA99f`7iI_6XmPzJWZ~RfF%rp zxk~5du?XKWotmf*HdQzWq2%cud`Oa&= z%ZWSDTnEEYNUODN9JInX$F5zco)YJ&U9F>OLE`LFOdT?EpE2?rDzeYrXoc73IL0BT zQ!@;?8q0t8PW~7PW1;+3ClAXTdhu7@d03r5pM61|xd+P^hdTKfS-DFaiY)ms{Z)H6 zw5H)bK1$Yf;nDgm1}@;n%4fQO1v+&}GD3iS>MZ+0hMVg6==clW{0Sz%8RZoVlvX(h zy|gU!VfBO{5`1E;C&_WW{42!bg~V~DEI*&$GHnDj{#t$TW_E>dRA?{b73Ro8S05iq z^nefV?J<6<#=Ey48mg@5$NgG#^W&c%;0u@@Nw$YeDb_B_OFLHK99c6b)_m_1WYvh; zxSMyVj#188soC4R;akyH%jc6ceAWHv6X@)^tL4!u*cl+GSfU3conLdQR)upOULC8^ zjwXQzPJWu)cLI(IGRTmDGib1sslhS~k;|HQ?B&H}ru&?uZWZQ0JsCDRr89Lp{2e{}1)WJZU}32eoNE?Ma>@uY4Fr2v&7peC}iXau$E(Vt$9}7MvS_ ztUSY{!g+N5uuYm7Evf@bFM<6ajZbvImp+SVX2wpCLV2l*lYE$6?N$+&l0B+bC>Juq zVY>?Fl)pLTpOE)8SAyD~J`2;*kS69Rd1n=BhczN17j4+miSnk;@mdv5*gRH@c-oJ; zz&W7?!=GSa+d2%!8JF@0;?G~o$FcZ{&+|e?%wXyC3oru9zR1tjBjfvDKqB~qUxa$j zyNusDmTUr0=E%%Sm`DVIbL7}B@nd65xjcCePnG-H5r>izO)b>p1;_hb?stxCd5woE z3rm!O(UO%uN9cF@bSN9i-)h`*|W5(b5Nk(aDk~o zu=^Tbq|AT=3;(XKt4K7_N}P7YVMmWiHfG>67qYSOm#*Ogqx(EKhiyNDb9(wh?Bq`T z3S83gvhniPukelX!>{m*<=_E+nQZtf^e*+^`IYq74Ee)<=Q57JUS;tD$*Z|Saj+IJ zh(IUA@4Aj})ua+-&0pi|4SDM@9O-Xv;NOy~zRst~TW;ig^y;ap1xt1%6JG z=iSObqBq&`tG>m@Gom_M-t}#Mo~ExJfBPN&86OiP67T{Kqhi0FDDOMMe<07lo!=-Y z{{U9^nji2~E^ob)|46p~7k^$ZyNmxqPKu*xhvNLIR2nGmuC2iIwX4fE?&r(feh5Se zHeQuW@8SQc><0nJ;1fJMq&>N}y30ZN?k@2n{?fhtd~#QF{Ew?Lgr7qqUj3`o}y83)AIp=gaiNXKa z!Be`LeWn#_Tn_60r&o8}C8sEP-)!rgZieVF+H z&yzPjN8S(0uN9MoCl5^&Q)K#USj>6n9-fv}tYZuxS~p=2i8JLzzrY+qbRw8)C7B z`w;%5OcKlF-(G?jKXEUV@tx=SRrF(a{Q4L8mGC2GcvQ)icffwr($|_|JwE<%ivK7N zeX{pB|E&DOaXzH1CsB}3y}%b793#F&%X4+|(PO;IF(+!^x;V=!*5l$-oH;E&{bPQH zo<|Jt@Wqo!VHcc^RY^yaRTx*?;ApauvpYhoqbe?oHc|}dS^Bb>IbfX}6+iMC&tRl| zbIAfN(88v5y)-?#>;wFI?q^q(Uw(@h%iiY^VPATZe@td$9*JqOeD^)RI)P-7USUlj zvHvom3)wga=jpHWtL4$x;rIh@z~?{u20Zq%U-8R{yR-z(fNOkd#q#Q%zDdfZcS?)Z zBuJ^;*>8g3XMV#6XheU#&^5e9!sgsNbxMTsr{Cg?B`_9f&*y^o@~3X7a=rDjcjj;T z-xA33U1Mw{_UJCFQ{#H@cl@sjTm`PtGZHR!>D8%`mA>m?%%$)nBKt(fZhPOw*kQF| z(@QtoA(Dk<}tlgdg4#~ ztb$KbbanC9sks^YyQ}wa3YLC~lI#1yPL1wQOe6*&(0z)U>ubbLjjkujjjo{dQ#74! z96L3#H%E)asDbo9P1E(gWT(crWt{lCTdnj=)dlZWE4sqW_0ppiE948~#fL6rWuKz& zblcfT$l@@Cr3s^T)4H-xQFiYQhc1*RYIl`*I9_(^gDIG7VpJ`VbB-f!R?c2M9y-oT zwJ#(spItV>2(jITLJ57|Y+gbh>W^{FB(Ps|jb)eCKKkC)6V8$r=s=}mS!7t0m1#3Ooo zv2B)^8ef(nKIXEbLs;_tonqQl+U3ObOKo{#B@xBe@lx>PeZB&{?R5e{VOOyq8)ao@t-nY6@msB9RU zagi$yCwa0Zz84VR5@V60pl)4}n~TJIo~P60-V$iuEk$BF{WVEGTr9rmK*SqM#ChI) zjE74hKO0|sZ<%;NlzQ_RdLp|Uho?}2=^m{RPR>Jo*K#qBC6HaSLL`mzaDU?(@d5Xg zTwf!^sM%hEY6ssDj(qxKk>;uMLY-L2;%N=y^XyY=&i2$izCldoW9Z!D>gKg_%4Ec> z`7X* zMSk2aK0eqXmdlGf#E<3j9b&Nuuxp3dk{A?6cZf-xVgA}FX2k#8DgNrE^|x-Z&{G?| zG)q3YSuB#fdc-ZBY+9wKVall0nKIDA`ZaQ2ugFN-9H0b7J!D+{DK|$&##p*8h%P~j z4@N~D8)xeqZsc4CMSZOYsSQ{5iC{eA{8PM+$kfLadf0SBPGx zD2^iBvSLF}GA#{Le?G6g*ToC+qpypdvi*?w z*{7S^x$1%$=xI_i?n1`^CckkY^Y>};whO_y+D|xgSS+RqaIUooxm70*1NoUZi}vv- zLXRt#BTv5^i~W}_5ZLc+#KK2cxfmzEe3=**pYRQF2b)64)0J3cLmJWUok*$blaIYG z<}ZNBwA#8Nxae%4R~ynjxH%@-!Yz=LQG{F3usCqvH$~Yp%z=@tW7}J$a#m{x@|Sg{ zP;XZ!(u$%;`P7-FC>ZJ@bq$ahi_V!VQi<0G>Dv=(q{10)nXLOT^@D9MMZ6eD(c;m-z6N`oXaX*_F&-;P6P0$^@4Fmm@?Tb5UH?Fmiveix3 zajK^}WQ)t-kHoa0n6;x5DZ(UT8pLe_$OG#kC}a;*u4_>D+S@5+bk}ptpcPY$nAzoz{~bRLi8yFx;0ZF4ckanDxM|xS7Rx0`K4m&0qewg zsJSOJfP`E7c5L37OW5>750dre>>r9E*?*r{F$$U2*_kqL5r$0f(-=!V$crSicVV*J zbH7+GAG=?aPf{gR3Whsla{2?JP_BDGXHMAbM=WDhLJvIj+(-2I?f zGTFH>Nrf6PWeqj7 z^2)nJ<`~1C{Lj1K)aRkEQc$N>RWM0bu~)wLj<8_2ljYH$h*4)*D|NeY?O6~x>WVP+7lhMe<>9*?Hc8DT zsF3XSF`NXy&ao5yVwt@CDv`QGYt2&PUPX4g-<_;dqhaWzWO?{9sM9qfTRneZs z_$mmEGUbJ`7RIt-xws;IXtzofCIbhT)!o_C3d5N2RHl){Ii+IL zaDvs(>2B@MK?jE54((xuE~ehRP>Dg?iB5>7AS`B;>CRA!)oJ&u1|$XZiFL10L%fmM z4~aQ>v|o%vCuHXa2RpYW;x;|NO>|~LHo5f?czXE|!of!#5|ibLheRs7NPh1TQRjsE zl07QknP#7hF?g3Q-MUr2_r91GpZJ)V%Hqp^E`I9E(T0A<#l$4=R3T>` zhij+D(dtxNg}m(*L`UZw2hcsoQD31O^D<~Pyew|^xayBy7FTeVFTe1bC~%^6S^tih zA3yP$*v5xP2utOc-w~VT6aOs=f_ArSi$Xu)>8t6O7s3Z5=WFG|o#$|j-O=_)K9&|{ zP^33Q{vG=%@x0f?$4s`I6w``Ou_hM7wefJRO3;?A{R2J5>0%(@j@lj|Z|&-fw6+i8 z_Quqt_RjYF0~?-5RTyQ#6Jpf3OfooxL{7O=OjD^pN4_eOWbaAQDv#a{>hs=4cz*mO zle-cYQB!BQHOe5$!DGhY& z$im^SMs+T#v!8C}#qGrUzF!^C_Wwovs@Q=g(CpqQ4$N-HRlIfB6U{kw)lDB_H|55^ zAW!`LK1`CTF+q?&{1C&?x<=_$(x_Q5#tv>WILc>O^IKZhN%lDEO#MjAb@MLEKN96> zP89^{%9yEYF;ykL?<28d^c1fJvCU5WO-yugE_46kii1f>pYSQDKoHmS7U6)m_nTJ5 z|Kz8jMkj3@;Zsn*;sG3$*w`3eUEf$!Te&LST!$ESb*=jLPGfatcx_G78O^K1tD9?T ztMGjwn(D8dkktAw)!HbRPfY3^MQdU)d1PYJnOJe~esqbZ9iKBkmISC{N| zK_wd^kt#uq_&Zf`;UrU_jyy3bDI?%e20Hu0NZ#)vS!PcbNO5qTNKXK)^F<^%VRBOX z*$x`TymwQ;=Fhu1P3Sxw2c%vI;_yQ z7mU=+Y$I2hX(y@BYv+4JxJQd*Q4f~f-!8(5Y1c=K%q?4PynuHgY)}iJZs_`Md)1faU%Uq zu9n5ZygVEjknL6JH<#1^d%OA#<0en`5P5o(~aOmi_`!2$yNv0!n;@h~6s6;yN;V zOUxubKxCQE3_(1&bIfjz`_n_{82g(;q4E-fSPl%POt%6-4_d}+(v#+$JoU#i VgSfvrSn9`J{14*(C@KEZ{{!5!$-n>r