diff --git a/CMakeLists.txt b/CMakeLists.txt index 27da45a80..6b3adb5cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,9 @@ include(CMakeDependentOption) include(CMakePushCheckState) include(GNUInstallDirs) +# Help debug dependencies with custom targets that PTG uses a lot +SET(GRAPHVIZ_CUSTOM_TARGETS TRUE) + # The current version number # This uses the numbering scheme from libtool -version_number c:r:a # see http://www.sourceware.org/autobook/autobook/autobook_61.html @@ -173,6 +176,12 @@ option(PARSEC_GPU_HIP_ALLOC_PER_TILE "Tile based allocation engine for GPU memory (instead of internal management of a complete allocation)" OFF) mark_as_advanced(PARSEC_GPU_HIP_ALLOC_PER_TILE) +option(PARSEC_GPU_WITH_LEVEL_ZERO + "Enable GPU support using LEVEL_ZERO kernels" ON) +option(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE + "Tile based allocation engine for GPU memory (instead of internal management + of a complete allocation)" OFF) +mark_as_advanced(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) option(PARSEC_GPU_WITH_OPENCL "Enable GPU support using OpenCL kernels" OFF) mark_as_advanced(PARSEC_GPU_WITH_OPENCL) # Hide this as it is not supported yet @@ -737,6 +746,19 @@ int main(int argc, char *argv[]) { endif() set(PARSEC_HAVE_HIP ${HIP_NOT_CUDA_FOUND} CACHE BOOL "True if PaRSEC provide support for HIP") endif( PARSEC_GPU_WITH_HIP ) + if( PARSEC_GPU_WITH_LEVEL_ZERO ) + find_package(level-zero) + find_package(DPCPP) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + endif( PARSEC_GPU_WITH_LEVEL_ZERO ) find_package(AYUDAME QUIET) set(PARSEC_HAVE_AYUDAME ${AYUDAME_FOUND}) @@ -915,6 +937,13 @@ include(AddDocumentedFiles) add_subdirectory(parsec) +# +# Add dependency to Level-Zero if it is enabled +# +if(PARSEC_HAVE_LEVEL_ZERO) + target_link_libraries(parsec PRIVATE level_zero::ze_loader) +endif(PARSEC_HAVE_LEVEL_ZERO) + # # Now continue with compiling the tests. # @@ -1237,6 +1266,8 @@ install(FILES ${PROJECT_SOURCE_DIR}/cmake_modules/FindPAPI.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindOTF2.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindHWLOC.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/Findlevel-zero.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/FindDPCPP.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/ParsecCompilePTG.cmake DESTINATION ${PARSEC_INSTALL_CMAKEDIR}) diff --git a/cmake_modules/FindDPCPP.cmake b/cmake_modules/FindDPCPP.cmake new file mode 100644 index 000000000..c4073d268 --- /dev/null +++ b/cmake_modules/FindDPCPP.cmake @@ -0,0 +1,44 @@ +if(DPCPP_EXECUTABLE) + get_filename_component( _dpcpp_path ${DPCPP_EXECUTABLE} PATH ) +else(DPCPP_EXECUTABLE) + set(_dpcpp_path "") +endif(DPCPP_EXECUTABLE) +find_program( DPCPP_EXECUTABLE NAMES dpcpp HINTS ${_dpcpp_path} ENV PATH ) +mark_as_advanced( DPCPP_EXECUTABLE ) + +if( DPCPP_EXECUTABLE ) + execute_process(COMMAND ${DPCPP_EXECUTABLE} --version + RESULT_VARIABLE _res + OUTPUT_VARIABLE _out + ERROR_VARIABLE _err) + if( _res EQUAL 0 ) + string(REGEX MATCH "([^\n]+)" _ ${_out}) + message(STATUS "Found dpcpp: ${DPCPP_EXECUTABLE} version ${CMAKE_MATCH_1}") + + get_filename_component( _dpcpp_dir ${DPCPP_EXECUTABLE} DIRECTORY ) + get_filename_component( _dpcpp_lib_dir "${_dpcpp_dir}/../lib" ABSOLUTE ) + + find_library(SYCL_LIBRARY sycl HINTS "${_dpcpp_lib_dir}" "${SYCL_LIBRARY_DIR}" ENV "SYCL_LIBRARY_DIR") + if( SYCL_LIBRARY ) + message(STATUS "SYCL library: ${SYCL_LIBRARY}") + + get_filename_component( _dpcpp_inc_dir "${_dpcpp_dir}/../include" ABSOLUTE ) + find_file(_sycl_include_file "sycl.hpp" HINTS "${_dpcpp_inc_dir}" "${_dpcpp_inc_dir}/sycl/CL" "${SYCL_INCLUDE_DIR}" ENV "SYCL_INCLUDE_DIR") + if(_sycl_include_file) + get_filename_component(SYCL_INCLUDE_DIR "${_sycl_include_file}" DIRECTORY) + message(STATUS "SYCL include dir: ${SYCL_INCLUDE_DIR}") + set(PARSEC_HAVE_DPCPP "TRUE" CACHE BOOL "True if PaRSEC provide support for dpc++") + else(_sycl_include_file) + message(WARNING "Found DPC++, and SYCL library, but could not find SYCL include directory. Define SYCL_INCLUDE_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif(_sycl_include_file) + else( SYCL_LIBRARY ) + message(WARNING "Found DPC++, but did not find SYCL library. Define SYCL_LIBRARY_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif( SYCL_LIBRARY ) + else( _res EQUAL 0 ) + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + message(WARNING "${DPCPP_EXECUTABLE} does not work: 'dpcpp -v' returned ${_res}, with error ${_err}") + message(WARNING "dpc++ support is disabled, set DPCPP_EXECUTABLE to the path of dpcpp to enable DPC++ support, and/or fix your environment to run dpcpp") + endif( _res EQUAL 0 ) +endif( DPCPP_EXECUTABLE ) \ No newline at end of file diff --git a/cmake_modules/Findlevel-zero.cmake b/cmake_modules/Findlevel-zero.cmake new file mode 100644 index 000000000..51b63912e --- /dev/null +++ b/cmake_modules/Findlevel-zero.cmake @@ -0,0 +1,46 @@ +if(LEVEL_ZERO_ROOT_DIR) + message(STATUS "Trying to locate level-zero library and headers under ${LEVEL_ZERO_ROOT_DIR}") + find_library(ZE_LOADER_LIBRARY "ze_loader" HINTS "${LEVEL_ZERO_ROOT_DIR}/lib" "${LEVEL_ZERO_ROOT_DIR}/lib64" NO_DEFAULT_PATH) + find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH) + + if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY) + check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT) + + if(ZE_LOADER_HAVE_ZEINIT) + message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}") + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + set(LEVEL_ZERO_FOUND TRUE) + else(ZE_LOADER_HAVE_ZEINIT) + if(NOT ZE_LOADER_HAVE_ZEINIT) + message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search") + endif(NOT ZE_LOADER_HAVE_ZEINIT) + endif(ZE_LOADER_HAVE_ZEINIT) + else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + if(NOT ZE_LOADER_LIBRARY) + message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search") + endif(NOT ZE_LOADER_LIBRARY) + if(NOT LEVEL_ZERO_INCLUDE_DIR) + message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search") + endif(NOT LEVEL_ZERO_INCLUDE_DIR) + endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) +endif(LEVEL_ZERO_ROOT_DIR) + +if(NOT LEVEL_ZERO_FOUND) + find_package(PkgConfig QUIET) + + if(PKG_CONFIG_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if(LEVEL_ZERO_FOUND) + pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir) + pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir) + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + endif(LEVEL_ZERO_FOUND) + endif(PKG_CONFIG_FOUND) +endif(NOT LEVEL_ZERO_FOUND) diff --git a/cmake_modules/PaRSECConfig.cmake.in b/cmake_modules/PaRSECConfig.cmake.in index 60460a99b..507adf6e0 100644 --- a/cmake_modules/PaRSECConfig.cmake.in +++ b/cmake_modules/PaRSECConfig.cmake.in @@ -73,6 +73,23 @@ if(@PARSEC_HAVE_HIP@) SET(PARSEC_HAVE_HIP TRUE) endif(@PARSEC_HAVE_HIP@) +if(@PARSEC_HAVE_LEVEL_ZERO@) + if(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + set(LEVEL_ZERO_ROOT_DIR "@LEVEL_ZERO_ROOT_DIR@") + endif(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + find_package(level-zero) + find_package(DPCPP) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) +endif(@PARSEC_HAVE_LEVEL_ZERO@) + if(@PARSEC_PROF_TRACE@) # Nothing exportable here, if this test succeed then PaRSEC supports tracing endif(@PARSEC_PROF_TRACE@) diff --git a/cmake_modules/ParsecCompilePTG.cmake b/cmake_modules/ParsecCompilePTG.cmake index affdcb4dd..10a449584 100644 --- a/cmake_modules/ParsecCompilePTG.cmake +++ b/cmake_modules/ParsecCompilePTG.cmake @@ -7,7 +7,7 @@ # function(target_ptg_source_ex) set(options DEBUG LINE FORCE_PROFILE) - set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H FUNCTION_NAME DEP_MANAGEMENT) + set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H DESTINATION_DPCPP FUNCTION_NAME DEP_MANAGEMENT) set(multipleValueArgs WARNINGS IGNORE_PROPERTIES PTGPP_FLAGS) cmake_parse_arguments(PARSEC_PTGPP "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -51,6 +51,9 @@ function(target_ptg_source_ex) else() set(outname_h "${outname}.h") endif() + if(DEFINED PARSEC_PTGPP_DESTINATION_DPCPP) + set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPCPP}") + endif() if(DEFINED PARSEC_PTGPP_FUNCTION_NAME) set(fnname "${PARSEC_PTGPP_FUNCTION_NAME}") @@ -91,12 +94,21 @@ function(target_ptg_source_ex) # its cmake source_file name, yet we depend on the source_file name as it is how cmake tracks it get_property(location SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY LOCATION) - add_custom_command( + if(DEFINED outname_dpcpp) + add_custom_command( + OUTPUT ${outname_h} ${outname_c} ${outname_dpcpp} + COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -S ${outname_dpcpp} -f ${fnname} + MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} + DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c} ${outname_dpcpp}) + else(DEFINED outname_dpcpp) + add_custom_command( OUTPUT ${outname_h} ${outname_c} COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -f ${fnname} MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) - add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + endif(DEFINED outname_dpcpp) # Copy the properties to the generated files get_property(cflags SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY COMPILE_OPTIONS) @@ -110,11 +122,24 @@ function(target_ptg_source_ex) COMPILE_OPTIONS "${cflags}" INCLUDE_DIRECTORIES "${includes}" COMPILE_DEFINITIONS "${defs}") + if(DEFINED outname_dpcpp) + set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}" + TARGET_DIRECTORY ${target} + PROPERTIES + GENERATED 1 + COMPILE_OPTIONS "${cflags}" + INCLUDE_DIRECTORIES "${includes}" + COMPILE_DEFINITIONS "${defs}") + endif(DEFINED outname_dpcpp) # make sure we produce .h before we build other .c in the target add_dependencies(${target} ptgpp_${target}.${outname}) # add to the target - target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + if(DEFINED outname_dpcpp) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}") + else(DEFINED outname_dpcpp) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + endif(DEFINED outname_dpcpp) get_target_property(_includes ${target} INCLUDE_DIRECTORIES) list(FIND _includes "${CMAKE_CURRENT_BINARY_DIR}" _i1) diff --git a/contrib/pregen_flex_bison.tar b/contrib/pregen_flex_bison.tar index 0c7a4baa5..a0322692f 100644 Binary files a/contrib/pregen_flex_bison.tar and b/contrib/pregen_flex_bison.tar differ diff --git a/parsec/data.c b/parsec/data.c index 525b04bfd..6a98bc8d5 100644 --- a/parsec/data.c +++ b/parsec/data.c @@ -94,8 +94,7 @@ static void parsec_data_destruct(parsec_data_t* obj ) #endif /* defined(PARSEC_DEBUG_PARANOID) */ assert(obj->super.obj_reference_count > 1); parsec_data_copy_detach( obj, copy, i ); - if ( !(device->type & PARSEC_DEV_CUDA) - && !(device->type & PARSEC_DEV_HIP) ) { + if ( !(device->type & PARSEC_DEV_GPU) ){ /** * GPU copies are normally stored in LRU lists, and must be * destroyed by the release list to free the memory on the device diff --git a/parsec/include/parsec/parsec_options.h.in b/parsec/include/parsec/parsec_options.h.in index 44cabc508..b10875997 100644 --- a/parsec/include/parsec/parsec_options.h.in +++ b/parsec/include/parsec/parsec_options.h.in @@ -123,6 +123,8 @@ #cmakedefine PARSEC_HAVE_HWLOC #cmakedefine PARSEC_HAVE_PAPI #cmakedefine PARSEC_HAVE_CUDA +#cmakedefine PARSEC_HAVE_LEVEL_ZERO +#cmakedefine PARSEC_HAVE_DPCPP #cmakedefine PARSEC_HAVE_HIP #cmakedefine PARSEC_HAVE_OPENCL #cmakedefine PARSEC_HAVE_MPI diff --git a/parsec/interfaces/dtd/insert_function.c b/parsec/interfaces/dtd/insert_function.c index 2d6877f02..85ca2b48f 100644 --- a/parsec/interfaces/dtd/insert_function.c +++ b/parsec/interfaces/dtd/insert_function.c @@ -41,6 +41,9 @@ #if defined(PARSEC_HAVE_CUDA) #include "parsec/mca/device/cuda/device_cuda.h" #endif /* defined(PARSEC_HAVE_CUDA) */ +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero.h" +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ #include "parsec/mca/mca_repository.h" #include "parsec/constants.h" @@ -1494,6 +1497,9 @@ parsec_dtd_startup(parsec_context_t *context, // If CUDA is enabled, let the CUDA device activated for this // taskpool. if( PARSEC_DEV_CUDA == device->type ) continue; + // If LEVEL_ZERO is enabled, let the LEVEL_ZERO device activated for this + // taskpool. + if( PARSEC_DEV_LEVEL_ZERO == device->type ) continue; if( NULL != device->taskpool_register ) if( PARSEC_SUCCESS != device->taskpool_register(device, (parsec_taskpool_t *)tp)) { @@ -2334,6 +2340,12 @@ static parsec_hook_return_t parsec_dtd_gpu_task_submit(parsec_execution_stream_t gpu_task->stage_in = parsec_default_cuda_stage_in; gpu_task->stage_out = parsec_default_cuda_stage_out; return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index); +#endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + case PARSEC_DEV_LEVEL_ZERO: + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + return parsec_level_zero_kernel_scheduler(es, gpu_task, dev_index); #endif default: parsec_fatal("DTD scheduling on device type %d: this is not a valid GPU device type in this build", device->type); @@ -2404,8 +2416,10 @@ int parsec_dtd_task_class_add_chore(parsec_taskpool_t *tp, if(PARSEC_DEV_CUDA == device_type) { incarnations[i].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; - } - else { + } else if(PARSEC_DEV_LEVEL_ZERO == device_type) { + incarnations[i].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; + } else { dtd_tc->cpu_func_ptr = function; incarnations[i].hook = parsec_dtd_cpu_task_submit; } @@ -3289,8 +3303,11 @@ __parsec_dtd_taskpool_create_task(parsec_taskpool_t *tp, /* Special case for CUDA: we need an intermediate */ (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; - } - else { + } if( device_type == PARSEC_DEV_LEVEL_ZERO ) { + /* Special case for LEVEL ZERO: we need an intermediate */ + (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; + } else { /* Default case: the user-provided function is directly the hook to call */ (*incarnations)[0].hook = fpointer; // We can directly call the CPU hook dtd_tc->cpu_func_ptr = fpointer; diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf.h b/parsec/interfaces/ptg/ptg-compiler/jdf.h index e76029ec3..29c23bbee 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf.h @@ -99,6 +99,7 @@ typedef struct jdf_compiler_global_args { char *input; char *output_c; char *output_h; + char *output_dpcpp; char *output_o; char *funcid; jdf_warning_mask_t wmask; @@ -143,10 +144,17 @@ extern int jdfdebug; * We remember the line number in the JDF file where this external code was found */ typedef struct jdf_external_entry { - struct jdf_object_t super; + struct jdf_object_t super; + struct jdf_external_entry *next; + char *language; char *external_code; } jdf_external_entry_t; +typedef struct jdf_code_string { + char *language; + char *string; +} jdf_code_string_t; + /** A global is a variable name, optionally an expression to define it, * and a line number associated with it for error printing purposes */ diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index 1cf949484..9cce36695 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -3,7 +3,6 @@ * of Tennessee Research Foundation. All rights * reserved. */ - #include "parsec/parsec_config.h" #include @@ -32,10 +31,14 @@ extern const char *yyfilename; static FILE *cfile; static int cfile_lineno; +static FILE *dpcppfile = NULL; +static int dpcppfile_lineno = 0; static FILE *hfile; static int hfile_lineno; static const char *jdf_basename; static const char *jdf_cfilename; +static char *jdf_hfilename; +static char *jdf_dpcppfilename; /* Optional declarations of local functions */ static int jdf_expr_depends_on_symbol(const char *varname, const jdf_expr_t *expr); @@ -137,6 +140,43 @@ static void coutput(const char *format, ...) } } +#if defined(__GNUC__) +static void dpcpp_output(const char *format, ...) __attribute__((format(printf,1,2))); +#endif +static void dpcpp_output(const char *format, ...) +{ + va_list ap; + char *res; + int len; + + va_start(ap, format); + len = vasprintf(&res, format, ap); + va_end(ap); + + if( len == -1 ) { + fprintf(stderr, "Unable to ouptut a string: %s\n", strerror(errno)); + } else if( 0 < len ) { +#if (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) + char *start = res, *end; + while( NULL != (end = strchr(start, '\n'))) { + if( (end != start) && (end[-1] != '\r')) { + fwrite(start, (end - start), 1, dpcppfile); + fwrite("\r\n", 2, 1, dpcppfile); + } else { + fwrite(start, (end - start) + 1, 1, dpcppfile); + } + len -= (end - start) + 1; + start = end + 1; /* skip the current \n */ + } + fwrite(start, len, 1, dpcppfile); +#else + fwrite(res, len, 1, dpcppfile); +#endif /* (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) */ + dpcppfile_lineno += nblines(res); + free(res); + } +} + #if defined(__GNUC__) static void houtput(const char *format, ...) __attribute__((format(printf,1,2))); #endif @@ -1254,7 +1294,7 @@ static jdf_expr_t *jdf_expr_lv_next(jdf_expr_t *head, jdf_expr_t *cur) return head; } -static void jdf_coutput_prettycomment(char marker, const char *format, ...) +static char *jdf_prettycomment(string_arena_t *sa, char marker, const char *format, ...) { int ls, rs, i, length, vs; va_list ap, ap2; @@ -1294,15 +1334,17 @@ static void jdf_coutput_prettycomment(char marker, const char *format, ...) ls = (80 - length) / 2; rs = 80 - length - ls; } - coutput("/*"); + string_arena_init(sa); + string_arena_add_string(sa, "/*"); for(i = 0; i < 5; i++) - coutput("%c", marker); - coutput("%s%s", indent(ls/2), v); /* indent drop two spaces */ - coutput("%s", indent(rs/2)); /* dont merge these two calls. Read the comment on the indent function */ + string_arena_add_string(sa, "%c", marker); + string_arena_add_string(sa, "%s%s", indent(ls/2), v); /* indent drop two spaces */ + string_arena_add_string(sa, "%s", indent(rs/2)); /* dont merge these two calls. Read the comment on the indent function */ for(i = 0; i < 5; i++) - coutput("%c", marker); - coutput("*/\n\n"); + string_arena_add_string(sa, "%c", marker); + string_arena_add_string(sa, "*/\n\n"); free(v); + return string_arena_get_string(sa); } /** Structure Generators **/ @@ -1511,6 +1553,9 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) "#if defined(PARSEC_HAVE_CUDA)\n" "#include \"parsec/mca/device/cuda/device_cuda.h\"\n" "#endif /* defined(PARSEC_HAVE_CUDA) */\n" + "#if defined(PARSEC_HAVE_LEVEL_ZERO)\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */\n" "#if defined(PARSEC_HAVE_HIP)\n" "#include \"parsec/mca/device/hip/device_hip.h\"\n" "#endif /* defined(PARSEC_HAVE_HIP) */\n" @@ -1530,19 +1575,15 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) jdf_generate_predeclarations(jdf); } -static void jdf_generate_structure(jdf_t *jdf) +static void jdf_dump_internal_structure(string_arena_t *sa, jdf_t *jdf) { - int nbfunctions, need_profile = 0; - string_arena_t *sa1, *sa2; + int nbfunctions = 0; jdf_function_entry_t* f; jdf_param_list_t *pl; JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); - sa1 = string_arena_new(64); - sa2 = string_arena_new(64); - - coutput("#include \"%s.h\"\n\n" + string_arena_add_string(sa, "#include \"%s.h\"\n\n" "struct __parsec_%s_internal_taskpool_s {\n" " parsec_%s_taskpool_t super;\n" " volatile int32_t sync_point;\n" @@ -1550,30 +1591,64 @@ static void jdf_generate_structure(jdf_t *jdf) " parsec_task_t* startup_queue;\n", jdf_basename, jdf_basename, jdf_basename); - coutput(" /* The ranges to compute the hash key */\n"); + string_arena_add_string(sa, " /* The ranges to compute the hash key */\n"); for(f = jdf->functions; f != NULL; f = f->next) { if( 0 == (f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { for(pl = f->parameters; pl != NULL; pl = pl->next) { - coutput(" int %s_%s_range;\n", f->fname, pl->name); + string_arena_add_string(sa, " int %s_%s_range;\n", f->fname, pl->name); } } else { - coutput(" /* nothing for %s as it gets a user-defined make_key */\n", + string_arena_add_string(sa, " /* nothing for %s as it gets a user-defined make_key */\n", f->fname); } } - coutput(" /* The list of data repositories "); + string_arena_add_string(sa, " /* The list of data repositories "); for(f = jdf->functions; NULL != f; f = f->next) { if( 0 != function_has_data_output(f) ) { - coutput(" %s ", f->fname); + string_arena_add_string(sa, " %s ", f->fname); } } - coutput("*/\n"); + string_arena_add_string(sa, "*/\n"); if(nbfunctions != 0 ) { - coutput(" data_repo_t* repositories[%d];\n", nbfunctions ); + string_arena_add_string(sa, " data_repo_t* repositories[%d];\n", nbfunctions ); + } + + string_arena_add_string(sa, "};\n\n"); +} + +static int jdf_has_dpcpp_chore(const jdf_t *jdf, const char *fname) +{ + jdf_function_entry_t *f; + jdf_body_t* body; + jdf_def_list_t *type_property; + + for(f = jdf->functions; f != NULL; f = f->next) { + if( (NULL != fname) && strcmp(f->fname, fname) ) continue; + for(body = f->bodies; body != NULL; body = body->next) { + jdf_find_property(body->properties, "type", &type_property); + if( NULL != type_property && !strcmp(type_property->expr->jdf_var, "DPCPP")) + return 1; + } + if(NULL != fname) return 0; } + return 0; +} + +static void jdf_generate_structure(jdf_t *jdf) +{ + int nbfunctions, need_profile = 0; + string_arena_t *sa1, *sa2; + jdf_function_entry_t* f; + + JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); - coutput("};\n\n"); + sa1 = string_arena_new(64); + sa2 = string_arena_new(64); + + jdf_dump_internal_structure(sa1, jdf); + coutput("%s\n", string_arena_get_string(sa1)); + string_arena_init(sa1); for( f = jdf->functions; need_profile == 0 && NULL != f; f = f->next ) { /* If the profile property is ON then enable the profiling array */ @@ -1595,6 +1670,9 @@ static void jdf_generate_structure(jdf_t *jdf) if( 1 < strlen(string_arena_get_string(sa1)) ) { coutput("/* Globals */\n%s\n", string_arena_get_string(sa1)); } + if( 1 < strlen(string_arena_get_string(sa1)) && jdf_has_dpcpp_chore(jdf, NULL) ) { + dpcpp_output("/* Globals */\n%s\n", string_arena_get_string(sa1)); + } coutput("static inline int parsec_imin(int a, int b) { return (a <= b) ? a : b; };\n\n" "static inline int parsec_imax(int a, int b) { return (a >= b) ? a : b; };\n\n"); @@ -4101,7 +4179,8 @@ static void jdf_generate_one_function( const jdf_t *jdf, jdf_function_entry_t *f has_in_in_dep |= (fl->flow_flags & JDF_FLOW_HAS_IN_DEPS); } - jdf_coutput_prettycomment('*', "%s", f->fname); + coutput("%s", jdf_prettycomment(sa, '*', "%s", f->fname)); + string_arena_init(sa); prefix = (char*)malloc(strlen(f->fname) + strlen(jdf_basename) + 32); @@ -4805,7 +4884,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e string_arena_t *sa_range_multiplier = string_arena_new(64); jdf_variable_list_t *vl; expr_info_t info = EMPTY_EXPR_INFO; - int idx; if( !(f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { coutput("static inline parsec_key_t %s(const parsec_taskpool_t *tp, const parsec_assignment_t *as)\n" @@ -4828,7 +4906,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e info.sa = sa_range_multiplier; info.assignments = "assignment"; - idx = 0; for(vl = f->locals; vl != NULL; vl = vl->next) { string_arena_init(sa_range_multiplier); @@ -4860,7 +4937,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e */ coutput(" (void)%s;\n", vl->name); } - idx++; } string_arena_init(sa_range_multiplier); @@ -6125,20 +6201,20 @@ jdf_generate_code_flow_final_writes(const jdf_t *jdf, string_arena_free(sa); } -static void jdf_generate_code_dry_run_before(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_dry_run_before(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { (void)jdf; (void)f; - coutput("\n\n#if !defined(PARSEC_PROF_DRY_BODY)\n\n"); + output_fn("\n\n#if !defined(PARSEC_PROF_DRY_BODY)\n\n"); } -static void jdf_generate_code_dry_run_after(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_dry_run_after(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { (void)jdf; (void)f; - coutput("\n\n#endif /*!defined(PARSEC_PROF_DRY_BODY)*/\n\n"); + output_fn("\n\n#endif /*!defined(PARSEC_PROF_DRY_BODY)*/\n\n"); } static void jdf_generate_code_grapher_task_done(const jdf_t *jdf, const jdf_function_entry_t *f, const char* context_name) @@ -6153,7 +6229,7 @@ static void jdf_generate_code_grapher_task_done(const jdf_t *jdf, const jdf_func jdf_property_get_string(f->properties, JDF_PROP_UD_HASH_STRUCT_NAME, NULL), context_name, context_name, context_name); } -static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf_function_entry_t *f) +static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf_function_entry_t *f, void (*output_fn)(const char *format, ...)) { string_arena_t *sa; sa = string_arena_new(64); @@ -6163,11 +6239,11 @@ static void jdf_generate_code_cache_awareness_update(const jdf_t *jdf, const jdf dump_dataflow_varname, NULL, "", " cache_buf_referenced(es->closest_cache, ", ");\n", ""); if( strlen(string_arena_get_string(sa)) ) { - coutput(" /** Cache Awareness Accounting */\n" - "#if defined(PARSEC_CACHE_AWARENESS)\n" - "%s);\n" - "#endif /* PARSEC_CACHE_AWARENESS */\n", - string_arena_get_string(sa)); + output_fn(" /** Cache Awareness Accounting */\n" + "#if defined(PARSEC_CACHE_AWARENESS)\n" + "%s);\n" + "#endif /* PARSEC_CACHE_AWARENESS */\n", + string_arena_get_string(sa)); } string_arena_free(sa); } @@ -6581,6 +6657,9 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, int di; int profile_on; char* output; + void (*gpu_output)(const char *format, ...); + int *gpu_lineno; + const char *gpu_filename; profile_on = profile_enabled(f->properties) && profile_enabled(body->properties); @@ -6588,6 +6667,16 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, char* dev_upper = strdup_upper(type_property->expr->jdf_var); char* dev_lower = strdup_lower(type_property->expr->jdf_var); + if(0 == strcmp(dev_lower, "dpcpp")) { + gpu_output = dpcpp_output; + gpu_lineno = &dpcppfile_lineno; + gpu_filename = jdf_dpcppfilename; + } else { + gpu_output = coutput; + gpu_lineno = &cfile_lineno; + gpu_filename = jdf_cfilename; + } + /* Get the dynamic function properties */ dyld = jdf_property_get_string(body->properties, "dyld", NULL); dyldtype = jdf_property_get_string(body->properties, "dyldtype", "void*"); @@ -6609,123 +6698,158 @@ static void jdf_generate_code_hook_gpu(const jdf_t *jdf, dump_string, NULL, "", " (void)", ";", ";\n")); /* Generate the kernel_submit structure and function */ - coutput("struct parsec_body_%s_%s_%s_s {\n" - " uint8_t index;\n" - " %sStream_t stream;\n" - " %s dyld_fn;\n" - "};\n" - "\n" - "static int %s_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" - " parsec_gpu_task_t *gpu_task,\n" - " parsec_gpu_exec_stream_t *gpu_stream )\n" - "{\n" - " %s *this_task = (%s *)gpu_task->ec;\n" - " parsec_device_%s_module_t *%s_device = (parsec_device_%s_module_t*)gpu_device;\n" - " parsec_%s_exec_stream_t *%s_stream = (parsec_%s_exec_stream_t*)gpu_stream;\n" - " __parsec_%s_internal_taskpool_t *__parsec_tp = (__parsec_%s_internal_taskpool_t *)this_task->taskpool;\n" - " struct parsec_body_%s_%s_%s_s parsec_body = { %s_device->%s_index, %s_stream->%s_stream, NULL };\n" - "%s\n" - " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)parsec_body; (void)%s_device; (void)%s_stream;\n", - dev_lower, jdf_basename, f->fname, - dev_lower, - dyldtype, - dev_lower, jdf_basename, f->fname, - parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), - dev_lower, dev_lower, dev_lower, - dev_lower, dev_lower, dev_lower, - jdf_basename, jdf_basename, - dev_lower, jdf_basename, f->fname, dev_lower, dev_lower, dev_lower, dev_lower, - string_arena_get_string( sa3 ), - dev_lower, dev_lower); + if( 0 == strcmp(dev_lower, "dpcpp") ) { + dpcpp_output("extern \"C\" {\n" + " int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream);\n" + "}\n" + "\n" + "int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream )\n" + "{\n" + " %s *this_task = reinterpret_cast<%s *>(gpu_task->ec);\n" + " parsec_device_level_zero_module_t *level_zero_device = reinterpret_cast(gpu_device);\n" + " parsec_level_zero_exec_stream_t *level_zero_stream = reinterpret_cast(gpu_stream);\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = reinterpret_cast<__parsec_%s_internal_taskpool_t *>(this_task->taskpool);\n" + " parsec_sycl_wrapper_platform_t *parsec_sycl_platform = level_zero_device->driver->swp;\n" + " parsec_sycl_wrapper_device_t *parsec_sycl_device = level_zero_device->swd;\n" + " parsec_sycl_wrapper_queue_t *parsec_sycl_queue = level_zero_stream->swq;\n" + "\n" + "%s\n" + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)level_zero_device; (void)level_zero_stream; (void)parsec_sycl_platform; (void)parsec_sycl_device; (void)parsec_sycl_queue;\n", + jdf_basename, f->fname, + jdf_basename, f->fname, + parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), + jdf_basename, jdf_basename, + string_arena_get_string( sa3 )); + } else { + coutput("struct parsec_body_%s_%s_%s_s {\n" + " uint8_t index;\n" + " %sStream_t stream;\n" + " %s dyld_fn;\n" + "};\n" + "\n" + "static int %s_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream )\n" + "{\n" + " %s *this_task = (%s *)gpu_task->ec;\n" + " parsec_device_%s_module_t *%s_device = (parsec_device_%s_module_t*)gpu_device;\n" + " parsec_%s_exec_stream_t *%s_stream = (parsec_%s_exec_stream_t*)gpu_stream;\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = (__parsec_%s_internal_taskpool_t *)this_task->taskpool;\n" + " struct parsec_body_%s_%s_%s_s parsec_body = { %s_device->%s_index, %s_stream->%s_stream, NULL };\n" + "%s\n" + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)parsec_body; (void)%s_device; (void)%s_stream;\n", + dev_lower, jdf_basename, f->fname, + dev_lower, + dyldtype, + dev_lower, jdf_basename, f->fname, + parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), + dev_lower, dev_lower, dev_lower, + dev_lower, dev_lower, dev_lower, + jdf_basename, jdf_basename, + dev_lower, jdf_basename, f->fname, dev_lower, dev_lower, dev_lower, dev_lower, + string_arena_get_string( sa3 ), + dev_lower, dev_lower); + } ai2.sa = sa2; ai2.where = "out"; output = UTIL_DUMP_LIST(sa, f->dataflow, next, dump_data_initialization_from_data_array, &ai2, "", "", "", ""); if( 0 != strlen(output) ) { - coutput(" /** Declare the variables that will hold the data, and all the accounting for each */\n" - "%s\n", - output); + gpu_output(" /** Declare the variables that will hold the data, and all the accounting for each */\n" + "%s\n", + output); } /** * Generate code for the simulation. */ - coutput(" /** Update starting simulation date */\n" - "#if defined(PARSEC_SIM)\n" - " {\n" - " this_task->sim_exec_date = 0;\n"); + gpu_output(" /** Update starting simulation date */\n" + "#if defined(PARSEC_SIM)\n" + " {\n" + " this_task->sim_exec_date = 0;\n"); for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { if(fl->flow_flags & JDF_FLOW_TYPE_CTL) continue; /* control flow, nothing to store */ - coutput(" data_repo_entry_t *e%s = this_task->data._f_%s.source_repo_entry;\n" - " if( (NULL != e%s) && (e%s->sim_exec_date > this_task->sim_exec_date) )\n" - " this_task->sim_exec_date = e%s->sim_exec_date;\n", - fl->varname, fl->varname, - fl->varname, fl->varname, - fl->varname); - } - coutput(" if( this_task->task_class->sim_cost_fct != NULL ) {\n" - " this_task->sim_exec_date += this_task->task_class->sim_cost_fct(this_task);\n" - " }\n" - " if( es->largest_simulation_date < this_task->sim_exec_date )\n" - " es->largest_simulation_date = this_task->sim_exec_date;\n" - " }\n" - "#endif\n"); - - jdf_generate_code_cache_awareness_update(jdf, f); - - coutput("#if defined(PARSEC_DEBUG_NOISIER)\n" - " {\n" - " char tmp[MAX_TASK_STRLEN];\n" - " PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, \"GPU[%%s]:\\tEnqueue on device %%s priority %%d\", gpu_device->super.name, \n" - " parsec_task_snprintf(tmp, MAX_TASK_STRLEN, (parsec_task_t *)this_task),\n" - " this_task->priority );\n" - " }\n" - "#endif /* defined(PARSEC_DEBUG_NOISIER) */\n"); - - jdf_generate_code_dry_run_before(jdf, f); - jdf_coutput_prettycomment('-', "%s BODY", f->fname); + gpu_output(" data_repo_entry_t *e%s = this_task->data._f_%s.source_repo_entry;\n" + " if( (NULL != e%s) && (e%s->sim_exec_date > this_task->sim_exec_date) )\n" + " this_task->sim_exec_date = e%s->sim_exec_date;\n", + fl->varname, fl->varname, + fl->varname, fl->varname, + fl->varname); + } + gpu_output(" if( this_task->task_class->sim_cost_fct != NULL ) {\n" + " this_task->sim_exec_date += this_task->task_class->sim_cost_fct(this_task);\n" + " }\n" + " if( es->largest_simulation_date < this_task->sim_exec_date )\n" + " es->largest_simulation_date = this_task->sim_exec_date;\n" + " }\n" + "#endif\n"); + + jdf_generate_code_cache_awareness_update(jdf, f, gpu_output); + + gpu_output("#if defined(PARSEC_DEBUG_NOISIER)\n" + " {\n" + " char tmp[MAX_TASK_STRLEN];\n" + " PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, \"GPU[%%s]:\\tEnqueue on device %%s priority %%d\", gpu_device->super.name, \n" + " parsec_task_snprintf(tmp, MAX_TASK_STRLEN, (parsec_task_t *)this_task),\n" + " this_task->priority );\n" + " }\n" + "#endif /* defined(PARSEC_DEBUG_NOISIER) */\n"); + + jdf_generate_code_dry_run_before(jdf, f, gpu_output); + gpu_output("%s", jdf_prettycomment(sa, '-', "%s BODY", f->fname)); if( profile_on ) { - coutput("#if defined(PARSEC_PROF_TRACE)\n" - " if(gpu_stream->prof_event_track_enable) {\n" - " PARSEC_TASK_PROF_TRACE(gpu_stream->profiling,\n" - " PARSEC_PROF_FUNC_KEY_START(this_task->taskpool,\n" - " this_task->task_class->task_class_id),\n" - " (parsec_task_t*)this_task, 1);\n" - " gpu_task->prof_key_end = PARSEC_PROF_FUNC_KEY_END(this_task->taskpool,\n" - " this_task->task_class->task_class_id);\n" - " gpu_task->prof_event_id = this_task->task_class->key_functions->\n" - " key_hash(this_task->task_class->make_key(this_task->taskpool, ((parsec_task_t*)this_task)->locals), NULL);\n" - " gpu_task->prof_tp_id = this_task->taskpool->taskpool_id;\n" - " }\n" - "#endif /* PARSEC_PROF_TRACE */\n"); + gpu_output("#if defined(PARSEC_PROF_TRACE)\n" + " if(gpu_stream->prof_event_track_enable) {\n" + " PARSEC_TASK_PROF_TRACE(gpu_stream->profiling,\n" + " PARSEC_PROF_FUNC_KEY_START(this_task->taskpool,\n" + " this_task->task_class->task_class_id),\n" + " (parsec_task_t*)this_task, 1);\n" + " gpu_task->prof_key_end = PARSEC_PROF_FUNC_KEY_END(this_task->taskpool,\n" + " this_task->task_class->task_class_id);\n" + " gpu_task->prof_event_id = this_task->task_class->key_functions->\n" + " key_hash(this_task->task_class->make_key(this_task->taskpool, ((parsec_task_t*)this_task)->locals), NULL);\n" + " gpu_task->prof_tp_id = this_task->taskpool->taskpool_id;\n" + " }\n" + "#endif /* PARSEC_PROF_TRACE */\n"); } if ( NULL != dyld ) { - coutput(" /* Pointer to dynamic gpu function */\n" - " {\n" - " int chore_idx = 0;\n" - " for ( ; PARSEC_DEV_NONE != this_task->task_class->incarnations[chore_idx].type; ++chore_idx) {\n" - " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_%s) break;\n" - " }\n" - " /* The void* cast prevents the compiler from complaining about the type change */\n" - " parsec_body.dyld_fn = (%s)(void*)this_task->task_class->incarnations[chore_idx].dyld_fn;\n" - " }\n\n", - dev_upper, - dyldtype ); + gpu_output(" /* Pointer to dynamic gpu function */\n" + " {\n" + " int chore_idx = 0;\n" + " for ( ; PARSEC_DEV_NONE != this_task->task_class->incarnations[chore_idx].type; ++chore_idx) {\n" + " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_%s) break;\n" + " }\n" + " /* The void* cast prevents the compiler from complaining about the type change */\n" + " parsec_body.dyld_fn = (%s)(void*)this_task->task_class->incarnations[chore_idx].dyld_fn;\n" + " }\n\n", + dev_upper, + dyldtype ); + } + + gpu_output("%s\n", body->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { + gpu_output("#line %d \"%s\"\n", (*gpu_lineno)+1, gpu_filename); } + gpu_output("%s\n", jdf_prettycomment(sa, '-', "END OF %s BODY", f->fname)); + jdf_generate_code_dry_run_after(jdf, f, gpu_output); + gpu_output(" return PARSEC_HOOK_RETURN_DONE;\n" + "}\n\n"); - coutput("%s\n", body->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { - coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); + if(0 == strcmp(dev_lower, "dpcpp")) { + coutput("extern int dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream);\n" + "\n", jdf_basename, f->fname); } - jdf_coutput_prettycomment('-', "END OF %s BODY", f->fname); - jdf_generate_code_dry_run_after(jdf, f); - coutput(" return PARSEC_HOOK_RETURN_DONE;\n" - "}\n\n"); /* Generate the hook device */ coutput("static int %s_%s(parsec_execution_stream_t *es, %s *this_task)\n" @@ -6949,10 +7073,11 @@ static void jdf_generate_code_hook(const jdf_t *jdf, } if( NULL != type_property) { type_upper = strdup_upper(type_property->expr->jdf_var); - coutput("#if defined(PARSEC_HAVE_%s)\n", type_upper); + coutput("#if defined(PARSEC_HAVE_%s) /* thomas */\n", type_upper); if (!strcasecmp(type_property->expr->jdf_var, "cuda") - || !strcasecmp(type_property->expr->jdf_var, "hip")) { + || !strcasecmp(type_property->expr->jdf_var, "hip") + || !strcasecmp(type_property->expr->jdf_var, "dpcpp")) { jdf_generate_code_hook_gpu(jdf, f, body, name); goto hook_end_block; } @@ -7020,7 +7145,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, if ((NULL == type_property) || (!strcmp(type_property->expr->jdf_var, "RECURSIVE"))) { coutput(" /** Transfer the ownership to the CPU */\n" - "#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP)\n"); + "#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO) || defined(PARSEC_HAVE_HIP)\n"); for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { /* Update the ownership of read/write data */ @@ -7038,19 +7163,22 @@ static void jdf_generate_code_hook(const jdf_t *jdf, ((fl->flow_flags & JDF_FLOW_TYPE_WRITE) ? "PARSEC_FLOW_ACCESS_RW" : "PARSEC_FLOW_ACCESS_READ") : "PARSEC_FLOW_ACCESS_WRITE"))); } } - coutput("#endif /* defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) */\n"); + coutput("#endif /* defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO) || defined(PARSEC_HAVE_HIP) */\n"); } - jdf_generate_code_cache_awareness_update(jdf, f); + jdf_generate_code_cache_awareness_update(jdf, f, coutput); - jdf_generate_code_dry_run_before(jdf, f); - jdf_coutput_prettycomment('-', "%s BODY", f->fname); + jdf_generate_code_dry_run_before(jdf, f, coutput); + coutput("%s", jdf_prettycomment(sa, '-', "%s BODY", f->fname)); + string_arena_init(sa); coutput("%s\n", body->external_code); if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); } - jdf_coutput_prettycomment('-', "END OF %s BODY", f->fname); - jdf_generate_code_dry_run_after(jdf, f); + coutput("%s", jdf_prettycomment(sa, '-', "END OF %s BODY", f->fname)); + string_arena_init(sa); + + jdf_generate_code_dry_run_after(jdf, f, coutput); coutput(" return PARSEC_HOOK_RETURN_DONE;\n" "}\n"); @@ -7058,7 +7186,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, string_arena_free(sa2); hook_end_block: - if( NULL != type_property) { + if( NULL != type_upper ) { coutput("#endif /* defined(PARSEC_HAVE_%s) */\n", type_upper); free(type_upper); } @@ -7667,7 +7795,7 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, string_arena_t *sa_tmp_type_r = string_arena_new(256); string_arena_t *sa_temp_r = string_arena_new(1024); - int depnb, last_datatype_idx; + int last_datatype_idx; assignment_info_t ai; expr_info_t info = EMPTY_EXPR_INFO; int nb_open_ldef; @@ -7717,7 +7845,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, for(fl = f->dataflow; fl != NULL; fl = fl->next) { flowempty = 1; flowtomem = 0; - depnb = 0; last_datatype_idx = -1; string_arena_init(sa_coutput); string_arena_init(sa_deps); @@ -7951,8 +8078,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, jdf_dump_context_assignment(sa1, jdf, f, fl, string_arena_get_string(sa_ontask), dl->guard->calltrue, dl, JDF_OBJECT_LINENO(dl), " ", "nc")); - depnb++; - string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -7973,7 +8098,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, "\n"); } } else { - depnb++; string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -8004,7 +8128,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, } break; } - depnb++; /* Dump the previous dependencies */ OUTPUT_PREV_DEPS((1U << dl->dep_index), sa_datatype, sa_deps); @@ -8432,14 +8555,41 @@ int jdf_force_termdet_dynamic(jdf_t* jdf) #include #endif -int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, jdf_t *jdf) +int jdf2c(const char *output_c, const char *output_h, + const char *output_dpcpp, const char *_jdf_basename, jdf_t *jdf) { int ret = 0; jdf_cfilename = output_c; + jdf_hfilename = strdup(output_h); + jdf_dpcppfilename = strdup(output_dpcpp); jdf_basename = _jdf_basename; cfile = NULL; hfile = NULL; + dpcppfile = NULL; + + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa2; + sa2 = string_arena_new(64); + dpcppfile = fopen(jdf_dpcppfilename, "w"); + if( dpcppfile == NULL ) { + fprintf(stderr, "unable to create %s: %s\n", jdf_dpcppfilename, strerror(errno)); + ret = -1; + goto err; + } + + dpcpp_output("#include \"parsec.h\"\n" + "#include \"level_zero/ze_api.h\"\n" + "#include \"sycl/ext/oneapi/backend/level_zero.hpp\"\n" + "#include \"parsec/parsec_internal.h\"\n" + "#include \"parsec/execution_stream.h\"\n" + "#include \"parsec/mca/device/device.h\"\n" + "#include \"parsec/mca/device/device_gpu.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero_dpcpp.h\"\n" + "\n"); + string_arena_free(sa2); + } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) /* When we apply indent/awk to the output of jdf2c, we need to make @@ -8527,6 +8677,7 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, cfile_lineno = 1; hfile_lineno = 1; + dpcppfile_lineno = 1; /** * Now generate the code. @@ -8543,20 +8694,38 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, /** * Dump the prologue section */ - if( NULL != jdf->prologue ) { - coutput("%s", jdf->prologue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->prologue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in prologue: code ignored", ent->language); + } + } + + /* Now we can include the header generated to define the datatypes */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa = string_arena_new(64); + jdf_dump_internal_structure(sa, jdf); + dpcpp_output("%s\n" + "typedef struct __parsec_%s_internal_taskpool_s __parsec_%s_internal_taskpool_t;\n" + "\n", + string_arena_get_string(sa), + jdf_basename, jdf_basename); + string_arena_free(sa); } /* Dump references to arenas_datatypes array */ struct jdf_name_list* g; - int datatype_index = 0; for( g = jdf->datatypes; NULL != g; g = g->next ) { coutput("#define PARSEC_%s_%s_ADT (&__parsec_tp->super.arenas_datatypes[PARSEC_%s_%s_ADT_IDX])\n", jdf_basename, g->name, jdf_basename, g->name); - datatype_index++; } jdf_generate_structure(jdf); @@ -8583,13 +8752,35 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, free_name_placeholders(); + /* Sometimes (typically because of a try {} catch block), sycl re-includes some definitions at the end of the file + * these definitions can be polluted by macros. Undefine the macros. */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa1; + sa1 = string_arena_new(64); + + UTIL_DUMP_LIST_FIELD(sa1, jdf->globals, next, name, + dump_string, NULL, + "", "#undef ", "\n", "\n"); + + dpcpp_output("%s\n", string_arena_get_string(sa1)); + string_arena_free(sa1); + } + /** * Dump all the epilogue sections */ - if( NULL != jdf->epilogue ) { - coutput("%s", jdf->epilogue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n",cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->epilogue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in epilogue: code ignored", ent->language); + } } /** @@ -8603,10 +8794,22 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, err: if( NULL != cfile ) { fclose(cfile); + cfile = NULL; } if( NULL != hfile ) { fclose(hfile); + hfile = NULL; + } + + if( NULL != dpcppfile ) { + fclose(dpcppfile); + dpcppfile = NULL; + } + + if(NULL != jdf_dpcppfilename) { + free(jdf_dpcppfilename); + jdf_dpcppfilename = NULL; } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h index 4d02c7ada..769b98ad3 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h @@ -12,6 +12,9 @@ int jdf_optimize( jdf_t* jdf ); int jdf_force_termdet_dynamic(jdf_t* jdf); -int jdf2c(const char *output_c, const char *output_h, const char *_basename, jdf_t *jdf); +int jdf_force_termdet_dynamic(jdf_t* jdf); + +int jdf2c(const char *output_c, const char *output_h, + const char *output_dpcpp, const char *_basename, jdf_t *jdf); #endif /* _jdf2c_h */ diff --git a/parsec/interfaces/ptg/ptg-compiler/main.c b/parsec/interfaces/ptg/ptg-compiler/main.c index ab705d00f..8168cd2b1 100644 --- a/parsec/interfaces/ptg/ptg-compiler/main.c +++ b/parsec/interfaces/ptg/ptg-compiler/main.c @@ -29,6 +29,7 @@ static jdf_compiler_global_args_t DEFAULTS = { .input = "-", .output_c = "a.c", .output_h = "a.h", + .output_dpcpp = "a", .output_o = "a.o", .funcid = "a", .wmask = JDF_ALL_WARNINGS, @@ -56,10 +57,11 @@ static void usage(void) " --debug|-d Enable debug output\n" " --input|-i Input File (JDF) (default '%s')\n" " --output|-o Set the BASE name for .c, .h, .o and function name (no default).\n" - " Changing this value has precendence over the defaults of\n" + " Changing this value has precedence over the defaults of\n" " --output-c, --output-h, and --function-name\n" " --output-c|-C Set the name of the .c output file (default '%s' or BASE.c)\n" " --output-h|-H Set the name of the .h output file (default '%s' or BASE.h)\n" + " --output-dpcpp|-S Set the based name of the .dpcpp.* output files (default '%s' or BASE.dpcpp)\n" " --function-name|-f Set the unique identifier of the generated function\n" " The generated function will be called PaRSEC__new\n" " (default %s)\n" @@ -96,6 +98,7 @@ static void usage(void) DEFAULTS.input, DEFAULTS.output_c, DEFAULTS.output_h, + DEFAULTS.output_dpcpp, DEFAULTS.funcid, (DEFAULTS.dep_management == DEP_MANAGEMENT_INDEX_ARRAY ? DEP_MANAGEMENT_INDEX_ARRAY_STRING : (DEFAULTS.dep_management == DEP_MANAGEMENT_DYNAMIC_HASH_TABLE ? DEP_MANAGEMENT_DYNAMIC_HASH_TABLE_STRING : @@ -171,6 +174,7 @@ static void parse_args(int argc, char *argv[]) char *O = NULL; char *h = NULL; char *o = NULL; + char *s = NULL; char *f = NULL; struct option longopts[] = { @@ -179,6 +183,7 @@ static void parse_args(int argc, char *argv[]) { "output-c", required_argument, NULL, 'C' }, { "output-h", required_argument, NULL, 'H' }, { "output-o", required_argument, NULL, 'O' }, + { "output-dpcpp", required_argument, NULL, 'S' }, { "output", required_argument, NULL, 'o' }, { "function-name", required_argument, NULL, 'f' }, { "Wmasked", no_argument, &wmasked, 1 }, @@ -204,7 +209,7 @@ static void parse_args(int argc, char *argv[]) print_jdf_line = !DEFAULTS.noline; - while( (ch = getopt_long(argc, argv, "dDi:C:H:o:f:hEsIO:M:I:", longopts, NULL)) != -1) { + while( (ch = getopt_long(argc, argv, "dDi:C:H:o:S:f:hEsIO:M:I:", longopts, NULL)) != -1) { switch(ch) { case 'd': yydebug = 1; @@ -238,6 +243,11 @@ static void parse_args(int argc, char *argv[]) free( o ); o = strdup(optarg); break; + case 'S': + if( NULL != s ) + free( s ); + s = strdup(optarg); + break; case 'f': if( NULL != f ) free( f ); @@ -319,6 +329,15 @@ static void parse_args(int argc, char *argv[]) JDF_COMPILER_GLOBAL_ARGS.output_o = DEFAULTS.output_o; } + if( NULL != s ) { + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(s); + } else { + if(NULL != o) { + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(o); + } else + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = DEFAULTS.output_dpcpp; + } + if( NULL == c) { if( NULL != o ) { JDF_COMPILER_GLOBAL_ARGS.output_c = (char*)malloc(strlen(o) + 3); @@ -361,6 +380,8 @@ static void parse_args(int argc, char *argv[]) free(h); if( NULL != o ) free(o); + if( NULL != s ) + free(s); if( print_compile_cmd ) { /* print the compilation options used to compile the preprocessed output */ @@ -433,6 +454,7 @@ int main(int argc, char *argv[]) if( jdf2c(JDF_COMPILER_GLOBAL_ARGS.output_c, JDF_COMPILER_GLOBAL_ARGS.output_h, + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp, JDF_COMPILER_GLOBAL_ARGS.funcid, ¤t_jdf) < 0 ) { return 1; diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.l b/parsec/interfaces/ptg/ptg-compiler/parsec.l index edb6a4e0c..cbf015a3e 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.l +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.l @@ -110,7 +110,8 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} /* find the end of the language */ for(lend = lstart + 1; yytext[lend] != '"'; lend++) /* nothing */; - assert( (yytext[lstart] == 'C') && ((lend-lstart) == 1) ); + YYLVAL(code).language = (char*)calloc(lend-lstart+1, 1); + memcpy(YYLVAL(code).language, &yytext[lstart], lend-lstart); /* Search for the start of the body */ for(bstart = lend + 1; yytext[bstart] != '{'; bstart++ ) /* nothing */; @@ -120,14 +121,15 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} for( lend = 0; lend <= bstart; lend++ ) if( yytext[lend] == '\n' ) skip_line++; current_lineno += skip_line; - YYLVAL(string) = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); + YYLVAL(code).string = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); ECHO; return EXTERN_DECL; } (inline_c)?{WHITE}*\%c?\{([^\%]|(\%[^\}]))+\%\} { int start; for(start = 0; yytext[start] != '{'; start++) /* nothing */ ; start++; - YYLVAL(string) = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).string = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).language = strdup("C"); ECHO; return EXTERN_DECL; } SIMCOST { ECHO; return SIMCOST; } diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.y b/parsec/interfaces/ptg/ptg-compiler/parsec.y index 92393ca41..0478bdbe8 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.y +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.y @@ -271,6 +271,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %union { int number; + jdf_code_string_t code; char* string; jdf_expr_operand_t expr_op; jdf_external_entry_t *external_code; @@ -291,7 +292,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, }; %glr-parser -%expect 5 +%expect 6 %type function %type param_list @@ -323,7 +324,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %type bodies %type VAR -%type EXTERN_DECL +%type EXTERN_DECL %type BODY_END %type ARROW %type PROPERTIES_ON @@ -372,23 +373,28 @@ jdf_file: prologue jdf epilogue current_jdf.epilogue = $3; } ; -prologue: EXTERN_DECL +prologue: EXTERN_DECL prologue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; } ; -epilogue: EXTERN_DECL +epilogue: EXTERN_DECL epilogue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; - } + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; @@ -1325,7 +1331,7 @@ expr_simple: expr_simple EQUAL expr_simple { $$ = new(jdf_expr_t); $$->op = JDF_C_CODE; - $$->jdf_c_code.code = $1; + $$->jdf_c_code.code = $1.string; $$->jdf_type = PARSEC_RETURN_TYPE_INT32; $$->local_variables = current_locally_bound_variables; $$->scope = -1; diff --git a/parsec/mca/device/CMakeLists.txt b/parsec/mca/device/CMakeLists.txt index ad676ee55..f2eb1af08 100644 --- a/parsec/mca/device/CMakeLists.txt +++ b/parsec/mca/device/CMakeLists.txt @@ -1,8 +1,8 @@ set(MCA_${COMPONENT}_SOURCES mca/device/device.c) -if(PARSEC_HAVE_CUDA OR PARSEC_HAVE_HIP) +if(PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO OR PARSEC_HAVE_HIP) list(APPEND MCA_${COMPONENT}_SOURCES mca/device/device_gpu.c mca/device/transfer_gpu.c) -endif() +endif(PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO OR PARSEC_HAVE_HIP ) set_property(TARGET parsec APPEND PROPERTY diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index 793032ca4..b7c373e5f 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -88,7 +88,7 @@ int parsec_get_best_device( parsec_task_t* this_task, double ratio ) /* Make sure data_in is not NULL */ if( NULL == this_task->data[i].data_in ) continue; /* And that we have a data (aka it is not NEW) */ - if( NULL == this_task->data[i].source_repo_entry ) continue; + if( 0 && NULL == this_task->data[i].source_repo_entry ) continue; /* Data is updated by the task, and we try to minimize the data movements */ if( (NULL != this_task->task_class->out[i]) && diff --git a/parsec/mca/device/device.h b/parsec/mca/device/device.h index 25c1fa3a1..66f34ff42 100644 --- a/parsec/mca/device/device.h +++ b/parsec/mca/device/device.h @@ -63,12 +63,15 @@ typedef struct parsec_device_base_component_2_0_0 parsec_device_base_component_t #define PARSEC_DEV_CPU ((uint8_t)(1 << 0)) #define PARSEC_DEV_RECURSIVE ((uint8_t)(1 << 1)) #define PARSEC_DEV_CUDA ((uint8_t)(1 << 2)) -#define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 3)) -#define PARSEC_DEV_OPENCL ((uint8_t)(1 << 4)) -#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 5)) -#define PARSEC_DEV_HIP ((uint8_t)(1 << 6)) -#define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x3f) -#define PARSEC_DEV_ALL ((uint8_t) 0x3f) +#define PARSEC_DEV_LEVEL_ZERO ((uint8_t)(1 << 3)) +#define PARSEC_DEV_DPCPP ((uint8_t)(1 << 3)) +#define PARSEC_DEV_HIP ((uint8_t)(1 << 4)) +#define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 5)) +#define PARSEC_DEV_OPENCL ((uint8_t)(1 << 6)) +#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 7)) +#define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x7f) +#define PARSEC_DEV_ALL ((uint8_t) 0x7f) +#define PARSEC_DEV_GPU (PARSEC_DEV_CUDA|PARSEC_DEV_LEVEL_ZERO|PARSEC_DEV_HIP) #define PARSEC_DEV_MAX_NB_TYPE (7) #define PARSEC_DEV_DATA_ADVICE_PREFETCH ((int) 0x01) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 9fb543fa4..710fb9219 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -72,9 +72,9 @@ parsec_gpu_check_space_needed(parsec_device_gpu_module_t *gpu_device, return space_needed; } -#if defined(PARSEC_PROF_TRACE) void parsec_gpu_init_profiling(void) { +#if defined(PARSEC_PROF_TRACE) if(parsec_gpu_profiling_initiated == 0) { parsec_profiling_add_dictionary_keyword("cuda", "fill:#66ff66", 0, NULL, @@ -100,8 +100,8 @@ void parsec_gpu_init_profiling(void) &parsec_gpu_use_memory_key_start, &parsec_gpu_use_memory_key_end); parsec_gpu_profiling_initiated = 1; } -} #endif +} void parsec_gpu_enable_debug(void) { diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index 129bd2b03..6c0d83aa2 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -227,8 +227,6 @@ extern int parsec_gpu_prefetch_key_start; extern int parsec_gpu_prefetch_key_end; extern int parsec_device_gpu_one_profiling_stream_per_gpu_stream; -void parsec_gpu_init_profiling(void); - typedef struct { uint64_t size; uint64_t data_key; @@ -237,6 +235,7 @@ typedef struct { #define PARSEC_DEVICE_GPU_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" #endif /* defined(PROFILING) */ +void parsec_gpu_init_profiling(void); void dump_exec_stream(parsec_gpu_exec_stream_t* exec_stream); void dump_GPU_state(parsec_device_gpu_module_t* gpu_device); diff --git a/parsec/mca/device/level_zero/ValidateModule.CMake b/parsec/mca/device/level_zero/ValidateModule.CMake new file mode 100644 index 000000000..1c2d08bd7 --- /dev/null +++ b/parsec/mca/device/level_zero/ValidateModule.CMake @@ -0,0 +1,37 @@ +# For now assume that the upper level did the CUDA search and that all +# necessary dependencies on CUDA have been correctly setup + +if( PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP ) + SET(MCA_${COMPONENT}_${MODULE} ON) + FILE(GLOB MCA_${COMPONENT}_${MODULE}_SOURCES ${MCA_BASE_DIR}/${COMPONENT}/${MODULE}/[^\\.]*.c) + ADD_CUSTOM_COMMAND(OUTPUT ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero + COMMAND ${DPCPP_EXECUTABLE} + -I${LEVEL_ZERO_INCLUDE_DIR} + -I${SYCL_INCLUDE_DIR} + -I${PROJECT_SOURCE_DIR} + -I${PROJECT_SOURCE_DIR}/parsec/include + -I${PROJECT_BINARY_DIR}/parsec/include + -I${PROJET_BINARY_DIR} + -Wall -fpic -g -c ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp) + SET_SOURCE_FILES_PROPERTIES( + ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + PROPERTIES + EXTERNAL_OBJECT true + GENERATED true) + add_library(parsec-dpcpp ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o) + target_link_libraries(parsec PRIVATE parsec-dpcpp ${SYCL_LIBRARY}) + install(TARGETS parsec-dpcpp + EXPORT parsec-targets + DESTINATION ${PARSEC_INSTALL_LIBDIR}) + SET(MCA_${COMPONENT}_${MODULE}_CONSTRUCTOR "${COMPONENT}_${MODULE}_static_component") + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_internal.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_dpcpp.h + DESTINATION include/parsec/mca/device/level_zero ) +else (PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) + MESSAGE(STATUS "Module ${MODULE} not selectable: does not have LEVEL_ZERO (${PARSEC_HAVE_LEVEL_ZERO}) or DPCPP (${PARSEC_HAVE_DPCPP})") + SET(MCA_${COMPONENT}_${MODULE} OFF) +endif(PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h new file mode 100644 index 000000000..95d791926 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED + + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/device.h" +#include "parsec/mca/device/device_gpu.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" + +#include + +BEGIN_C_DECLS + +struct parsec_level_zero_task_s; +typedef struct parsec_level_zero_task_s parsec_level_zero_task_t; + +struct parsec_level_zero_exec_stream_s; +typedef struct parsec_level_zero_exec_stream_s parsec_level_zero_exec_stream_t; + +struct parsec_device_level_zero_module_s; +typedef struct parsec_device_level_zero_module_s parsec_device_level_zero_module_t; + +struct parsec_device_level_zero_driver_s; +typedef struct parsec_device_level_zero_driver_s parsec_device_level_zero_driver_t; + +struct parsec_level_zero_workspace_s; +typedef struct parsec_level_zero_workspace_s parsec_level_zero_workspace_t; + +extern parsec_device_base_component_t parsec_device_level_zero_component; + +struct parsec_level_zero_task_s { + parsec_gpu_task_t super; +}; + +struct parsec_device_level_zero_driver_s { + ze_driver_handle_t ze_driver; + ze_context_handle_t ze_context; + uint32_t ref_count; + parsec_sycl_wrapper_platform_t *swp; +}; + +struct parsec_device_level_zero_module_s { + parsec_device_gpu_module_t super; + uint8_t level_zero_index; + parsec_device_level_zero_driver_t *driver; + ze_device_handle_t ze_device; + parsec_sycl_wrapper_device_t *swd; +}; + +PARSEC_OBJ_CLASS_DECLARATION(parsec_device_level_zero_module_t); + +struct parsec_level_zero_exec_stream_s { + parsec_gpu_exec_stream_t super; + /* There is exactly one task per active fence (max_fences being the uppoer bound). + * Upon fence completion the complete_stage function associated with the task is + * called, and this will decide what is going on next with the task. If the task + * remains in the system the function is supposed to update it. + */ + ze_fence_handle_t *fences; + ze_command_list_handle_t *command_lists; + ze_command_queue_handle_t level_zero_cq; + parsec_sycl_wrapper_queue_t *swq; +}; + + +/** +* Progress +*/ +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on GPU fences, + * such a fence indicates that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ); +parsec_hook_return_t +parsec_dpcpp_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ); + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ +/** + * Overload the default data_copy_t with a GPU specialized type + */ +typedef parsec_data_copy_t parsec_gpu_data_copy_t; + +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); +int +parsec_default_dpcpp_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); +int +parsec_default_dpcpp_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + +END_C_DECLS + +#define PARSEC_LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + parsec_warning( "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c new file mode 100644 index 000000000..51ae04d57 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" + +#include + +PARSEC_OBJ_CLASS_INSTANCE(parsec_device_level_zero_module_t, parsec_device_module_t, NULL, NULL); + +static int device_level_zero_component_open(void); +static int device_level_zero_component_close(void); +static int device_level_zero_component_query(mca_base_module_2_0_0_t **module, int *priority); +static int device_level_zero_component_register(void); + +int parsec_device_level_zero_index, parsec_device_level_zero_enabled; +int parsec_level_zero_sort_pending = 0, parsec_level_zero_max_streams = PARSEC_GPU_MAX_STREAMS; +int level_zero_mask, level_zero_nvlink_mask; +int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; + +int32_t parsec_LEVEL_ZERO_sort_pending_list = 0; + +char* level_zero_lib_path = NULL; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +parsec_device_base_component_t parsec_device_level_zero_component = { + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + PARSEC_DEVICE_BASE_VERSION_2_0_0, + + /* Component name and version */ + "level_zero", + /* Component options */ +#if defined(PARSEC_HAVE_PEER_DEVICE_MEMORY_ACCESS) + "+peer_access" +#endif + "", + PARSEC_VERSION_MAJOR, + PARSEC_VERSION_MINOR, + + /* Component open and close functions */ + device_level_zero_component_open, + device_level_zero_component_close, + device_level_zero_component_query, + /*< specific query to return the module and add it to the list of available modules */ + device_level_zero_component_register, + "", /*< no reserve */ + }, + { + /* The component has no metadata */ + MCA_BASE_METADATA_PARAM_NONE, + "", /*< no reserve */ + }, + NULL +}; + +mca_base_component_t * device_level_zero_static_component(void) +{ + return (mca_base_component_t *)&parsec_device_level_zero_component; +} + +static int device_level_zero_component_query(mca_base_module_t **module, int *priority) +{ + int i, j, rc; + ze_result_t ze_rc; + ze_device_handle_t *devices = NULL; + ze_driver_handle_t *allDrivers = NULL; + + *module = NULL; + *priority = 0; + if( 0 == parsec_device_level_zero_enabled ) { + return MCA_SUCCESS; + } + parsec_gpu_init_profiling(); + + if( parsec_device_level_zero_enabled >= 1) { + uint32_t driverCount = 0; + uint32_t totalDeviceCount = 0, maxDeviceCount = 0; + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { return MCA_ERR_NOT_AVAILABLE; } ); + + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { free(allDrivers); return MCA_ERR_NOT_AVAILABLE; } ); + + for(uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + zeDeviceGet(allDrivers[did], &deviceCount, NULL); + totalDeviceCount += deviceCount; + if(maxDeviceCount < deviceCount) + maxDeviceCount = deviceCount; + } + + parsec_device_level_zero_enabled = totalDeviceCount < (uint32_t)parsec_device_level_zero_enabled ? (int)totalDeviceCount : parsec_device_level_zero_enabled; + + if(parsec_device_level_zero_enabled > 0) { + parsec_device_level_zero_component.modules = + (parsec_device_module_t **)calloc(parsec_device_level_zero_enabled + 1, + sizeof(parsec_device_module_t *)); + devices = (ze_device_handle_t *)malloc(maxDeviceCount * sizeof(ze_device_handle_t)); + + i = j = 0; + for(uint32_t did = 0; i < parsec_device_level_zero_enabled && did < driverCount; ++did ) { + uint32_t deviceCount = maxDeviceCount; + parsec_device_level_zero_driver_t *driver = malloc(sizeof(parsec_device_level_zero_driver_t)); + + driver->ze_driver = allDrivers[did]; + driver->ref_count = 0; + driver->swp = NULL; + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_result_t ze_rc = zeContextCreate(driver->ze_driver, &ctxtDesc, &driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { free(driver); continue; } ); + + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, devices); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, { zeContextDestroy(driver->ze_context); free(driver); continue; } ); + + for(uint32_t devid = 0; i < parsec_device_level_zero_enabled && devid < deviceCount; devid++) { + ze_device_properties_t device_properties; + ze_rc = zeDeviceGetProperties(devices[devid], &device_properties); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetProperties ", ze_rc, { continue; } ); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + if( !((1 << i) & level_zero_mask) ) { i++; continue; } + rc = parsec_level_zero_module_init(i, driver, devices[devid], &device_properties, + &parsec_device_level_zero_component.modules[j]); + if( PARSEC_SUCCESS != rc ) { + assert( NULL == parsec_device_level_zero_component.modules[j] ); + continue; + } + driver->ref_count++; + parsec_device_level_zero_component.modules[j]->component = &parsec_device_level_zero_component; + j++; /* next available spot */ + parsec_device_level_zero_component.modules[j] = NULL; + i++; + } + + if( driver->ref_count == 0 ) { + zeContextDestroy(driver->ze_context); + free(driver); + } + } + parsec_device_level_zero_enabled = j; + if(0 == parsec_device_level_zero_enabled) { + free( parsec_device_level_zero_component.modules ); + } + } + } + + if(0 == parsec_device_level_zero_enabled ) + parsec_device_level_zero_component.modules = NULL; + + if(NULL != devices) + free(devices); + if(NULL != allDrivers) + free(allDrivers); + + /* SYCL wrappers must be built once all devices and queues and contexts and drivers are up, + * and they must be built in the following order: driver wrapper, then device wrappers for this + * driver, then context wrapper for this driver, then queue wrappers for each queue. + * Also, it MUST be one to one: a single device wrapper per device, a single context wrapper + * per context etc... */ + for(int did = 0; did < parsec_device_level_zero_enabled; did++) { + if(NULL == parsec_device_level_zero_component.modules[did]) + continue; + parsec_device_level_zero_module_t *module = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did]; + if(NULL != module->driver->swp) + continue; + + module->driver->swp = parsec_sycl_wrapper_platform_create(module->driver->ze_driver); + int nbdev = 1; + for(int i = 1; did + i < parsec_device_level_zero_enabled; i++) { + if( ((parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i])->driver == module->driver ) + nbdev++; + } + parsec_sycl_wrapper_device_t **devices = (parsec_sycl_wrapper_device_t **)malloc(nbdev*sizeof(parsec_sycl_wrapper_device_t*)); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + mod2->swd = parsec_sycl_wrapper_device_create(mod2->ze_device); + devices[j++] = mod2->swd; + } + } + parsec_sycl_wrapper_platform_add_context(module->driver->swp, module->driver->ze_context, devices, nbdev); + free(devices); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + j++; + for(int s = 0; s < mod2->super.max_exec_streams; s++) { + parsec_level_zero_exec_stream_t *exec_stream = (parsec_level_zero_exec_stream_t *)mod2->super.exec_stream[s]; + if(NULL != exec_stream->level_zero_cq) + exec_stream->swq = parsec_sycl_wrapper_queue_create(module->driver->swp, mod2->swd, exec_stream->level_zero_cq); + } + } + } + } + + parsec_gpu_enable_debug(); + + /* module type should be: const mca_base_module_t ** */ + void *ptr = parsec_device_level_zero_component.modules; + *priority = 10; + *module = (mca_base_module_t *)ptr; + + return MCA_SUCCESS; +} + +static int device_level_zero_component_register(void) +{ + parsec_device_level_zero_index = parsec_mca_param_reg_int_name("device_level_zero", "enabled", + "The number of LEVEL_ZERO device to enable for the next PaRSEC context (-1 for all available)", + false, false, -1, &parsec_device_level_zero_enabled); + (void)parsec_mca_param_reg_int_name("device_level_zero", "mask", + "The bitwise mask of LEVEL_ZERO devices to be enabled (default all)", + false, false, 0xffffffff, &level_zero_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "nvlink_mask", + "What devices are allowed to use NVLINK if available (default all)", + false, false, 0xffffffff, &level_zero_nvlink_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "verbose", + "Set the verbosity level of the LEVEL_ZERO device (negative value: use debug verbosity), higher is less verbose)\n", + false, false, -1, &parsec_gpu_verbosity); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_block_size", + "The LEVEL_ZERO memory page for PaRSEC internal management (in bytes).", + false, false, 512*1024, &level_zero_memory_block_size); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_use", + "The percentage of the total GPU memory to be used by this PaRSEC context", + false, false, 95, &level_zero_memory_percentage); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_number_of_blocks", + "Alternative to device_level_zero_memory_use: sets exactly the number of blocks to allocate (-1 means to use a percentage of the available memory)", + false, false, -1, &level_zero_memory_number_of_blocks); + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", + "Sets up the maximum number of blocks that can be ejected from GPU memory", + false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_LEVEL_ZERO_sort_pending_list); +#if defined(PARSEC_PROF_TRACE) + (void)parsec_mca_param_reg_int_name("device_level_zero", "one_profiling_stream_per_level_zero_stream", + "Boolean to separate the profiling of each level_zero stream into a single profiling stream", + false, false, 0, &parsec_device_gpu_one_profiling_stream_per_gpu_stream); +#endif + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_streams", + "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", + false, false, PARSEC_GPU_MAX_STREAMS, &parsec_level_zero_max_streams); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_level_zero_sort_pending); + /* If LEVEL_ZERO was not requested avoid initializing the devices */ + return (0 == parsec_device_level_zero_enabled ? MCA_ERROR : MCA_SUCCESS); +} + +/** + * Open LEVEL_ZERO and check that devices are available and ready to be used. This operation should + * only be done once during the initialization, and the devices should from there on be managed + * by PaRSEC. + */ +static int device_level_zero_component_open(void) +{ + ze_result_t ze_rc; + ze_driver_handle_t *allDrivers = NULL; + int ndevices = 0; + uint32_t driverCount = 0; + + if( 0 == parsec_device_level_zero_enabled ) { + return MCA_ERROR; /* Nothing to do around here */ + } + + ze_rc = zeInit( 0 ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, + { + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + if(driverCount > 0) { + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + + for( uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); + return MCA_ERROR; + } ); + ndevices += (int)deviceCount; + } + + free(allDrivers); + } + + + if( ndevices > parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { + ndevices = parsec_device_level_zero_enabled; + } + } else if (ndevices < parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { + parsec_warning("User requested %d LEVEL_ZERO devices, but only %d are available on %s\n." + " PaRSEC will enable all %d of them.", + parsec_device_level_zero_enabled, ndevices, parsec_hostname, ndevices); + parsec_mca_param_set_int(parsec_device_level_zero_index, ndevices); + } + } + + /* Update the number of GPU for the upper layer */ + parsec_device_level_zero_enabled = ndevices; + if( 0 == ndevices ) { + return -1; + } + + return MCA_SUCCESS; +} + +/** + * Remove all LEVEL_ZERO devices from the PaRSEC available devices, and turn them off. + * At the end of this function all LEVEL_ZERO initialization in the context of PaRSEC + * should be undone, and pending tasks either completed or transferred to another + * chore (if available), and all LEVEL_ZERO resources (fences, streams and memory) + * released. + */ +static int device_level_zero_component_close(void) +{ + parsec_device_level_zero_module_t* cdev; + int i, rc; + + if( NULL == parsec_device_level_zero_component.modules ) { /* No devices */ + return PARSEC_SUCCESS; + } + + for( i = 0; NULL != (cdev = (parsec_device_level_zero_module_t*)parsec_device_level_zero_component.modules[i]); i++ ) { + parsec_device_level_zero_component.modules[i] = NULL; + + rc = parsec_level_zero_module_fini((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to release resources on LEVEL_ZERO device\n", + cdev->level_zero_index); + } + if(0 == --cdev->driver->ref_count) { + ze_result_t ze_rc = zeContextDestroy(cdev->driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextDestroy ", ze_rc, {}); + parsec_sycl_wrapper_platform_destroy(cdev->driver->swp); + } + + /* unregister the device from PaRSEC */ + rc = parsec_mca_device_remove((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to unregister LEVEL_ZERO device %d\n", + cdev->level_zero_index, cdev->level_zero_index); + } + + free(cdev); + } + +#if defined(PARSEC_DEBUG_NOISIER) + /* Check that no LEVEL_ZERO devices are still registered with PaRSEC */ + for(i = 0; i < parsec_mca_device_enabled(); i++) { + if( NULL == (cdev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(i)) ) continue; + if(PARSEC_DEV_LEVEL_ZERO != cdev->super.super.type) continue; + + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] LEVEL_ZERO device still registered with PaRSEC at the end of LEVEL_ZERO finalize.\n" + " Please contact the developers or fill an issue.\n", + cdev->level_zero_index); + } +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + if( parsec_device_output != parsec_gpu_output_stream ) + parsec_output_close(parsec_gpu_output_stream); + parsec_gpu_output_stream = parsec_device_output; + + if ( level_zero_lib_path ) { + free(level_zero_lib_path); + } + + return PARSEC_SUCCESS; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h new file mode 100644 index 000000000..ed3a77998 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h @@ -0,0 +1,42 @@ +#ifndef PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H +#define PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H + +typedef struct parsec_sycl_wrapper_platform_s parsec_sycl_wrapper_platform_t; +typedef struct parsec_sycl_wrapper_device_s parsec_sycl_wrapper_device_t; +typedef struct parsec_sycl_wrapper_queue_s parsec_sycl_wrapper_queue_t; + +#include + +#if defined(c_plusplus) || defined(__cplusplus) +#include + +struct parsec_sycl_wrapper_platform_s { + sycl::platform platform; + sycl::context context; +}; + +struct parsec_sycl_wrapper_device_s { + sycl::device device; +}; + +struct parsec_sycl_wrapper_queue_s { + sycl::queue queue; +}; + +extern "C" { +#endif + +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device); +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device); +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp); +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd); +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp new file mode 100644 index 000000000..bafd03cca --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp @@ -0,0 +1,59 @@ +#include +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" + +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + parsec_sycl_wrapper_platform_t *res = new parsec_sycl_wrapper_platform_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device) +{ + std::vectordevices; + + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + swp->context = sycl::make_context(hContextInteropInput); +} + +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + parsec_sycl_wrapper_device_t *res = new parsec_sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); + + return res; +} + +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) +{ + parsec_sycl_wrapper_queue_t *swq = new parsec_sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} + +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp) +{ + if(nullptr != swp) + delete swp; + return 0; +} + +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd) +{ + if(nullptr != swd) + delete swd; + return 0; +} + +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq) +{ + if(nullptr != swq) + delete swq; + return 0; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_internal.h b/parsec/mca/device/level_zero/device_level_zero_internal.h new file mode 100644 index 000000000..b2bd8de42 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_internal.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED + +#include "parsec/mca/device/level_zero/device_level_zero.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) + +BEGIN_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN 0x0001 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT 0x0002 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN 0x0004 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC 0x0008 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE 0x0010 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH 0x0020 + +extern int parsec_level_zero_trackable_events; +extern int parsec_level_zero_movein_key_start; +extern int parsec_level_zero_movein_key_end; +extern int parsec_level_zero_moveout_key_start; +extern int parsec_level_zero_moveout_key_end; +extern int parsec_level_zero_own_GPU_key_start; +extern int parsec_level_zero_own_GPU_key_end; +extern int parsec_level_zero_allocate_memory_key; +extern int parsec_level_zero_free_memory_key; +extern int parsec_level_zero_use_memory_key_start; +extern int parsec_level_zero_use_memory_key_end; +extern int parsec_level_zero_prefetch_key_start; +extern int parsec_level_zero_prefetch_key_end; +extern int parsec_device_level_zero_one_profiling_stream_per_level_zero_stream; +#endif /* defined(PROFILING) */ + +#define GPU_TASK_TYPE_KERNEL 0x0000 +#define GPU_TASK_TYPE_D2HTRANSFER 0x1000 +#define GPU_TASK_TYPE_PREFETCH 0x2000 +#define GPU_TASK_TYPE_WARMUP 0x4000 +#define GPU_TASK_TYPE_D2D_COMPLETE 0x8000 + +/* From MCA parameters */ +extern int use_level_zero_index, use_level_zero; +extern int level_zero_mask, level_zero_verbosity; +extern int parsec_level_zero_sort_pending, parsec_level_zero_max_streams; +extern int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; +extern char* level_zero_lib_path; +extern int32_t parsec_LEVEL_ZERO_d2h_max_flows; +extern int32_t parsec_LEVEL_ZERO_sort_pending_list; + +PARSEC_DECLSPEC extern const parsec_device_module_t parsec_device_level_zero_module; + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ + +int parsec_level_zero_module_init( int device_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, + ze_device_properties_t *device_properties, parsec_device_module_t** module ); +int parsec_level_zero_module_fini(parsec_device_module_t* device); + +END_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +typedef struct { + uint64_t size; + uint64_t data_key; + uint64_t dc_id; +} parsec_device_level_zero_memory_prof_info_t; +#define PARSEC_DEVICE_LEVEL_ZERO_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" +#endif /* PARSEC_PROF_TRACE */ + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c new file mode 100644 index 000000000..db2e461ee --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -0,0 +1,2678 @@ +/* + * Copyright (c) 2021-2022 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" + +#include + +static int parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice); +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* gpu_device, + int memory_percentage, + int number_of_elements, + size_t eltsize ); +static int parsec_level_zero_memory_release( parsec_device_level_zero_module_t* gpu_device ); +static int parsec_level_zero_flush_lru( parsec_device_module_t *device ); + +static void* level_zero_find_incarnation(parsec_device_level_zero_module_t* level_zero_device, + const char* fname) +{ + char library_name[FILENAME_MAX], *env; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + void *fn = NULL; + char** argv = NULL; + + /** + * Prepare the list of PATH or FILE to be searched for a LEVEL_ZERO shared library. + * In any case this list might be a list of ; separated possible targets, + * where each target can be either a directory or a specific file. + */ + env = getenv("PARSEC_CUCORES_LIB"); + if( NULL != env ) { + argv = parsec_argv_split(env, ';'); + } else if( NULL != level_zero_lib_path ) { + argv = parsec_argv_split(level_zero_lib_path, ';'); + } + + fn = parsec_device_find_function(fname, library_name, (const char**)argv); + if( NULL == fn ) { /* look for the function with lesser capabilities */ + parsec_warning("No function '%s' found for LEVEL_ZERO device %s", fname, gpu_device->super.name); + } + + if( NULL != argv ) + parsec_argv_free(argv); + + return fn; +} + +/** + * Register a taskpool with a device by checking that the device + * supports the dynamic function required by the different incarnations. + * If multiple devices of the same type exists we assume thay all have + * the same capabilities. + */ +static int +parsec_level_zero_taskpool_register(parsec_device_module_t* device, + parsec_taskpool_t* tp) +{ + parsec_device_level_zero_module_t* gpu_device = (parsec_device_level_zero_module_t*)device; + int32_t rc = PARSEC_ERR_NOT_FOUND; + uint32_t i, j; + + /** + * Detect if a particular chore has a dynamic load dependency and if yes + * load the corresponding module and find the function. + */ + assert(PARSEC_DEV_LEVEL_ZERO == device->type); + assert(tp->devices_index_mask & (1 << device->device_index)); + + for( i = 0; i < tp->nb_task_classes; i++ ) { + const parsec_task_class_t* tc = tp->task_classes_array[i]; + __parsec_chore_t* chores = (__parsec_chore_t*)tc->incarnations; + for( j = 0; NULL != chores[j].hook; j++ ) { + if( chores[j].type != device->type ) + continue; + if( NULL != chores[j].dyld_fn ) { + /* the function has been set for another device of the same type */ + return PARSEC_SUCCESS; + } + if ( NULL == chores[j].dyld ) { + chores[j].dyld_fn = NULL; /* No dynamic support required for this kernel */ + rc = PARSEC_SUCCESS; + } else { + void* devf = level_zero_find_incarnation(gpu_device, chores[j].dyld); + if( NULL != devf ) { + chores[j].dyld_fn = devf; + rc = PARSEC_SUCCESS; + } + } + } + } + if( PARSEC_SUCCESS != rc ) { + tp->devices_index_mask &= ~(1 << device->device_index); /* drop support for this device */ + parsec_debug_verbose(10, parsec_gpu_output_stream, + "Device %d (%s) disabled for taskpool %p", device->device_index, device->name, tp); + } + return rc; +} + +static int +parsec_level_zero_taskpool_unregister(parsec_device_module_t* device, parsec_taskpool_t* tp) +{ + (void)device; (void)tp; + return PARSEC_SUCCESS; +} + +/** + * Attach a device to a PaRSEC context. A device can only be attached to + * a single context at the time. + */ +static int +parsec_device_level_zero_attach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + return parsec_mca_device_add(context, device); +} + +/** + * Detach a device from a context. Both the context and the device remain + * valid, they are simply disconnected. + * This function should only be called once all tasks and all data related to the + * context has been removed from the device. + */ +static int +parsec_device_level_zero_detach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + (void)context; + return parsec_mca_device_remove(device); +} + +int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, + ze_device_properties_t *prop, parsec_device_module_t** module ) +{ + int streaming_multiprocessor, len; + parsec_device_level_zero_module_t* level_zero_device; + parsec_device_gpu_module_t* gpu_device; + parsec_device_module_t* device; + ze_result_t ze_rc; + int show_caps_index, show_caps = 0, j, k; + char *szName; + float clockRate; + + show_caps_index = parsec_mca_param_find("device", NULL, "show_capabilities"); + if(0 < show_caps_index) { + parsec_mca_param_lookup_int(show_caps_index, &show_caps); + } + + *module = NULL; + + szName = prop->name; + clockRate = prop->coreClockRate/1e3f; + streaming_multiprocessor = prop->numThreadsPerEU; + + level_zero_device = (parsec_device_level_zero_module_t*)calloc(1, sizeof(parsec_device_level_zero_module_t)); + gpu_device = &level_zero_device->super; + device = &gpu_device->super; + PARSEC_OBJ_CONSTRUCT(level_zero_device, parsec_device_level_zero_module_t); + level_zero_device->level_zero_index = (uint8_t)dev_id; + level_zero_device->driver = driver; + level_zero_device->ze_device = ze_device; + + len = asprintf(&gpu_device->super.name, "%s ZE(%d)", szName, dev_id); + if(-1 == len) + gpu_device->super.name = ""; + gpu_device->data_avail_epoch = 0; + + gpu_device->max_exec_streams = parsec_level_zero_max_streams; + gpu_device->exec_stream = + (parsec_gpu_exec_stream_t**)malloc(gpu_device->max_exec_streams * sizeof(parsec_gpu_exec_stream_t*)); + gpu_device->exec_stream[0] = + (parsec_gpu_exec_stream_t*)malloc( gpu_device->max_exec_streams * sizeof(parsec_level_zero_exec_stream_t)); + for(j = 1; j < gpu_device->max_exec_streams; j++) + gpu_device->exec_stream[j] = (parsec_gpu_exec_stream_t*)( + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[0]+j); + + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, NULL); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, cmdqueueGroupProperties); + // Find a command queue type that support compute + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; + i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); + ++i ) { + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + computeQueueGroupOrdinal = i; + } + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + copyQueueGroupOrdinal = i; + } + } + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + /* CommandListImmediate would seem better for I/O, but mixing CLImmediate and CL+CQ + * seems to create some synchronization issues. */ + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandQueueDesc, &level_zero_stream->level_zero_cq); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, {goto release_device;} ); + exec_stream->workspace = NULL; + PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); + parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); + exec_stream->max_events = PARSEC_MAX_EVENTS_PER_STREAM; + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + exec_stream->name = NULL; + exec_stream->fifo_pending = (parsec_list_t*)PARSEC_OBJ_NEW(parsec_list_t); + PARSEC_OBJ_CONSTRUCT(exec_stream->fifo_pending, parsec_list_t); + exec_stream->tasks = (parsec_gpu_task_t**)malloc(exec_stream->max_events + * sizeof(parsec_gpu_task_t*)); + level_zero_stream->fences = (ze_fence_handle_t*)malloc(exec_stream->max_events * sizeof(ze_fence_handle_t)); + level_zero_stream->command_lists = (ze_command_list_handle_t*)malloc(exec_stream->max_events*sizeof(ze_command_list_handle_t)); + + /* create the fences and command lists */ + for( k = 0; k < exec_stream->max_events; k++ ) { + ze_command_list_desc_t commandListDesc = { + ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, + NULL, + computeQueueGroupOrdinal, + 0 // flags + }; + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 + }; + level_zero_stream->fences[k] = NULL; + exec_stream->tasks[k] = NULL; + ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandListDesc, &level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); + ze_rc = zeFenceCreate(level_zero_stream->level_zero_cq, &fence_desc, &(level_zero_stream->fences[k])); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, {goto release_device;} ); + } + if(j == 0) { + len = asprintf(&exec_stream->name, "h2d(%d)", j); + if(-1 == len) + exec_stream->name = "h2d"; + } else if(j == 1) { + len = asprintf(&exec_stream->name, "d2h(%d)", j); + if(-1 == len) + exec_stream->name = "d2h"; + } else { + len = asprintf(&exec_stream->name, "level_zero(%d)", j); + if(-1 == len) + exec_stream->name = "level_zero"; + } +#if defined(PARSEC_PROF_TRACE) + /* Each 'exec' stream gets its own profiling stream, except IN and OUT stream that share it. + * It's good to separate the exec streams to know what was submitted to what stream + * We don't have this issue for the IN and OUT streams because types of event discriminate + * what happens where, and separating them consumes memory and increases the number of + * events that needs to be matched between streams because we cannot differentiate some + * ends between IN or OUT, so they are all logged on the same stream. */ + if(j == 0 || (parsec_device_gpu_one_profiling_stream_per_gpu_stream == 1 && j != 1)) + exec_stream->profiling = parsec_profiling_stream_init( 2*1024*1024, PARSEC_PROFILE_STREAM_STR, dev_id, j ); + else + exec_stream->profiling = gpu_device->exec_stream[0]->profiling; + if(j == 0) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else if(j == 1) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + + device->type = PARSEC_DEV_LEVEL_ZERO; + device->executed_tasks = 0; + device->data_in_array_size = 0; // We'll let the modules_attach allocate the array of the right size for us + device->data_in_from_device = NULL; + device->data_out_to_host = 0; + device->required_data_in = 0; + device->required_data_out = 0; + + device->attach = parsec_device_level_zero_attach; + device->detach = parsec_device_level_zero_detach; + device->memory_register = NULL; // TODO: could not find a memory register equivalent in level zero + device->memory_unregister = NULL; // TODO: could not find a memory unregister equivalent in level zero + device->taskpool_register = parsec_level_zero_taskpool_register; + device->taskpool_unregister = parsec_level_zero_taskpool_unregister; + device->data_advise = parsec_level_zero_data_advise; + device->memory_release = parsec_level_zero_flush_lru; + + device->device_hweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_tweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_sweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_dweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + + /* Initialize internal lists */ + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_owned_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->pending, parsec_fifo_t); + + gpu_device->sort_starting_p = NULL; + gpu_device->peer_access_mask = 0; /* No GPU to GPU direct transfer by default */ + + if( PARSEC_SUCCESS != parsec_level_zero_memory_reserve(level_zero_device, + level_zero_memory_percentage, + level_zero_memory_number_of_blocks, + level_zero_memory_block_size) ) { + goto release_device; + } + + if( show_caps ) { + parsec_inform("LEVEL ZERO GPU Device %d: %s\n" + "\tLocation (PCI Bus/Device/Domain): %x:%x.%x\n" + "\tnumThreadsPerEU : %u\n" + "\tphysicalEUSimdWidth : %u\n" + "\tnumEUsPerSubslice : %u\n" + "\tnumSubslicesPerSlice: %u\n" + "\tnumSlices: %u\n" + "\tcoreClockRate (GHz): %2.2f\n" + "\tpeak Gflops : double %2.3f, single %2.3f tensor %2.3f half %2.3f\n", + level_zero_device->level_zero_index, device->name, + prop->deviceId, prop->subdeviceId, prop->vendorId, + streaming_multiprocessor, + prop->physicalEUSimdWidth, + prop->numEUsPerSubslice, + prop->numSubslicesPerSlice, + prop->numSlices, + clockRate*1e-3, + device->device_dweight, device->device_sweight, device->device_tweight, device->device_hweight); + } + + *module = device; + return PARSEC_SUCCESS; + + release_device: + if( NULL != gpu_device->exec_stream) { + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t *level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + if( NULL != exec_stream->fifo_pending ) { + PARSEC_OBJ_RELEASE(exec_stream->fifo_pending); + } + if( NULL != exec_stream->tasks ) { + free(exec_stream->tasks); exec_stream->tasks = NULL; + } + if( NULL != level_zero_stream->fences ) { + for( k = 0; k < exec_stream->max_events; k++ ) { + if( NULL != level_zero_stream->fences[k] ) { + (void)zeFenceDestroy(level_zero_stream->fences[k]); + } + } + free(level_zero_stream->fences); level_zero_stream->fences = NULL; + } + if( NULL != exec_stream->name ) { + free(exec_stream->name); exec_stream->name = NULL; + } +#if defined(PARSEC_PROF_TRACE) + if( NULL != exec_stream->profiling ) { + /* No function to clean the profiling stream. If one is introduced + * some day, remember that exec streams 0 and 1 always share the same + * ->profiling stream, and that all of them share the same + * ->profiling stream if parsec_device_gpu_one_profiling_stream_per_gpu_stream == 0 */ + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + } + free(gpu_device); + return PARSEC_ERROR; +} + +int +parsec_level_zero_module_fini(parsec_device_module_t* device) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t* level_zero_device = (parsec_device_level_zero_module_t*)device; + ze_result_t status; + int j, k; + + /* Release the registered memory */ + parsec_level_zero_memory_release(level_zero_device); + + /* Release pending queue */ + PARSEC_OBJ_DESTRUCT(&gpu_device->pending); + + /* Release all streams */ + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + + for( k = 0; k < exec_stream->max_events; k++ ) { + assert( NULL == exec_stream->tasks[k] ); + status = zeFenceDestroy(level_zero_stream->fences[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) zeFenceDestroy ", status, {} ); + status = zeCommandListDestroy(level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", status, {} ); + } + exec_stream->max_events = 0; + free(level_zero_stream->fences); level_zero_stream->fences = NULL; + free(exec_stream->tasks); exec_stream->tasks = NULL; + free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; + /* Deleting the sycl queue wrapper and/or the command queue conflicts with the + * cleaning procedure of the Level Zero runtime... Didn't find a way to do it + * cleanly. Don't cleanup for now... */ + //parsec_sycl_wrapper_queue_destroy(level_zero_stream->swq); + level_zero_stream->swq = NULL; + free(exec_stream->name); + + /* Release Info object array */ + PARSEC_OBJ_DESTRUCT(&exec_stream->infos); + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + level_zero_device->level_zero_index = -1; + parsec_sycl_wrapper_device_destroy(level_zero_device->swd); + level_zero_device->swd = NULL; + + /* Cleanup the GPU memory. */ + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_lru); + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_owned_lru); + + return PARSEC_SUCCESS; +} + +/** + * This function reserve the memory_percentage of the total device memory for PaRSEC. + * This memory will be managed in chuncks of size eltsize. However, multiple chuncks + * can be reserved in a single allocation. + */ +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_device, + int memory_percentage, + int number_blocks, + size_t eltsize ) +{ + ze_result_t status; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + (void)eltsize; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + uint32_t count = 0; + + size_t how_much_we_allocate; + size_t initial_free_mem, alignment = 1 << 3; + int memIndex = -1; + uint32_t mem_elem_per_gpu = 0; + + status = zeDeviceGetMemoryAccessProperties(level_zero_device->ze_device, &memAccessProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { + return PARSEC_ERROR; + }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + parsec_warning("%s:%d -- Device %s does not have memory allocation capabilities with RW access\n", + __FILE__, __LINE__, gpu_device->super.name); + return PARSEC_ERROR; + } + status = zeDeviceGetProperties(level_zero_device->ze_device, &devProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { + return PARSEC_ERROR; + }); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + return PARSEC_ERROR; + }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + free(devMemProperties); + return PARSEC_ERROR; + }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + initial_free_mem = devProperties.maxMemAllocSize < devMemProperties[memIndex].totalSize ? + devProperties.maxMemAllocSize : devMemProperties[memIndex].totalSize; + free(devMemProperties); devMemProperties = NULL; + + if( number_blocks != -1 ) { + if( number_blocks == 0 ) { + parsec_warning("LEVEL_ZERO[%d] Invalid argument: requesting 0 bytes of memory on LEVEL_ZERO device %s", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } else { + how_much_we_allocate = number_blocks * eltsize; + } + } else { + /** number_blocks == -1 means memory_percentage is used */ + how_much_we_allocate = (memory_percentage * initial_free_mem) / 100; + } + if( how_much_we_allocate > initial_free_mem ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + parsec_warning("LEVEL_ZERO[%d] Requested %zd bytes on LEVEL_ZERO device %s, but only %zd bytes are available -- reducing allocation to max available", + level_zero_device->level_zero_index, how_much_we_allocate, gpu_device->super.name, initial_free_mem); + how_much_we_allocate = initial_free_mem; + } + if( how_much_we_allocate < eltsize ) { + /** Handle another kind of jokers entirely, and cases of + * not enough memory on the device + */ + parsec_warning("LEVEL_ZERO[%d] Cannot allocate at least one element on LEVEL_ZERO device %s", + level_zero_device->level_zero_index, gpu_device->super.name); + return PARSEC_ERROR; + } + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + size_t free_mem = initial_free_mem; + /* + * We allocate a bunch of tiles that will be used + * during the computations + */ + while( (free_mem > eltsize ) + && ((total_mem - free_mem) < how_much_we_allocate) ) { + parsec_gpu_data_copy_t* gpu_elem; + void *device_ptr; + + status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, eltsize, alignment, + level_zero_device->ze_device, &device_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { break; } ); + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate LEVEL_ZERO copy %p [ref_count %d] for data [%p]", + gpu_device->super.name,gpu_elem, gpu_elem->super.obj_reference_count, NULL); + gpu_elem->device_private = (void*)(long)device_ptr; + gpu_elem->flags |= PARSEC_DATA_FLAG_PARSEC_OWNED; + gpu_elem->device_index = gpu_device->super.device_index; + mem_elem_per_gpu++; + PARSEC_OBJ_RETAIN(gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, gpu_elem, gpu_elem->super.obj_reference_count); + parsec_list_push_back( &gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem ); + free_mem -= alignment * ((size + alignment - 1) / alignment); + } + if( 0 == mem_elem_per_gpu && parsec_list_is_empty( &gpu_device->gpu_mem_lru ) ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", gpu_device->super.name, gpu_device->super.name); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu ); + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", gpu_device->super.name, mem_elem_per_gpu); +#else + if( NULL == gpu_device->memory ) { + void* base_ptr; + /* We allocate all the memory on the GPU and we use our memory management. */ + /* This computation leads to allocating more than available if we asked for more than GPU memory */ + mem_elem_per_gpu = (how_much_we_allocate + eltsize - 1 ) / eltsize; + size_t total_size = (size_t)mem_elem_per_gpu * eltsize; + + if (total_size > initial_free_mem) { + /* Mapping more than 100% of GPU memory is obviously wrong */ + /* Mapping exactly 100% of the GPU memory ends up producing errors about __global__ function call is not configured */ + /* Mapping 95% works with low-end GPUs like 1060, how much to let available for level_zero runtime, I don't know how to calculate */ + total_size = (size_t)((int)(.9*initial_free_mem / eltsize)) * eltsize; + mem_elem_per_gpu = total_size / eltsize; + } + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, total_size, alignment, + level_zero_device->ze_device, &base_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, + ({ parsec_warning("GPU[%s] Allocating %zu bytes of memory on the GPU device failed", + gpu_device->super.name, total_size); }) ); + + gpu_device->memory = zone_malloc_init( base_ptr, mem_elem_per_gpu, eltsize ); + + if( gpu_device->memory == NULL ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u segments of size %d on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu, eltsize ); + } +#endif + gpu_device->mem_block_size = eltsize; + gpu_device->mem_nb_blocks = mem_elem_per_gpu; + + return PARSEC_SUCCESS; +} + +static void parsec_level_zero_memory_release_list(parsec_device_level_zero_module_t* level_zero_device, + parsec_list_t* list) +{ + parsec_list_item_t* item; +#if defined(PARSEC_DEBUG_VERBOSE) + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; +#endif + + while(NULL != (item = parsec_list_pop_front(list)) ) { + parsec_gpu_data_copy_t* gpu_copy = (parsec_gpu_data_copy_t*)item; + parsec_data_t* original = gpu_copy->original; + + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p, in map %p", + gpu_device->super.name, gpu_copy, gpu_copy->device_private, gpu_copy->super.super + .obj_reference_count, + original, (NULL != original ? original->dc : NULL)); + assert( gpu_copy->device_index == level_zero_device->super.super.device_index ); + + if( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ) { + parsec_warning("GPU[%s] still OWNS the master memory copy for data %d and it is discarding it!", + gpu_device->super.name, original->key); + } + assert(0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + level_zeroFree( gpu_copy->device_private ); +#else + +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)gpu_copy->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)gpu_copy->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + zone_free( level_zero_device->super.memory, (void*)gpu_copy->device_private ); +#endif + gpu_copy->device_private = NULL; + + /* At this point the data copies should have no attachement to a data_t. Thus, + * before we get here (aka below parsec_fini), the destructor of the data + * collection must have been called, releasing all the copies. + */ + PARSEC_OBJ_RELEASE(gpu_copy); assert(NULL == gpu_copy); + } +} + +/** + * This function only flushes the data copies pending in LRU, and checks + * (in debug mode) that the entire allocated memory is free to use */ +static int +parsec_level_zero_flush_lru( parsec_device_module_t *device ) +{ + size_t in_use; + parsec_device_gpu_module_t *gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)device; + /* Free all memory on GPU */ + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_lru); + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_owned_lru); +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) && !defined(_NDEBUG) + if( (in_use = zone_in_use(gpu_device->memory)) != 0 ) { + parsec_warning("GPU[%s] memory leak detected: %lu bytes still allocated on GPU", + device->name, in_use); + assert(0); + } +#endif + return PARSEC_SUCCESS; +} + +/** + * This function release the LEVEL_ZERO memory reserved for this device. + * + * One has to notice that all the data available on the GPU is stored in one of + * the two used to keep track of the allocated data, either the gpu_mem_lru or + * the gpu_mem_owner_lru. Thus, going over all the elements in these two lists + * should be enough to enforce a clean release. + */ +static int +parsec_level_zero_memory_release( parsec_device_level_zero_module_t* level_zero_device ) +{ + ze_result_t status; + + parsec_level_zero_flush_lru(&level_zero_device->super.super); + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + assert( NULL != level_zero_device->super.memory ); + void* ptr = zone_malloc_fini(&level_zero_device->super.memory); + status = zeMemFree(level_zero_device->driver->ze_context, ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, + { parsec_warning("Failed to free the GPU backend memory."); } ); +#endif + + return PARSEC_SUCCESS; +} + +/** + * Try to find memory space to move all data on the GPU. We attach a device_elem to + * a memory_elem as soon as a device_elem is available. If we fail to find enough + * available elements, we push all the elements handled during this allocation + * back into the pool of available device_elem, to be picked up by another call + * (this call will remove them from the current task). + * Returns: + * PARSEC_HOOK_RETURN_DONE: All gpu_mem/mem_elem have been initialized + * PARSEC_HOOK_RETURN_AGAIN: At least one flow is marked under transfer, task cannot be scheduled yet + * PARSEC_HOOK_RETURN_NEXT: The task needs to rescheduled + */ +static inline int +parsec_gpu_data_reserve_device_space( parsec_device_level_zero_module_t* level_zero_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t* temp_loc[MAX_PARAM_COUNT], *gpu_elem, *lru_gpu_elem; + parsec_data_t* master, *oldmaster; + const parsec_flow_t *flow; + int i, j, data_avail_epoch = 0; + parsec_gpu_data_copy_t *gpu_mem_lru_cycling; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + +#if defined(PARSEC_DEBUG_NOISIER) + char task_name[MAX_TASK_STRLEN]; + parsec_task_snprintf(task_name, MAX_TASK_STRLEN, this_task); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + /** + * Parse all the input and output flows of data and ensure all have + * corresponding data on the GPU available. + */ + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + gpu_mem_lru_cycling = NULL; + flow = gpu_task->flow[i]; + assert( flow && (flow->flow_index == i) ); + + /* Skip CTL flows only */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Investigating flow %s:%d", + gpu_device->super.name, task_name, flow->name, i); + temp_loc[i] = NULL; + if (this_task->data[i].data_in == NULL) + continue; + + master = this_task->data[i].data_in->original; + parsec_atomic_lock(&master->lock); + gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + + /* There is already a copy on the device */ + if( NULL != gpu_elem ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Flow %s:%i has a copy on the device %p%s", + gpu_device->super.name, task_name, + flow->name, i, gpu_elem, + gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ? " [in transfer]" : ""); + if ( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Copy %p [ref_count %d] is still in transfer, descheduling...", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + SET_HIGHEST_PRIORITY(gpu_task->ec, parsec_execution_context_priority_comparator); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_AGAIN; + } + parsec_atomic_unlock(&master->lock); + continue; + } + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Allocate LEVEL_ZERO copy %p sz %d[ref_count %d] for data %p", + gpu_device->super.name, task_name, + gpu_elem, gpu_task->flow_nb_elts[i], gpu_elem->super.super.obj_reference_count, master); + gpu_elem->flags = PARSEC_DATA_FLAG_PARSEC_OWNED | PARSEC_DATA_FLAG_PARSEC_MANAGED; + malloc_data: + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + gpu_elem->device_private = zone_malloc(gpu_device->memory, gpu_task->flow_nb_elts[i]); + if( NULL == gpu_elem->device_private ) { +#endif + + find_another_data: + /* Look for a data_copy to free */ + lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru); + if( NULL == lru_gpu_elem ) { + /* We can't find enough room on the GPU. Insert the tiles in the begining of + * the LRU (in order to be reused asap) and return without scheduling the task. + */ +#if defined(PARSEC_DEBUG_NOISIER) + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]:%s:\tRequest space on GPU failed for flow %s index %d/%d for task %s", + gpu_device->super.name, task_name, + flow->name, i, this_task->task_class->nb_flows, task_name ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + for( j = 0; j < i; j++ ) { + /* This flow could be a control flow */ + if( NULL == temp_loc[j] ) continue; + /* This flow could be non-parsec-owned, in which case we can't reclaim it */ + if( 0 == (temp_loc[j]->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\tAdd copy %p [ref_count %d] back to the LRU list", + gpu_device->super.name, task_name, + temp_loc[j], temp_loc[j]->super.super.obj_reference_count); + /* push them at the head to reach them again at the next iteration */ + parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)temp_loc[j]); + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + PARSEC_OBJ_RELEASE(gpu_elem); +#endif + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } + + PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Evaluate LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, + lru_gpu_elem->original); + + /* If there are pending readers, let the gpu_elem loose. This is a weak coordination + * protocol between here and the parsec_gpu_data_stage_in, where the readers don't necessarily + * always remove the data from the LRU. + */ + if( 0 != lru_gpu_elem->readers ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + goto find_another_data; // TODO: add an assert of some sort to check for leaks here? + } + /* It's also possible that the ref_count of that element is bigger than 1 + * In that case, it's because some task completion did not execute yet, and + * we need to keep it in the list until it reaches 1. + */ + if( lru_gpu_elem->super.super.obj_reference_count > 1 ) { + /* It's also possible (although unlikely) that we livelock here: + * if gpu_mem_lru has *only* elements with readers == 0 but + * ref_count > 1, then we might pop/push forever, and we need + * to make progress on something else to get ref_count == 1 && readers == 0. + * So, we return that there is no more free to do. + * To detect that, we use the first one we push back: if we see it + * again, we're cycling. */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Push back LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + + /* Make sure the new GPU element is clean and ready to be used */ + assert( master != lru_gpu_elem->original ); + if ( NULL != lru_gpu_elem->original ) { + /* Let's check we're not trying to steal one of our own data */ + oldmaster = lru_gpu_elem->original; + if( !parsec_atomic_trylock( &oldmaster->lock ) ) { + /* Even if we have the lock on oldmaster, any other thread + * might be adding/removing other elements to the list, so we + * need to protect all accesses to gpu_mem_lru with the locked version */ + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + for( j = 0; j < i; j++ ) { + if( NULL == this_task->data[j].data_in ) continue; + if( this_task->data[j].data_in->original == oldmaster ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] already in use by same task %d:%d original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, i, j, lru_gpu_elem->original); + /* If we are the owner of this tile we need to make sure it remains available for + * other tasks or we run in deadlock situations. + */ + if( temp_loc[j] != lru_gpu_elem ) + temp_loc[j] = lru_gpu_elem; +#if defined(PARSEC_DEBUG_NOISIER) + /* Make sure the data copy is indeed referenced from the current task */ + for( j = 0; j < i; j++ ) { + if( lru_gpu_elem == temp_loc[j] ) break; + } + assert( j < i ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + } + if( lru_gpu_elem->readers != 0 ) { + /* Damn, another thread started to use this data. */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + int do_unlock = oldmaster->super.obj_reference_count != 1; + parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index); + if( do_unlock ) + parsec_atomic_unlock( &oldmaster->lock ); + assert(lru_gpu_elem->readers == 0); + /* The data is not used, it's not one of ours, and it has been detached from the device + * so no other device can use it as a source for their copy : we can free it or reuse it */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d repurpose copy %p [ref_count %d] to data %p instead of %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem, + lru_gpu_elem->super.super.obj_reference_count, master, oldmaster); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d found detached memory from previously destructed data %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem); + oldmaster = NULL; + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + /* Let's free this space, and try again to malloc some space */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p", + gpu_device->super.name, + lru_gpu_elem, lru_gpu_elem->device_private, lru_gpu_elem->super.super.obj_reference_count, + oldmaster); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + zone_free( gpu_device->memory, (void*)(lru_gpu_elem->device_private) ); + lru_gpu_elem->device_private = NULL; + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:%s: Release LRU-retrieved LEVEL_ZERO copy %p [ref_count %d: must be 1]", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count); + PARSEC_OBJ_RELEASE(lru_gpu_elem); + assert( NULL == lru_gpu_elem ); + goto malloc_data; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Succeeded Allocating LEVEL_ZERO copy %p at real address %p [ref_count %d] for data %p", + gpu_device->super.name, + gpu_elem, gpu_elem->device_private, gpu_elem->super.super.obj_reference_count, master); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_allocate_memory_key, (int64_t)gpu_elem->device_private, + gpu_device->super.device_index, + &gpu_task->flow_nb_elts[i], PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif +#else + gpu_elem = lru_gpu_elem; +#endif + assert( 0 == gpu_elem->readers ); + gpu_elem->coherency_state = PARSEC_DATA_COHERENCY_INVALID; + gpu_elem->version = 0; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets created with version 0 at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + parsec_data_copy_attach(master, gpu_elem, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + /* set the new datacopy type to the correct one */ + this_task->data[i].data_out->dtt = this_task->data[i].data_in->dtt; + temp_loc[i] = gpu_elem; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem); + parsec_atomic_unlock(&master->lock); + } + if( data_avail_epoch ) { + gpu_device->data_avail_epoch++; + } + return PARSEC_HOOK_RETURN_DONE; +} + +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_in; + copy_out = task->data[i].data_out; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? + copy_in->original->nb_elts : copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +int +parsec_default_dpcpp_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + return parsec_default_level_zero_stage_in(gtask, flow_mask, gpu_stream); +} + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_out; + copy_out = copy_in->original->device_copies[0]; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? copy_in->original->nb_elts : + copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +int +parsec_default_dpcpp_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + return parsec_default_level_zero_stage_out(gtask, flow_mask, gpu_stream); +} + +/** + * If the most current version of the data is not yet available on the GPU memory + * schedule a transfer. + * Returns: + * 0: The most recent version of the data is already available on the GPU + * 1: A copy has been scheduled on the corresponding stream + * -1: A copy cannot be issued due to LEVEL_ZERO. + */ +static inline int +parsec_gpu_data_stage_in( parsec_device_level_zero_module_t* level_zero_device, + const parsec_flow_t *flow, + parsec_data_pair_t* task_data, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream ) +{ + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + int32_t type = flow->flow_flags; + parsec_data_copy_t* in_elem = task_data->data_in; + parsec_data_copy_t* release_after_data_in_is_attached = NULL; + parsec_data_t* original = in_elem->original; + parsec_gpu_data_copy_t* gpu_elem = task_data->data_out; + uint32_t nb_elts = gpu_task->flow_nb_elts[flow->flow_index]; + int transfer_from = -1; + int undo_readers_inc_if_no_transfer = 0; + + if( gpu_task->task_type == GPU_TASK_TYPE_PREFETCH ) { + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: Prefetch task %p is staging in", + gpu_device->super.name, gpu_task); + } + + parsec_atomic_lock( &original->lock ); + + /** + * If the data will be accessed in write mode, remove it from any GPU data management + * lists until the task is completed. + */ + if( PARSEC_FLOW_ACCESS_WRITE & type ) { + if (gpu_elem->readers > 0 ) { + if( !((1 == gpu_elem->readers) && (PARSEC_FLOW_ACCESS_READ & type)) ) { + parsec_warning("GPU[%s]:\tWrite access to data copy %p [ref_count %d] with existing readers [%d] " + "(possible anti-dependency,\n" + "or concurrent accesses), please prevent that with CTL dependencies\n", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->readers); + parsec_atomic_unlock( &original->lock ); + return -1; + } + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tDetach writable LEVEL_ZERO copy %p [ref_count %d] from any lists", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count); + /* make sure the element is not in any tracking lists */ + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_elem); + PARSEC_LIST_ITEM_SINGLETON(gpu_elem); + } + + /* Detect if we can do a device to device copy. + * Current limitations: only for read-only data used read-only on the hosting GPU. */ + parsec_device_level_zero_module_t *in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get( in_elem->device_index ); + if( (PARSEC_FLOW_ACCESS_READ & type) && !(PARSEC_FLOW_ACCESS_WRITE & type) ) { + int potential_alt_src = 0; + if( PARSEC_DEV_LEVEL_ZERO == in_elem_dev->super.super.type ) { + if( gpu_device->peer_access_mask & (1 << in_elem_dev->level_zero_index) ) { + /* We can directly do D2D, so let's skip the selection */ + goto src_selected; + } + } + + /* If gpu_elem is not invalid, then it is already there and the right version, + * and we're not going to transfer from another source, skip the selection */ + if( gpu_elem->coherency_state != PARSEC_DATA_COHERENCY_INVALID ) + goto src_selected; + + for(int t = 1; t < (int)parsec_nb_devices; t++) { + parsec_device_level_zero_module_t *target = (parsec_device_level_zero_module_t*)parsec_mca_device_get(t); + if( PARSEC_DEV_LEVEL_ZERO != target->super.super.type ) continue; + if(gpu_device->peer_access_mask & (1 << target->level_zero_index)) { + parsec_data_copy_t *candidate = original->device_copies[t]; + if( NULL != candidate && candidate->version == in_elem->version ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is a potential alternative source for in_elem %p on data %p", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, in_elem, original); + if(PARSEC_DATA_COHERENCY_INVALID == candidate->coherency_state) { + /* We're already pulling this data on candidate... + * If there is another candidate that already has it, we'll use + * that one; otherwise, we'll fall back on the CPU version. */ + potential_alt_src = 1; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is invalid, continuing to look for alternatives", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index); + continue; + } + /* candidate is the best candidate to do D2D. Let's register as a reader for this + * data copy, and we can unlock and schedule the D2D. */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is the best candidate to to Device to Device copy, increasing its readers to %d", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, candidate->readers+1); + parsec_atomic_fetch_inc_int32( &candidate->readers ); + undo_readers_inc_if_no_transfer = 1; + /* We swap data_in with candidate, so we update the reference counters */ + PARSEC_OBJ_RETAIN(candidate); + release_after_data_in_is_attached = task_data->data_in; + task_data->data_in = candidate; + in_elem = candidate; + in_elem_dev = target; + goto src_selected; + } + } + } + if( potential_alt_src ) { + /* We found a potential alternative source, but it's not ready now, + * we delay the scheduling of this task. */ + /** TODO: when considering RW acccesses, don't forget to unchop gpu_elem + * from its queue... */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tThere is a potential alternative source for in_elem %p [ref_count %d] in original %p to go in copy %p [ref_count %d], but it is not ready, falling back on CPU source", + gpu_device->super.name, in_elem, in_elem->super.super.obj_reference_count, original, gpu_elem, gpu_elem->super.super.obj_reference_count); + //return PARSEC_HOOK_RETURN_NEXT; + } + + /* We fall back on the CPU copy */ + assert( original->device_copies[0] != NULL && in_elem->version == original->device_copies[0]->version ); + in_elem = original->device_copies[0]; + assert(task_data->data_in == in_elem); + assert(in_elem->device_index == 0); + in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(in_elem->device_index); + } + + src_selected: + transfer_from = parsec_data_start_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + if( PARSEC_FLOW_ACCESS_WRITE & type && gpu_task->task_type != GPU_TASK_TYPE_PREFETCH ) { + gpu_elem->version++; /* on to the next version */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] increments version to %d at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, + __FILE__, __LINE__); + } + + gpu_device->super.required_data_in += nb_elts; + if( -1 != transfer_from ) { + /* If it is already under transfer, don't schedule the transfer again. + * This happens if the task refers twice (or more) to the same input flow */ + if( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p): data copy is already under transfer, ignoring double request", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*) + in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*) + gpu_elem->device_private); + } else { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p)", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*)in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*)gpu_elem->device_private); + + assert((gpu_elem->version < in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + parsec_profile_data_collection_info_t info; + + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + assert( GPU_TASK_TYPE_PREFETCH != gpu_task->task_type ); + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = -1; + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH) ) { + gpu_task->prof_key_end = parsec_gpu_prefetch_key_end; + gpu_task->prof_event_id = (int64_t)gpu_elem->device_private; + gpu_task->prof_tp_id = level_zero_device->level_zero_index; + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_prefetch_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } + if(GPU_TASK_TYPE_PREFETCH != gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) ) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_start, + (int64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, + &info); + } + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) { + parsec_device_level_zero_memory_prof_info_t _info; + _info.size = (uint64_t)nb_elts; + _info.data_key = gpu_elem->original->key; + _info.dc_id = (uint64_t)(gpu_elem->original->dc); + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_gpu_use_memory_key_start, (uint64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, &_info, PARSEC_PROFILING_EVENT_HAS_INFO); + } + } +#endif + /* Push data into the GPU from the source device */ + if(PARSEC_SUCCESS != gpu_task->stage_in(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_in"); + if( in_elem_dev->super.super.type != PARSEC_DEV_LEVEL_ZERO ) + parsec_warning("<<%p>> -> <<%p on LEVEL_ZERO device %d>> [%d, H2D]", + in_elem->device_private, gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + else + parsec_warning("<<%p on LEVEL_ZERO device %d>> -> <<%p on LEVEL_ZERO device %d>> [%d, D2D]", + in_elem->device_private, in_elem_dev->level_zero_index, + gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + assert(0); + return -1; + } + assert(in_elem_dev->super.super.device_index < gpu_device->super.data_in_array_size); + gpu_device->super.data_in_from_device[in_elem_dev->super.super.device_index] += nb_elts; + if( PARSEC_GPU_TASK_TYPE_KERNEL == gpu_task->task_type ) + gpu_device->super.nb_data_faults += nb_elts; + + /* update the data version in GPU immediately, and mark the data under transfer */ + assert((gpu_elem->version != in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + gpu_elem->version = in_elem->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets the same version %d as copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, in_elem, in_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + + gpu_elem->data_transfer_status = PARSEC_DATA_STATUS_UNDER_TRANSFER; + } + gpu_elem->push_task = gpu_task->ec; /* only the task who does the transfer can modify the data status later. */ + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + return 1; + } + if( undo_readers_inc_if_no_transfer ) + parsec_atomic_fetch_dec_int32( &in_elem->readers ); + assert( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER ); + + parsec_data_end_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tNO Move %s for data copy %p [ref_count %d, key %x] of %d bytes (host v:%d / device v:%d)", + gpu_device->super.name, + (NULL == in_elem_dev) ? "h2d" : (in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D"), + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem->version, gpu_elem->version); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + /* TODO: data keeps the same coherence flags as before */ + return 0; +} + +static parsec_flow_t parsec_level_zero_data_prefetch_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_data_prefetch_tc = { + .name = "LEVEL_ZERO PREFETCH", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_data_prefetch_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static int +parsec_level_zero_destroy_task(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_task_t** out_task) +{ +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + parsec_gpu_task_t *gpu_task = *out_task; + (void)gpu_device; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: Destroying task %s (%p with ec %p)", + gpu_device->super.name, parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task, gpu_task->ec); + assert( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type || GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ); + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type) PARSEC_DATA_COPY_RELEASE( gpu_task->ec->data[0].data_in); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, "GPU[%s]: gpu_task %p freed at %s:%d\n", + gpu_device->super.name, gpu_task, __FILE__, __LINE__); + free( gpu_task->ec ); + gpu_task->ec = NULL; + free( *out_task ); + *out_task = NULL; + return 0; +} + +#if defined(PARSEC_DEBUG_NOISIER) +static char *parsec_level_zero_debug_advice_to_string(int advice) +{ + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + return "Prefetch"; + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + return "Set Preferred Device"; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return "Mark data as recently used"; + default: + assert(0); + return "Undefined advice"; + } +} +#endif + +static int +parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)dev; +#if defined(PARSEC_DEBUG_NOISIER) + char buffer[64]; + if(NULL != data->dc) { + data->dc->key_to_string(data->dc, data->key, buffer, 64); + } else { + snprintf(buffer, 64, "unbound data"); + } +#endif + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: User provides advice %s of %s (%p)", + gpu_device->super.name, + parsec_level_zero_debug_advice_to_string(advice), + buffer, + data); + + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + data->preferred_device = dev->device_index; + /* We continue on to the next case, as we want to also + * prefetch the data on the target device, if it is the + * preferred device */ + break; //__attribute__ ((fallthrough)); + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + { + if( parsec_type_contiguous(data->device_copies[ data->owner_device ]->dtt) != PARSEC_SUCCESS){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + " PARSEC_DEV_DATA_ADVICE_PREFETCH cannot be applied to non contiguous types "); + return PARSEC_ERROR; + } + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_PREFETCH; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_data_prefetch_tc; + gpu_task->flow[0] = &parsec_level_zero_data_prefetch_flow; + gpu_task->flow_nb_elts[0] = data->device_copies[ data->owner_device ]->original->nb_elts; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + PARSEC_DEBUG_VERBOSE(20, parsec_debug_output, "Retain data copy %p [ref_count %d] at %s:%d", + data->device_copies[ data->owner_device ], + data->device_copies[ data->owner_device ]->super.super.obj_reference_count, + __FILE__, __LINE__); + PARSEC_OBJ_RETAIN(data->device_copies[ data->owner_device ]); + gpu_task->ec->data[0].data_in = data->device_copies[ data->owner_device ]; + gpu_task->ec->data[0].data_out = NULL; + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] linked to prefetch gpu task %p on GPU copy %p [ref_count %d]", + gpu_device->super.name, gpu_task->ec->data[0].data_in, gpu_task->ec->data[0].data_in->super.super.obj_reference_count, + gpu_task, gpu_task->ec->data[0].data_out, gpu_task->ec->data[0].data_out->super.super.obj_reference_count); + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_SUCCESS; + } + break; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return PARSEC_ERR_NOT_IMPLEMENTED; + break; + default: + assert(0); + return PARSEC_ERR_NOT_FOUND; + } + return PARSEC_SUCCESS; +} + +#if PARSEC_GPU_USE_PRIORITIES + +static inline parsec_list_item_t* parsec_push_task_ordered( parsec_list_t* list, + parsec_list_item_t* elem ) +{ + parsec_list_push_sorted(list, elem, parsec_execution_context_priority_comparator); + return elem; +} +#define PARSEC_PUSH_TASK parsec_push_task_ordered +#else +#define PARSEC_PUSH_TASK parsec_list_push_back +#endif + +static parsec_flow_t parsec_level_zero_d2d_complete_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_d2d_complete_tc = { + .name = "D2D TRANSFER COMPLETE", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_d2d_complete_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static void +parsec_gpu_send_transfercomplete_cmd_to_device(parsec_data_copy_t *copy, + parsec_device_module_t *current_dev, + parsec_device_module_t *dst_dev) +{ + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_D2D_COMPLETE; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_d2d_complete_tc; + gpu_task->flow[0] = &parsec_level_zero_d2d_complete_flow; + gpu_task->flow_nb_elts[0] = copy->original->nb_elts; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + gpu_task->ec->data[0].data_in = copy; /* We need to set not-null in data_in, so that the fake flow is + * not ignored when poping the data from the fake task */ + gpu_task->ec->data[0].data_out = copy; /* We "free" data[i].data_out if its readers reaches 0 */ + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* D2D complete tasks are pure internal management, we do not trace them */ +#endif + (void)current_dev; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] D2D transfer is complete, sending order to count it " + "to LEVEL_ZERO Device %s", + current_dev->name, gpu_task->ec->data[0].data_out, + gpu_task->ec->data[0].data_out->super.super.obj_reference_count, + dst_dev->name); + parsec_fifo_push( &(((parsec_device_gpu_module_t*)dst_dev)->pending), (parsec_list_item_t*)gpu_task ); +} + +static int +parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t **gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + (void)gpu_stream; + + parsec_gpu_task_t *gtask = *gpu_task; + parsec_task_t *task; + int32_t i; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; + char task_str2[MAX_TASK_STRLEN]; +#endif + const parsec_flow_t *flow; + /** + * Even though level_zero fence returns success, the PUSH may not be + * completed if no PUSH is required by this task and the PUSH is + * actually done by another task, so we need to check if the data is + * actually ready to use + */ + assert(gpu_stream == gpu_device->exec_stream[0]); + task = gtask->ec; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: parsec_gpu_callback_complete_push, PUSH of %s", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task)); + + for( i = 0; i < task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->data[i].data_in ) continue; + /* We also don't push back non-parsec-owned copies */ + if(NULL != task->data[i].data_out && + 0 == (task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED)) continue; + + flow = gtask->flow[i]; + assert( flow ); + assert( flow->flow_index == i ); + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if(task->data[i].data_out->push_task == task ) { /* only the task who did this PUSH can modify the status */ + parsec_atomic_lock(&task->data[i].data_out->original->lock); + task->data[i].data_out->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER; + parsec_data_end_transfer_ownership_to_copy(task->data[i].data_out->original, + gpu_device->super.device_index, + flow->flow_flags); +#if defined(PARSEC_PROF_TRACE) + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_end, + (int64_t)(int64_t)task->data[i].data_out->device_private, + level_zero_device->level_zero_index, + NULL); + } +#endif + task->data[i].data_out->push_task = NULL; + parsec_atomic_unlock(&task->data[i].data_out->original->lock); + parsec_device_gpu_module_t *src_device = + (parsec_device_gpu_module_t*)parsec_mca_device_get( task->data[i].data_in->device_index ); + if( PARSEC_DEV_LEVEL_ZERO == src_device->super.type ) { + int om; + while(1) { + /* There are two ways out: + * either we exit with om = 0, and then nobody was managing src_device, + * and nobody can start managing src_device until we make it change from -1 to 0 + * (but anybody who has work to do will wait until that happens), or + * we exit with om > 0, then there is a manager for that thread, and we have + * increased mutex to warn the manager that there is another task for it to do. + */ + om = src_device->mutex; + if(om == 0) { + /* Nobody at the door, let's try to lock the door */ + if( parsec_atomic_cas_int32(&src_device->mutex, 0, -1) ) + break; + continue; + } + if(om < 0 ) { + /* Damn, another thread is also trying to do an atomic operation on src_device, + * we give it some time and try again */ + struct timespec delay; + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + continue; + } + /* There is a manager, let's try to reserve another task to do. + * If that fails, the manager may have leaved, try a gain. */ + if( parsec_atomic_cas_int32(&src_device->mutex, om, om+1) ) + break; + } + if( 0 == om ) { + int rc; + /* Nobody is at the door to handle that event on the source of that data... + * we do the command directly */ + parsec_atomic_lock( &task->data[i].data_in->original->lock ); + task->data[i].data_in->readers--; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tExecuting D2D transfer complete for copy %p [ref_count %d] for " + "device %s -- readers now %d", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, src_device->super.name, + task->data[i].data_in->readers); + assert(task->data[i].data_in->readers >= 0); + if(0 == task->data[i].data_in->readers) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count); + parsec_list_item_ring_chop((parsec_list_item_t*)task->data[i].data_in); + PARSEC_LIST_ITEM_SINGLETON(task->data[i].data_in); + parsec_list_push_back(&src_device->gpu_mem_lru, (parsec_list_item_t*)task->data[i].data_in); + src_device->data_avail_epoch++; + } + parsec_atomic_unlock( &task->data[i].data_in->original->lock ); + /* Notify any waiting thread that we're done messing with that device structure */ + rc = parsec_atomic_cas_int32(&src_device->mutex, -1, 0); (void)rc; + assert(rc); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tSending D2D transfer complete command to %s for copy %p " + "[ref_count %d] -- readers is still %d", + gpu_device->super.name, src_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, task->data[i].data_in->readers); + parsec_gpu_send_transfercomplete_cmd_to_device(task->data[i].data_in, + (parsec_device_module_t*)gpu_device, + (parsec_device_module_t*)src_device); + } + } + continue; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tparsec_gpu_callback_complete_push, PUSH of %s: task->data[%d].data_out = %p [ref_count = %d], and push_task is %s, %s because transfer_status is %d", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task), + i, task->data[i].data_out, task->data[i].data_out->super.super.obj_reference_count, + (NULL != task->data[i].data_out->push_task) ? parsec_task_snprintf(task_str2, MAX_TASK_STRLEN, task->data[i].data_out->push_task) : "(null)", + (task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER) ? "all is good" : "Assertion", + task->data[i].data_out->data_transfer_status); + assert(task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + if( task->data[i].data_out->data_transfer_status != PARSEC_DATA_STATUS_COMPLETE_TRANSFER ) { /* data is not ready */ + /** + * As long as we have only one stream to push the data on the GPU we should never + * end up in this case. Remove previous assert if changed. + */ + return -1; + } + } + gtask->complete_stage = NULL; + + if( GPU_TASK_TYPE_PREFETCH == gtask->task_type ) { + parsec_data_copy_t *gpu_copy = task->data[0].data_out; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + assert(NULL != gpu_copy); + if( NULL != gpu_copy->original->dc ) + gpu_copy->original->dc->key_to_string(gpu_copy->original->dc, gpu_copy->original->key, tmp, MAX_TASK_STRLEN); + else + snprintf(tmp, MAX_TASK_STRLEN, "unbound data"); +#endif + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tPrefetch for data copy %p [ref_count %d] (%s) done. readers = %d, device_index = %d, version = %d, flags = %d, state = %d, data_transfer_status = %d", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, + tmp, + gpu_copy->readers, gpu_copy->device_index, gpu_copy->version, + gpu_copy->flags, gpu_copy->coherency_state, gpu_copy->data_transfer_status); + gpu_copy->readers--; + if( 0 == gpu_copy->readers ) { + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tMake copy %p [ref_count %d] available after prefetch from gpu_task %p, ec %p", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, gtask, gtask->ec); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + return parsec_level_zero_destroy_task(gpu_device, gpu_task); + } + + return 0; +} + +/** + * This function tries to progress a stream, by picking up a ready task + * and applying the progress function. The task to be progresses is + * always the highest priority in the waiting queue, even when a task + * has been specified as an input argument. + * The progress function is either specified by the caller via the + * upstream_progress_fct input argument or by the next task to be progresses + * via the submit function associated with the task. In any case, this + * function progresses a single task, which is then returned as the + * out_task parameter. + * + * Beware: this function does not generate errors by itself, instead + * it propagates upward the return code of the progress function. + * However, by convention the error code follows the parsec_hook_return_e + * enum. + */ +static inline int +progress_stream( parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* stream, + parsec_advance_task_function_t upstream_progress_fct, + parsec_gpu_task_t* task, + parsec_gpu_task_t** out_task ) +{ + parsec_advance_task_function_t progress_fct; + int saved_rc = 0, rc; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; +#endif + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)stream; + + /* We always handle the tasks in order. Thus if we got a new task, add it to the + * local list (possibly by reordering the list). Also, as we can return a single + * task first try to see if anything completed. */ + if( NULL != task ) { + PARSEC_PUSH_TASK(stream->fifo_pending, (parsec_list_item_t*)task); + task = NULL; + } + *out_task = NULL; + progress_fct = upstream_progress_fct; + + if( NULL != stream->tasks[stream->end] ) { + rc = zeFenceQueryStatus(level_zero_stream->fences[stream->end]); + if( ZE_RESULT_SUCCESS == rc ) { + /* Save the task for the next step */ + task = *out_task = stream->tasks[stream->end]; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: Completed %s priority %d on stream %s{%p}", + gpu_device->super.name, + parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task->ec), + task->ec->priority, stream->name, (void*)stream); + stream->tasks[stream->end] = NULL; + rc = zeCommandListReset(level_zero_stream->command_lists[stream->end]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", rc, {} ); + stream->end = (stream->end + 1) % stream->max_events; + +#if defined(PARSEC_PROF_TRACE) + if( stream->prof_event_track_enable ) { + if( task->prof_key_end != -1 ) { + PARSEC_PROFILING_TRACE(stream->profiling, task->prof_key_end, task->prof_event_id, task->prof_tp_id, NULL); + } + } +#endif /* (PARSEC_PROF_TRACE) */ + + rc = PARSEC_HOOK_RETURN_DONE; + if (task->complete_stage) + rc = task->complete_stage(gpu_device, out_task, stream); + /* the task can be withdrawn by the system */ + return rc; + } + if( ZE_RESULT_NOT_READY != rc ) { + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", rc, + {return PARSEC_HOOK_RETURN_AGAIN;} ); + } + } + + grab_a_task: + if( NULL == stream->tasks[stream->start] ) { /* there is room on the stream */ + task = (parsec_gpu_task_t*)parsec_list_pop_front(stream->fifo_pending); /* get the best task */ + } + if( NULL == task ) { /* No tasks, we're done */ + return saved_rc; + } + PARSEC_LIST_ITEM_SINGLETON((parsec_list_item_t*)task); + + assert( NULL == stream->tasks[stream->start] ); + /** + * In case the task is succesfully progressed, the corresponding profiling + * event is triggered. + */ + if ( NULL == upstream_progress_fct ) { + /* Grab the submit function */ + progress_fct = task->submit; +#if defined(PARSEC_DEBUG_PARANOID) + int i; + const parsec_flow_t *flow; + for( i = 0; i < task->ec->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->ec->data[i].data_in ) continue; + + flow = task->flow[i]; + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if( 0 == (task->ec->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + assert(task->ec->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + } +#endif /* defined(PARSEC_DEBUG_PARANOID) */ + } + + /* Prepare the command list and the Fence before progressing the function since it may + * enqueue stuff to the command list */ + ze_result_t ze_rc = zeCommandListReset(level_zero_stream->command_lists[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); + ze_rc = zeFenceReset(level_zero_stream->fences[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { } ); + + rc = progress_fct( gpu_device, task, stream ); + if( 0 > rc ) { + if( PARSEC_HOOK_RETURN_AGAIN != rc && + PARSEC_HOOK_RETURN_ASYNC != rc ) { + *out_task = task; + return rc; + } + + if( PARSEC_HOOK_RETURN_ASYNC == rc ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU task %p has been removed by the progress function", + gpu_device->super.name, (void*)task); + } else { + parsec_fifo_push(stream->fifo_pending, (parsec_list_item_t*)task); + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Reschedule task %p: no room available on the GPU for data", + gpu_device->super.name, (void*)task->ec); + } + *out_task = NULL; + return PARSEC_HOOK_RETURN_DONE; + } + /** + * Do not skip the level_zero fence generation. The problem is that some of the inputs + * might be in the pipe of being transferred to the GPU. If we activate this task + * too early, it might get executed before the data is available on the GPU. + * Obviously, this lead to incorrect results. + */ + ze_rc = zeCommandListClose(level_zero_stream->command_lists[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[stream->start], level_zero_stream->fences[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); + + stream->tasks[stream->start] = task; + stream->start = (stream->start + 1) % stream->max_events; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]: Submitted %s(task %p) priority %d on stream %s{%p}", + gpu_device->super.name, + task->ec->task_class->name, (void*)task->ec, task->ec->priority, + stream->name, (void*)stream); + + task = NULL; + goto grab_a_task; +} + +/** + * This function schedule the move of all the data required for a + * specific task from the main memory into the GPU memory. + * + * Returns: + * a positive number: the number of data to be moved. + * -1: data cannot be moved into the GPU. + * -2: No more room on the GPU to move this data. + */ +static int +parsec_gpu_kernel_push( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + parsec_task_t *this_task = gpu_task->ec; + const parsec_flow_t *flow; + int i, ret = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + +#if 0 + if( gpu_task->last_data_check_epoch == gpu_device->data_avail_epoch ) + return PARSEC_HOOK_RETURN_AGAIN; +#endif + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Push %s", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task) ); + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type ) { + if( NULL == gpu_task->ec->data[0].data_in->original ) { + /* The PREFETCH order comes after the copy was detached and released, ignore it */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s has been released already, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task)); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + if( NULL != gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index] && + gpu_task->ec->data[0].data_in->original->owner_device == gpu_device->super.device_index ) { + /* There is already a copy of this data in the GPU */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s data_copy at index %d is %p, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_device->super.device_index, + gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index]); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + } + + /* Do we have enough available memory on the GPU to hold the input and output data ? */ + ret = parsec_gpu_data_reserve_device_space( level_zero_device, gpu_task ); + if( ret < 0 ) { + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch; + return ret; + } + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + + flow = gpu_task->flow[i]; + /* Skip CTL flows */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* If there is already a GPU data copy (set by reserve_device_space), and this copy + * is not parsec-owned, don't stage in */ + if( NULL != this_task->data[i].data_out && + (0 == (this_task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) ) continue; + + assert( NULL != parsec_data_copy_get_ptr(this_task->data[i].data_in) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\t\tIN Data of %s <%x> on GPU", + gpu_device->super.name, flow->name, + this_task->data[i].data_out->original->key); + ret = parsec_gpu_data_stage_in( level_zero_device, flow, + &(this_task->data[i]), gpu_task, gpu_stream ); + if( ret < 0 ) { + return ret; + } + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Push task %s DONE", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + gpu_task->complete_stage = parsec_gpu_callback_complete_push; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* We do not log that event as the completion of this task */ +#endif + return ret; +} + +/** + * This function schedule the move of all the modified data for a + * specific task from the GPU memory into the main memory. + * + * Returns: negative number if any error occured. + * positive: the number of data to be moved. + */ +static int +parsec_gpu_kernel_pop( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy; + parsec_data_t *original; + uint32_t nb_elts; + const parsec_flow_t *flow; + int return_code = 0, how_many = 0, i, update_data_epoch = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + for( i = 0; i < this_task->locals[0].value; i++ ) { + gpu_copy = this_task->data[i].data_out; + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + original = gpu_copy->original; + if(PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << i), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + goto release_and_return_error; + } + } + return return_code; + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Pop %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* We need to manage all data that has been used as input, even if they were read only */ + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + flow = gpu_task->flow[i]; + if( PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags) ) continue; /* control flow */ + + gpu_copy = this_task->data[i].data_out; + + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + original = gpu_copy->original; + nb_elts = gpu_task->flow_nb_elts[i]; + + assert( this_task->data[i].data_in == NULL || original == this_task->data[i].data_in->original ); + + if( !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Do not propagate GPU copies to successors (temporary solution) */ + this_task->data[i].data_out = original->device_copies[0]; + } + parsec_atomic_lock(&original->lock); + if( flow->flow_flags & PARSEC_FLOW_ACCESS_READ ) { + gpu_copy->readers--; + if( gpu_copy->readers < 0 ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: While trying to Pop %s, gpu_copy %p [ref_count %d] on flow %d with original %p had already 0 readers", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), + gpu_copy, gpu_copy->super.super.obj_reference_count, + i, original); + + } + assert(gpu_copy->readers >= 0); + if( (0 == gpu_copy->readers) && + !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available on flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); /* TODO: singleton instead? */ + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + update_data_epoch = 1; + parsec_atomic_unlock(&original->lock); + continue; /* done with this element, go for the next one */ + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tread copy %p [ref_count %d] on flow %s has readers (%i)", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name, gpu_copy->readers); + } + if( flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE ) { + assert( gpu_copy == parsec_data_get_copy(gpu_copy->original, gpu_device->super.device_index) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tOUT Data copy %p [ref_count %d] for flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + + /* Stage the transfer of the data back to main memory */ + gpu_device->super.required_data_out += nb_elts; + assert( ((parsec_list_item_t*)gpu_copy)->list_next == (parsec_list_item_t*)gpu_copy ); + assert( ((parsec_list_item_t*)gpu_copy)->list_prev == (parsec_list_item_t*)gpu_copy ); + + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + if( gpu_task->pushout & (1 << i) ) { + /* TODO: make sure no readers are working on the CPU version */ + original = gpu_copy->original; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tMove D2H data <%s:%x> copy %p [ref_count %d] -- D:%p -> H:%p requested", + gpu_device->super.name, flow->name, original->key, gpu_copy, gpu_copy->super.super.obj_reference_count, + (void*)gpu_copy->device_private, original->device_copies[0]->device_private); +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT) { + parsec_profile_data_collection_info_t info; + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = parsec_gpu_moveout_key_end; + gpu_task->prof_tp_id = this_task->taskpool->taskpool_id; + gpu_task->prof_event_id = this_task->task_class->key_functions->key_hash(this_task->task_class->make_key(this_task->taskpool, this_task->locals), NULL); + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_moveout_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } else { + gpu_task->prof_key_end = -1; + } + } +#endif + /* Move the data back into main memory */ + if( PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + parsec_atomic_unlock(&original->lock); + goto release_and_return_error; + } + gpu_device->super.data_out_to_host += nb_elts; /* TODO: not hardcoded, use datatype size */ + how_many++; + } else { + assert( 0 == gpu_copy->readers ); + } + } + parsec_atomic_unlock(&original->lock); + } + + release_and_return_error: + if( update_data_epoch ) { + gpu_device->data_avail_epoch++; + } + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Pop %s DONE (return %d data epoch %"PRIu64")", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), return_code, gpu_device->data_avail_epoch ); + + return (return_code < 0 ? return_code : how_many); +} + +/** + * Make sure all data on the device is correctly put back into the queues. + */ +static int +parsec_gpu_kernel_epilog( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Epilog of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + + + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + cpu_copy = original->device_copies[0]; + + /* If it is a copy managed by the user, don't bother either */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + /** + * There might be a race condition here. We can't assume the first CPU + * version is the corresponding CPU copy, as a new CPU-bound data + * might have been created meanwhile. + * + * WARNING: For now we always forward the cpu_copy to the next task, to + * do that, we lie to the engine by updating the CPU copy to the same + * status than the GPU copy without updating the data itself. Thus, the + * cpu copy is really invalid. this is related to Issue #88, and the + * fact that: + * - we don't forward the gpu copy as output + * - we always take a cpu copy as input, so it has to be in the + * same state as the GPU to prevent an extra data movement. + */ + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + + /** + * The cpu_copy will be updated in the completion, and at that moment + * the two versions will be identical. + */ + cpu_copy->version = gpu_copy->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: CPU copy %p [ref_count %d] gets the same version %d as GPU copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + cpu_copy, cpu_copy->super.super.obj_reference_count, cpu_copy->version, gpu_copy, gpu_copy->super.super.obj_reference_count, + __FILE__, __LINE__); + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + + assert( 0 == gpu_copy->readers ); + + if( gpu_task->pushout & (1 << i) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the owned LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_push_back(&gpu_device->gpu_mem_owned_lru, (parsec_list_item_t*)gpu_copy); + } + } + return 0; +} + +/** @brief Release the LEVEL_ZERO copies of the data used in WRITE mode. + * + * @details This function can be used when the LEVEL_ZERO task didn't run + * to completion on the device (either due to an error, or + * simply because the body requested a reexecution on a + * different location). It releases the LEVEL_ZERO copies of the + * output data, allowing them to be reused by the runtime. + * This function has the drawback of kicking in too late, + * after all data transfers have been completed toward the + * device. + * + * @param [IN] gpu_device, the GPU device the the task has been + * supposed to execute. + * @param [IN] gpu_task, the task that has been cancelled, and which + * needs it's data returned to the runtime. + * @return Currently only success. + */ +static int +parsec_gpu_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i, data_avail_epoch = 0; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Cleanup of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + parsec_atomic_lock(&original->lock); + assert(gpu_copy->super.super.obj_reference_count > 1); + /* Issue #134 */ + parsec_data_copy_detach(original, gpu_copy, gpu_device->super.device_index); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy = original->device_copies[0]; + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + if( 0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + parsec_atomic_unlock(&original->lock); + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s\n", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + } + if( data_avail_epoch ) /* Update data availability epoch */ + gpu_device->data_avail_epoch++; + return 0; +} + +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on LEVEL_ZERO events, + * such an event indicate that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ) +{ + parsec_device_gpu_module_t* gpu_device; + int rc, exec_stream = 0; + parsec_gpu_task_t *progress_task, *out_task_submit = NULL, *out_task_pop = NULL; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + int pop_null = 0; + + gpu_device = (parsec_device_gpu_module_t*)parsec_mca_device_get(which_gpu); + +#if defined(PARSEC_PROF_TRACE) + PARSEC_PROFILING_TRACE_FLAGS( es->es_profile, + PARSEC_PROF_FUNC_KEY_END(gpu_task->ec->taskpool, + gpu_task->ec->task_class->task_class_id), + gpu_task->ec->task_class->key_functions->key_hash(gpu_task->ec->task_class->make_key(gpu_task->ec->taskpool, gpu_task->ec->locals), NULL), + gpu_task->ec->taskpool->taskpool_id, NULL, + PARSEC_PROFILING_EVENT_RESCHEDULED ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + /* Check the GPU status -- three kinds of values for rc: + * - rc < 0: somebody is doing a short atomic operation while there is no manager, + * so wait. + * - rc == 0: there is no manager, and at the exit of the while, this thread + * made rc go from 0 to 1, so it is the new manager of the GPU and + * needs to deal with gpu_task + * - rc > 0: there is a manager, and at the exit of the while, this thread has + * committed new work that the manager will need to do, but the work is + * not in the queue yet. + */ + while(1) { + rc = gpu_device->mutex; + struct timespec delay; + if( rc >= 0 ) { + if( parsec_atomic_cas_int32( &gpu_device->mutex, rc, rc+1 ) ) { + break; + } + } else { + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + } + } + if( 0 < rc ) { + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_HOOK_RETURN_ASYNC; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Entering GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_start, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + check_in_deps: + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tUpload data (if any) for %s priority %d", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[0], + parsec_gpu_kernel_push, + gpu_task, &progress_task ); + if( rc < 0 ) { /* In case of error progress_task is the task that raised it */ + if( -1 == rc ) + goto disable_gpu; + /* We are in the early stages, and if there no room on the GPU for a task we need to + * delay all retries for the same task for a little while. Meanwhile, put the task back + * trigger a device flush, and keep executing tasks that have their data on the device. + */ + if( NULL != progress_task ) { + PARSEC_PUSH_TASK(gpu_device->exec_stream[0]->fifo_pending, (parsec_list_item_t*)progress_task); + progress_task = NULL; + } + /* If we can extract data go for it, otherwise try to drain the pending tasks */ + gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); + if( NULL != gpu_task ) + goto get_data_out_of_device; + } + gpu_task = progress_task; + + /* Stage-in completed for this task: it is ready to be executed */ + exec_stream = (exec_stream + 1) % (gpu_device->max_exec_streams - 2); /* Choose an exec_stream */ + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tExecute %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[2+exec_stream], + NULL, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( PARSEC_HOOK_RETURN_DISABLE == rc ) + goto disable_gpu; + if( PARSEC_HOOK_RETURN_ASYNC != rc ) { + /* Reschedule the task. As the chore_id has been modified, + another incarnation of the task will be executed. */ + if( NULL != progress_task ) { + parsec_gpu_kernel_cleanout(gpu_device, progress_task); + __parsec_reschedule(es, progress_task->ec); + gpu_task = progress_task; + progress_task = NULL; + goto remove_gpu_task; + } + gpu_task = NULL; + goto fetch_task_from_shared_queue; + } + progress_task = NULL; + } + gpu_task = progress_task; + out_task_submit = progress_task; + + get_data_out_of_device: + if( NULL != gpu_task ) { /* This task has completed its execution */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tRetrieve data (if any) for %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + /* Task is ready to move the data back to main memory */ + rc = progress_stream( gpu_device, + gpu_device->exec_stream[1], + parsec_gpu_kernel_pop, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( -1 == rc ) + goto disable_gpu; + } + if( NULL != progress_task ) { + /* We have a succesfully completed task. However, it is not gpu_task, as + * it was just submitted into the data retrieval system. Instead, the task + * ready to move into the next level is the progress_task. + */ + gpu_task = progress_task; + progress_task = NULL; + goto complete_task; + } + gpu_task = progress_task; + out_task_pop = progress_task; + + fetch_task_from_shared_queue: + assert( NULL == gpu_task ); + if (1 == parsec_level_zero_sort_pending && out_task_submit == NULL && out_task_pop == NULL) { + parsec_gpu_sort_pending_list(gpu_device); + } + gpu_task = (parsec_gpu_task_t*)parsec_fifo_try_pop( &(gpu_device->pending) ); + if( NULL != gpu_task ) { + pop_null = 0; + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch - 1; /* force at least one tour */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tGet from shared queue %s priority %d", gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority); + if( GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ) { + goto get_data_out_of_device; + } + } else { + pop_null++; + if( pop_null % 1024 == 1023 ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, "GPU[%s]:\tStill waiting for %d tasks to execute, but poped NULL the last %d times I tried to pop something...", + gpu_device->super.name, gpu_device->mutex, pop_null); + } + } + goto check_in_deps; + + complete_task: + assert( NULL != gpu_task ); + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tComplete %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec)); + /* Everything went fine so far, the result is correct and back in the main memory */ + PARSEC_LIST_ITEM_SINGLETON(gpu_task); + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + parsec_gpu_complete_w2r_task(gpu_device, gpu_task, es); + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + } + if (gpu_task->task_type == GPU_TASK_TYPE_D2D_COMPLETE) { + free( gpu_task->ec ); + gpu_task->ec = NULL; + goto remove_gpu_task; + } + parsec_gpu_kernel_epilog( gpu_device, gpu_task ); + __parsec_complete_execution( es, gpu_task->ec ); + gpu_device->super.executed_tasks++; + remove_gpu_task: + // Load problem: was parsec_device_load[gpu_device->super.device_index] -= gpu_task->load; + parsec_device_load[gpu_device->super.device_index] -= parsec_device_sweight[gpu_device->super.device_index]; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream,"GPU[%s]: gpu_task %p freed at %s:%d", gpu_device->super.name, + gpu_task, __FILE__, __LINE__); + free( gpu_task ); + rc = parsec_atomic_fetch_dec_int32( &(gpu_device->mutex) ); + if( 1 == rc ) { /* I was the last one */ +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_end, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Leaving GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + + return PARSEC_HOOK_RETURN_ASYNC; + } + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + + disable_gpu: + /* Something wrong happened. Push all the pending tasks back on the + * cores, and disable the gpu. + */ + parsec_warning("Critical issue related to the GPU discovered. Giving up\n"); + return PARSEC_HOOK_RETURN_DISABLE; +} + +parsec_hook_return_t +parsec_dpcpp_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ) +{ + return parsec_level_zero_kernel_scheduler(es, gpu_task, which_gpu); +} +#endif /* PARSEC_HAVE_LEVEL_ZERO */ diff --git a/parsec/mca/device/parsec_gpu.c b/parsec/mca/device/parsec_gpu.c new file mode 100644 index 000000000..84fcb3c25 --- /dev/null +++ b/parsec/mca/device/parsec_gpu.c @@ -0,0 +1,97 @@ +/* + * + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/mca/device/device.h" +#include "parsec/mca/device/device_gpu.h" +#include "parsec/utils/mca_param.h" +#include "parsec/mca/mca_repository.h" +#include "parsec/constants.h" +#include "parsec/utils/debug.h" +#include "parsec/execution_stream.h" +#include "parsec/utils/argv.h" +#include "parsec/parsec_internal.h" + +void* parsec_gpu_pop_workspace(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* gpu_stream, size_t size) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; (void)gpu_stream; (void)size; + void *work = NULL; + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + if (gpu_stream->workspace == NULL) { + gpu_stream->workspace = (parsec_gpu_workspace_t *)malloc(sizeof(parsec_gpu_workspace_t)); + gpu_stream->workspace->total_workspace = PARSEC_GPU_MAX_WORKSPACE; + gpu_stream->workspace->stack_head = PARSEC_GPU_MAX_WORKSPACE - 1; + + for( int i = 0; i < PARSEC_GPU_MAX_WORKSPACE; i++ ) { + gpu_stream->workspace->workspace[i] = zone_malloc( gpu_device->memory, size); + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Succeeded Allocating workspace %d (device_ptr %p)", + gpu_device->super.name, + i, gpu_stream->workspace->workspace[i]); +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + &size, PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif + } + } + assert (gpu_stream->workspace->stack_head >= 0); + work = gpu_stream->workspace->workspace[gpu_stream->workspace->stack_head]; + gpu_stream->workspace->stack_head --; +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return work; +} + +int parsec_gpu_push_workspace(parsec_device_gpu_module_t* gpu_device, parsec_gpu_exec_stream_t* gpu_stream) +{ + (void)gpu_device; (void)gpu_stream; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_stream->workspace->stack_head ++; + assert (gpu_stream->workspace->stack_head < PARSEC_GPU_MAX_WORKSPACE); +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} + +int parsec_gpu_free_workspace(parsec_device_gpu_module_t * gpu_device) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + int i, j; + for( i = 0; i < gpu_device->max_exec_streams; i++ ) { + parsec_gpu_exec_stream_t *gpu_stream = &(level_zero_device->exec_stream[i].super); + if (gpu_stream->workspace != NULL) { + for (j = 0; j < gpu_stream->workspace->total_workspace; j++) { +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + } +#endif + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Release workspace %d (device_ptr %p)", + gpu_device->super.name, + j, gpu_stream->workspace->workspace[j]); + zone_free( gpu_device->memory, gpu_stream->workspace->workspace[j] ); + } + free(gpu_stream->workspace); + gpu_stream->workspace = NULL; + } + } +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index b3a28db19..7ddcec22b 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -18,9 +18,9 @@ #include "parsec/utils/output.h" #include "parsec/scheduling.h" -#if !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) -#error This file should not be included in a non-CUDA/HIP build -#endif /* !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) */ +#if !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) && !defined(PARSEC_HAVE_LEVEL_ZERO) +#error This file should not be included in a non-GPU (CUDA/HIP/LEVEL_ZERO) build +#endif /* !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_HIP) && !defined(PARSEC_HAVE_LEVEL_ZERO) */ /** * Entirely local tasks that should only be used to move data between a device and the main memory. Such @@ -133,6 +133,11 @@ static const __parsec_chore_t __gpu_d2h_task_chores[] = { .evaluate = NULL, .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, #endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + {.type = PARSEC_DEV_LEVEL_ZERO, + .evaluate = NULL, + .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, +#endif #if defined(PARSEC_HAVE_HIP) {.type = PARSEC_DEV_HIP, .evaluate = NULL, diff --git a/parsec/parsec_internal.h b/parsec/parsec_internal.h index 2bf4b7382..19f75d75f 100644 --- a/parsec/parsec_internal.h +++ b/parsec/parsec_internal.h @@ -339,6 +339,10 @@ typedef int (parsec_data_ref_fn_t)(parsec_task_t *task, #define PARSEC_TASK_CLASS_TYPE_PTG 0x01 #define PARSEC_TASK_CLASS_TYPE_DTD 0x02 +#if defined(__cplusplus) +#define restrict +#endif + /** * Find the dependency corresponding to a given execution context. */ diff --git a/parsec/remote_dep_mpi.c b/parsec/remote_dep_mpi.c index a55081966..a8f184c2d 100644 --- a/parsec/remote_dep_mpi.c +++ b/parsec/remote_dep_mpi.c @@ -1680,7 +1680,7 @@ remote_dep_mpi_put_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(dataptr, PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, // TODO JS: this interface is so broken, fix it! + -1, (parsec_datatype_t)NULL, // TODO JS: this interface is so broken, fix it! dtt_size, // TODO JS: what about nbdtt? Is it ok to ignore it?! &source_memory_handle, &source_memory_handle_size); @@ -2025,7 +2025,7 @@ static void remote_dep_mpi_get_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(PARSEC_DATA_COPY_GET_PTR(deps->output[k].data.data), PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, + -1, (parsec_datatype_t)NULL, dtt_size, &receiver_memory_handle, &receiver_memory_handle_size); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b489f82e4..023f52607 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -78,6 +78,7 @@ target_link_libraries(tests_common PRIVATE parsec) add_subdirectory(class) add_subdirectory(api) +add_subdirectory(subsystem) if( TARGET parsec-ptgpp ) add_subdirectory(runtime) add_subdirectory(dsl/ptg) diff --git a/tests/dsl/dtd/CMakeLists.txt b/tests/dsl/dtd/CMakeLists.txt index 50abef063..d642c342a 100644 --- a/tests/dsl/dtd/CMakeLists.txt +++ b/tests/dsl/dtd/CMakeLists.txt @@ -24,7 +24,25 @@ parsec_addtest_executable(C dtd_test_interleave_actions SOURCES dtd_test_interle parsec_addtest_executable(C dtd_test_ce SOURCES dtd_test_ce.c) parsec_addtest_executable(C dtd_test_new_tile SOURCES dtd_test_new_tile.c) -if( PARSEC_HAVE_CUDA ) +if( PARSEC_HAVE_LEVEL_ZERO ) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C dtd_test_simple_gemm_lz SOURCES dtd_test_simple_gemm_lz.c dtd_test_simple_gemm_lz_kernel.dpcpp.C) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader m) + find_package(BLAS) + if(BLAS_FOUND) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE ${BLAS_LIBRARIES}) + target_include_directories(dtd_test_simple_gemm_lz PRIVATE ${BLAS_INCLUDE_DIRS}) + target_compile_definitions(dtd_test_simple_gemm_lz PRIVATE HAVE_BLAS=1) + endif(BLAS_FOUND) + endif(TARGET MKL::MKL_DPCPP) +elseif( PARSEC_HAVE_CUDA ) parsec_addtest_executable(C dtd_test_cuda_task_insert SOURCES dtd_test_cuda_task_insert.c) if( TARGET CUDA::cublas ) parsec_addtest_executable(C dtd_test_simple_gemm SOURCES dtd_test_simple_gemm.c) @@ -37,4 +55,4 @@ if( PARSEC_HAVE_CUDA ) endif(BLAS_FOUND) target_sources(dtd_test_new_tile PUBLIC dtd_test_new_tile_cuda_kernels.cu) endif( TARGET CUDA::cublas ) -endif() +endif( PARSEC_HAVE_LEVEL_ZERO ) diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz.c b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c new file mode 100644 index 000000000..769fbf12c --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c @@ -0,0 +1,646 @@ +#include +#include "parsec.h" +#include "parsec/arena.h" +#include "parsec/data_dist/matrix/matrix.h" +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/interfaces/dtd/insert_function_internal.h" + +// The file is not compiled if LEVEL_ZERO is not present or cannot be compiled +#include "parsec/mca/device/cuda/device_cuda.h" + +#if defined(HAVE_BLAS) +// If our CMake finds a BLAS library, it defines HAVE_BLAS +// BLAS does not guarantee there is a cblas.h, we define our own prototype +typedef enum CBLAS_LAYOUT {CblasRowMajor=101, CblasColMajor=102} CBLAS_LAYOUT; +typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113} CBLAS_TRANSPOSE; +typedef enum CBLAS_UPLO {CblasUpper=121, CblasLower=122} CBLAS_UPLO; +typedef enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG; +typedef enum CBLAS_SIDE {CblasLeft=141, CblasRight=142} CBLAS_SIDE; +#define CBLAS_INDEX int + +extern void cblas_dgemm(const CBLAS_LAYOUT layout, const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, const CBLAS_INDEX M, const CBLAS_INDEX N, + const CBLAS_INDEX K, const double alpha, const double *A, + const CBLAS_INDEX lda, const double *B, const CBLAS_INDEX ldb, + const double beta, double *C, const CBLAS_INDEX ldc); +#endif + +#if defined(PARSEC_HAVE_MPI) +#include +#endif /* defined(PARSEC_HAVE_MPI) */ + +#include +#include + +static int TILE_FULL = -1; +int gemm_lz_verbose = 0; +static int device = PARSEC_DEV_LEVEL_ZERO; +static int P = -1; +static int Q = -1; + +#define Rnd64_A 6364136223846793005ULL +#define Rnd64_C 1ULL +#define RndF_Mul 5.4210108624275222e-20f +#define RndD_Mul 5.4210108624275222e-20 +#define NBELEM 1 + +#define EPSILON 1e-10 + +static unsigned long long int Rnd64_jump(unsigned long long int n, unsigned long long int seed) +{ + unsigned long long int a_k, c_k, ran; + int i; + + a_k = Rnd64_A; + c_k = Rnd64_C; + + ran = seed; + for( i = 0; n; n >>= 1, ++i ) { + if( n & 1 ) + ran = a_k * ran + c_k; + c_k *= (a_k + 1); + a_k *= a_k; + } + + return ran; +} + +int initialize_tile(parsec_execution_stream_t *es, parsec_task_t *this_task) +{ + (void)es; + double *data; + int i, j, mb, nb, m, n, M, ld; + unsigned int seed; + unsigned long long jump, ran; + + parsec_dtd_unpack_args(this_task, &data, &m, &n, &mb, &nb, &M, &ld, &seed); + + jump = (unsigned long long int)m + (unsigned long long int)n * (unsigned long long int)M; + + for( j = 0; j < nb; j++ ) { + ran = Rnd64_jump(NBELEM * jump, seed); + for( i = 0; i < mb; i++ ) { + *data = 0.5f - ran * RndF_Mul; + ran = Rnd64_A * ran + Rnd64_C; + data++; + } + data += ld - i; + jump += M; + } + return PARSEC_HOOK_RETURN_DONE; +} + +int initialize_matrix(parsec_context_t *parsec_context, int rank, parsec_matrix_block_cyclic_t *mat, unsigned int seed, + const char *name, int *gpu_device_index, int nb_gpus) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t key; + int perr; + + parsec_task_class_t *init_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + init_tc = parsec_dtd_create_task_class(tp, "init", + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* M */ + sizeof(int), PARSEC_VALUE, /* ld */ + sizeof(unsigned int), PARSEC_VALUE, /* seed */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, init_tc, PARSEC_DEV_CPU, initialize_tile); + + int g = 0; + for( int i = 0; i < mat->super.mt; i++ ) { + for( int j = 0; j < mat->super.nt; j++ ) { + key = mat->super.super.data_key(&mat->super.super, i, j); + parsec_dtd_insert_task_with_task_class(tp, init_tc, 1, PARSEC_DEV_CPU, + PARSEC_PUSHOUT, PARSEC_DTD_TILE_OF_KEY(&mat->super.super, key), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.nb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.m, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &seed, + PARSEC_DTD_ARG_END); + if(PARSEC_DEV_LEVEL_ZERO == device && + (int)mat->super.super.rank_of_key(&mat->super.super, key) == rank ) { + if( gemm_lz_verbose ) { + fprintf(stderr, "Advice %s(%d, %d) to prefer GPU device %d (parsec device %d) of rank %d\n", + name, i, j, g, gpu_device_index[g], (int)mat->super.super.rank_of_key(&mat->super.super, key)); + } + parsec_advise_data_on_device(mat->super.super.data_of_key(&mat->super.super, key), + gpu_device_index[g], + PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE); + g = (g + 1) % nb_gpus; + } + } + } + parsec_dtd_data_flush_all(tp, &mat->super.super); + + // Wait for task completion + perr = parsec_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, init_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); + +#if defined(HAVE_BLAS) +int gemm_kernel_cpu(parsec_execution_stream_t *es, + parsec_task_t *this_task) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + double alpha = 1.0; + double beta = 1.0; + double delta; + struct timeval start, end, diff; + + (void)es; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + gettimeofday(&start, NULL); + cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mb, nb, kb, alpha, A, mb, B, kb, beta, C, mb); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if( gemm_lz_verbose ) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d, %dx%d, %dx%d on node %d, on core %d: %g s\n", + m, n, k, mb, kb, kb, nb, mb, kb, + this_task->taskpool->context->my_rank, + es->core_id, + delta); + + return PARSEC_HOOK_RETURN_DONE; +} +#endif + +int simple_gemm(parsec_context_t *parsec_context, parsec_matrix_block_cyclic_t *A, parsec_matrix_block_cyclic_t *B, parsec_matrix_block_cyclic_t *C) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t keyA, keyB, keyC; + int perr; + + parsec_task_class_t *gemm_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + gemm_tc = parsec_dtd_create_task_class(tp, "GEMM", + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* A */ + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* B */ + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, /* C */ + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* k */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* kb */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_LEVEL_ZERO, gemm_kernel_lz); +#if defined(HAVE_BLAS) + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_CPU, gemm_kernel_cpu); +#endif + + for( int i = 0; i < C->super.mt; i++ ) { + for( int j = 0; j < C->super.nt; j++ ) { + keyC = C->super.super.data_key(&C->super.super, i, j); + for( int k = 0; k < A->super.nt; k++ ) { + keyA = A->super.super.data_key(&A->super.super, i, k); + keyB = B->super.super.data_key(&B->super.super, k, j); + parsec_dtd_insert_task_with_task_class(tp, gemm_tc, C->super.mt*C->super.nt*A->super.nt - i*C->super.nt + j, device, + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&A->super.super, keyA), + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&B->super.super, keyB), + k == A->super.nt - 1 ? (PARSEC_INOUT | PARSEC_PUSHOUT) : PARSEC_INOUT, + PARSEC_DTD_TILE_OF_KEY(&C->super.super, keyC), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &k, + PARSEC_DTD_EMPTY_FLAG, &C->super.mb, + PARSEC_DTD_EMPTY_FLAG, &C->super.nb, + PARSEC_DTD_EMPTY_FLAG, &B->super.mb, + PARSEC_DTD_ARG_END); + } + } + } + parsec_dtd_data_flush_all(tp, &A->super.super); + parsec_dtd_data_flush_all(tp, &B->super.super); + parsec_dtd_data_flush_all(tp, &C->super.super); + + // Wait for task completion + perr = parsec_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, gemm_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int get_nb_gpu_devices() +{ + int nb = 0; + + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + nb++; + } + } + + return nb; +} + +int *get_gpu_device_index() +{ + int *dev_index = NULL; + + dev_index = (int *)malloc(parsec_nb_devices * sizeof(int)); + int i = 0; + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + dev_index[i++] = device->device_index; + } + } + + return dev_index; +} + +static parsec_matrix_block_cyclic_t *create_initialize_matrix(parsec_context_t *parsec_context, int rank, unsigned int seed, const char *name, int mb, int nb, int M, int N, int *gpu_device_index, int nbgpus) +{ + parsec_matrix_block_cyclic_t *dc; + dc = calloc(1, sizeof(parsec_matrix_block_cyclic_t)); + parsec_matrix_block_cyclic_init(dc, PARSEC_MATRIX_DOUBLE, PARSEC_MATRIX_TILE, rank, + mb, nb, + M, N, + 0, 0, + M, N, + P, Q, + 1, 1, + 0, 0); + parsec_data_collection_t *A = &dc->super.super; + parsec_data_collection_set_key(A, name); + dc->mat = parsec_data_allocate((size_t)dc->super.nb_local_tiles * + (size_t)dc->super.bsiz * + (size_t)parsec_datadist_getsizeoftype(dc->super.mtype)); + parsec_dtd_data_collection_init(A); + initialize_matrix(parsec_context, rank, dc, seed, name, gpu_device_index, nbgpus); + + return dc; +} + +static void destroy_matrix(parsec_matrix_block_cyclic_t *dc) +{ + parsec_data_collection_t *A = &dc->super.super; + parsec_dtd_data_collection_fini(A); + if( NULL != dc->mat ) { + parsec_data_free(dc->mat); + } + parsec_tiled_matrix_destroy_data(&dc->super); + parsec_data_collection_destroy(&dc->super.super); + + free(dc); +} + +static void print_matrix(parsec_matrix_block_cyclic_t *dc, parsec_context_t *parsec_context, const char *info) +{ + for( int i = 0; i < dc->super.mt; i++ ) { + for( int j = 0; j < dc->super.nt; j++ ) { + if( (int)dc->super.super.rank_of(&dc->super.super, i, j) == parsec_context->my_rank ) { + fprintf(stderr, "%-5s(%2d, %2d): ", info, i, j); + parsec_data_t *tile = dc->super.super.data_of(&dc->super.super, i, j); + double *mat = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(tile, 0)); + for( int ii = 0; ii < dc->super.mb; ii++) { + for(int jj = 0; jj < dc->super.nb; jj++) { + fprintf(stderr, "%5.2g ", mat[ii*dc->super.nb + jj]); + } + fprintf(stderr, "\n "); + } + } + } + } +} + +int main(int argc, char **argv) +{ + int ret = 0, rc, nbgpus = 0; + parsec_context_t *parsec_context = NULL; + int rank, world; + int mb = 1024, nb = 1024, kb = 1024; + int M = 16 * mb, N = 16 * nb, K = 16 * kb; + double min_perf=0.0; + int runs = 5; + int debug=-1; + int check = 0; + +#if defined(PARSEC_HAVE_MPI) + { + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided); + } + MPI_Comm_size(MPI_COMM_WORLD, &world); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); +#else + world = 1; + rank = 0; +#endif + + while( 1 ) { + int option_index = 0; + static struct option long_options[] = { + {"M", required_argument, 0, 'M'}, + {"N", required_argument, 0, 'N'}, + {"K", required_argument, 0, 'K'}, + {"mb", required_argument, 0, 'm'}, + {"nb", required_argument, 0, 'n'}, + {"kb", required_argument, 0, 'k'}, + {"P", required_argument, 0, 'P'}, + {"Q", required_argument, 0, 'Q'}, + {"device", required_argument, 0, 'd'}, + {"nruns", required_argument, 0, 't'}, + {"verbose", no_argument, 0, 'v'}, + {"Debug", required_argument, 0, 'D'}, + {"Alarm", required_argument, 0, 'A'}, + {"Check", no_argument, 0, 'x'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int c = getopt_long(argc, argv, "M:N:K:m:n:k:P:Q:t:d:D:A:xvh", + long_options, &option_index); + if( c == -1 ) + break; + + switch( c ) { + case 'M': + M = atoi(optarg); + break; + case 'N': + N = atoi(optarg); + break; + case 'K': + K = atoi(optarg); + break; + case 'm': + mb = atoi(optarg); + break; + case 'n': + nb = atoi(optarg); + break; + case 'k': + kb = atoi(optarg); + break; + case 'P': + P = atoi(optarg); + break; + case 'Q': + Q = atoi(optarg); + break; + case 't': + runs = atoi(optarg); + break; + case 'v': + gemm_lz_verbose = !gemm_lz_verbose; + break; + case 'd': + if(strcmp(optarg, "GPU") == 0) { + device=PARSEC_DEV_LEVEL_ZERO; + } else if(strcmp(optarg, "CPU") == 0) { +#if defined(HAVE_BLAS) + device=PARSEC_DEV_CPU; +#else + fprintf(stderr, "Error: requested to run on CPU (--device=CPU), but no BLAS library has been found at configure time\n"); + exit(1); +#endif + } else { + fprintf(stderr, "Error: device parameter should either be 'GPU' or 'CPU' (got '%s')\n", optarg); + exit(1); + } + break; + case 'D': + debug = atoi(optarg); + break; + case 'A': + min_perf = strtod(optarg, NULL); + break; + case 'x': +#if defined(HAVE_BLAS) + check = 1; + runs=1; +#else + fprintf(stderr, "Error: requested to run with checks, but CPU BLAS not available\n"); +#endif + break; + case 'h': + case '?': + fprintf(stderr, + "Usage %s [flags] [-- ]\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n" + " Compute pdgemm on a process grid of PxQ, using all available GPUs on each\n" + " node (modulo parsec options), using DTD. Compute C += AxB, where A is MxK\n" + " tiled in mb x kb, B is KxN tiled in kb x nb, and C is MxN tiled in mb x nb\n" + " Executes nruns iterations of the GEMM operation.\n" + " flags:\n" + " --M|-M / --K|-K / --N|-N: set M, K and N (resp.)\n" + " --mb|-m / --kb/-k / --nb|-n: set mb, kb and nb (resp.)\n" + " --nruns|-t: set the number of runs to do\n" + " --device|-d: which device to use (CPU or GPU)\n" + " --verbose|-v: display which GEMM runs on which GPU\n" + " as execution is unfolding\n" + " --help|-h|-?: display this help\n" + " --debug|-D: blocks the process passed as parameter and\n" + " waits for gdb to connect to it\n" + " --Alarm|-A: sets the expected minimum performance for a\n" + " single GPU (kills the process if it takes longer\n" + " than the time corresponding to the expected\n" + " performance to complete the product)\n" + " --Check|-x: perform numerical check of the solution by comparing\n" + " the output of the GPU computation with the output of the\n" + " CPU BLAS computation\n" + "\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n", + argv[0]); + break; + } + } + int pargc = argc - optind + 1; + char **pargv = (char **)malloc((pargc + 1) * sizeof(char *)); + pargv[0] = argv[0]; + for( int i = 0; i < argc - optind; i++ ) + pargv[i + 1] = argv[optind + i]; + pargv[pargc] = NULL; + + if( -1 == P ) + P = (int)sqrt(world); + if( -1 == Q ) + Q = world / P; + while( P * Q != world ) { + P--; + Q = world / P; + } + + if(debug == rank) { + int loop=1; + char hostname[64]; + gethostname(hostname, 64); + fprintf(stderr, "ssh -t %s gdb -p %d\n", hostname, getpid()); + while(loop) { sleep(1); } + } + + // Number of CPU cores involved + int ncores = -1; // Use all available cores + parsec_context = parsec_init(ncores, &pargc, &pargv); + + int *gpu_device_index = NULL; + if( PARSEC_DEV_LEVEL_ZERO == device ) { + nbgpus = get_nb_gpu_devices(); + rc = !(nbgpus >= 1); + if( rc != 0 ) { + fprintf(stderr, "Rank %d doesn't have CUDA accelerators\n", rank); + MPI_Abort(MPI_COMM_WORLD, 0); + return -1; + } + gpu_device_index = get_gpu_device_index(); + } + + // Create datatypes + parsec_arena_datatype_t *adt = parsec_dtd_create_arena_datatype(parsec_context, &TILE_FULL); + parsec_add2arena_rect(adt, parsec_datatype_double_t, mb, nb, mb); + + if(check) { + fprintf(stderr, "Computing target GEMM\n"); + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcC, parsec_context, "C0"); + simple_gemm(parsec_context, dcA, dcB, dcC); + print_matrix(dcC, parsec_context, "C1"); + + device = PARSEC_DEV_CPU; + fprintf(stderr, "Computing CPU BLAS GEMM\n"); + parsec_matrix_block_cyclic_t *dcAcheck = create_initialize_matrix(parsec_context, rank, 1789, "Acheck", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcBcheck = create_initialize_matrix(parsec_context, rank, 1805, "Bcheck", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcCcheck = create_initialize_matrix(parsec_context, rank, 1901, "Ccheck", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcCcheck, parsec_context, "C'0"); + simple_gemm(parsec_context, dcAcheck, dcBcheck, dcCcheck); + print_matrix(dcCcheck, parsec_context, "C'1"); + + for( int i = 0; i < 0*dcC->super.mt; i++ ) { + for( int j = 0; j < dcC->super.nt; j++ ) { + if( (int)dcC->super.super.rank_of(&dcC->super.super, i, j) == parsec_context->my_rank ) { + parsec_data_t *CTile = dcC->super.super.data_of(&dcC->super.super, i, j); + parsec_data_t *DTile = dcCcheck->super.super.data_of(&dcCcheck->super.super, i, j); + double *c = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(CTile, 0)); + double *d = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(DTile, 0)); + for( int x = 0; x < mb*nb; x++) { + if( fabs(c[x] - d[x]) > EPSILON ) { + fprintf(stderr, "Tile (%d, %d), double %d is %g/%g -- differ by %g > %g\n", i, j, x, c[x], d[x], fabs(c[x]-d[x]), EPSILON ); + } + } + } + } + } + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + destroy_matrix(dcAcheck); + destroy_matrix(dcBcheck); + destroy_matrix(dcCcheck); + } else { + // Create and initialize the data + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + + for( int r = 0; r < runs + 1; r++ ) { + double gflop = 2.0 * M * N * K / 1e9; + double maxtime = 0.0; + if(min_perf > 0.0) + maxtime = gflop/world/nbgpus/min_perf; + struct timeval start, end, diff; + if(maxtime > 0.0 && maxtime < 60.0) maxtime=60.0; + if(rank == 0 && maxtime > 0.0) fprintf(stderr, "watchdog: %d seconds\n", (int)maxtime); + if(maxtime > 0.0) alarm((int)maxtime); + gettimeofday(&start, NULL); + simple_gemm(parsec_context, dcA, dcB, dcC); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + double t = (double)diff.tv_sec + (double)diff.tv_usec / 1e6; + double gflops = gflop / t; + (void)t; + (void)gflops; + if( 0 == rank && r > 0 ) { + fprintf(stderr, "DTD_GEMM PxQxg: %d %d %d M: %d N: %d K: %d mb: %d nb: %d kb: %d -- %g s, %g GFLop => %g GFLop/s\n", + P, Q, nbgpus, M, N, K, mb, nb, kb, t, gflop, gflops); + } + } + // deactivate the alarm if it was set + alarm(0); + + parsec_type_free(&adt->opaque_dtt); + PARSEC_OBJ_RELEASE(adt->arena); + parsec_dtd_destroy_arena_datatype(parsec_context, TILE_FULL); + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + } + + parsec_fini(&parsec_context); + +#if defined(PARSEC_HAVE_MPI) + MPI_Finalize(); +#endif + + return ret; +} diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C new file mode 100644 index 000000000..03d162262 --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C @@ -0,0 +1,77 @@ +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/mca/device/level_zero/device_level_zero.h" +#include "parsec/interfaces/dtd/insert_function.h" +#include "parsec/runtime.h" +#include "parsec/execution_stream.h" + +#include +#include + +extern "C" { + extern int gemm_lz_verbose; + int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + parsec_task_t *this_task = gpu_task->ec; + struct timeval start, end, diff; + double delta; + double *a_gpu, *b_gpu, *c_gpu; + parsec_level_zero_exec_stream_t *lz_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + cl::sycl::event gemm_event; + + (void)gpu_device; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + a_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 0)); + b_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 1)); + c_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 2)); + + gettimeofday(&start, NULL); + + double alpha=0.0; + double beta=1.0; + try { + gemm_event = oneapi::mkl::blas::gemm(lz_stream->swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, kb, nb, + alpha, static_cast(a_gpu), mb, + static_cast(b_gpu), kb, + beta, static_cast(c_gpu), nb); + gemm_event.wait(); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if(gemm_lz_verbose) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d [%p], %dx%d [%p], %dx%d [%p] on node %d, Stream %s of GPU %s submitted in %g s\n", + m, n, k, mb, kb, a_gpu, kb, nb, b_gpu, mb, kb, c_gpu, + this_task->taskpool->context->my_rank, + gpu_device->super.name, gpu_stream->name, delta); + + return PARSEC_HOOK_RETURN_DONE; +} diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index d4d73b014..69a493a9c 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -1,4 +1,22 @@ -if(PARSEC_HAVE_CUDA) +if(PARSEC_HAVE_LEVEL_ZERO) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + include(ParsecCompilePTG) + + parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) + target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") + target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) + else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink_lz test with DPCPP is disabled") + endif(TARGET MKL::MKL_DPCPP) +elseif(PARSEC_HAVE_CUDA) if( NOT TARGET CUDA::cublas ) message(WARNING "CUBLAS library not automatically detected when CUDA was found -- not enabling nvlink test") else( NOT TARGET CUDA::cublas ) @@ -27,4 +45,5 @@ if(PARSEC_HAVE_CUDA) target_ptg_sources(testing_get_best_device PRIVATE "get_best_device_check.jdf") parsec_addtest_cmd(get_best_device_shm ${SHM_TEST_CMD_LIST} ./testing_get_best_device -N 400 -t 20 -g 1) -endif(PARSEC_HAVE_CUDA) +endif(PARSEC_HAVE_LEVEL_ZERO) + diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index 2402b5cfe..5b473e20b 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -38,6 +38,13 @@ typedef cublasStatus_t (*cublas_dgemm_v2_t) ( cublasHandle_t handle, %} +extern "DPCPP" %{ + +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include + +%} + %option no_taskpool_instance = true /* can be anything */ /* @@ -150,6 +157,35 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } +} +END + BODY { fprintf(stderr, "Kernel GEMM(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", @@ -195,6 +231,36 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception that is also badly formatted..."); + } +} +END + + BODY { fprintf(stderr, "Kernel GEMM2(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 06a34079a..5b10b71e9 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -6,7 +6,6 @@ */ #include "parsec.h" -#include "parsec/mca/device/cuda/device_cuda_internal.h" #include "parsec/data_distribution.h" #include "parsec/data_dist/matrix/matrix.h" #include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" @@ -14,12 +13,20 @@ #include "parsec/class/info.h" #if defined(PARSEC_HAVE_CUDA) +#include "parsec/mca/device/cuda/device_cuda_internal.h" #include +#define TARGET_DEVICE_TYPE PARSEC_DEV_CUDA +#elif defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#define TARGET_DEVICE_TYPE PARSEC_DEV_LEVEL_ZERO +#else +#define TARGET_DEVICE_TYPE PARSEC_DEV_CPU #endif #include "nvlink.h" #if defined(PARSEC_HAVE_CUDA) +/* Only CUDA/CUBLAS requires to define handles on each GPU device */ static void destruct_cublas_handle(void *p) { cublasHandle_t handle = (cublasHandle_t)p; @@ -45,17 +52,15 @@ static void *create_cublas_handle(void *obj, void *p) (void)status; return (void*)handle; } -#endif static void destroy_cublas_handle(void *_h, void *_n) { -#if defined(PARSEC_HAVE_CUDA) cublasHandle_t cublas_handle = (cublasHandle_t)_h; cublasDestroy_v2(cublas_handle); -#endif (void)_n; (void)_h; } +#endif static void __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) @@ -65,22 +70,33 @@ __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) parsec_matrix_block_cyclic_t *dcA; parsec_del2arena( & nvlink_taskpool->arenas_datatypes[PARSEC_nvlink_DEFAULT_ADT_IDX] ); parsec_data_free(nvlink_taskpool->_g_descA->mat); +#if defined(PARSEC_HAVE_CUDA) parsec_info_unregister(&parsec_per_stream_infos, nvlink_taskpool->_g_CuHI, NULL); +#endif dcA = nvlink_taskpool->_g_descA; parsec_tiled_matrix_destroy( (parsec_tiled_matrix_t*)nvlink_taskpool->_g_descA ); userM = nvlink_taskpool->_g_userM; for(g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, userM->super.super.myrank); - parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, cuda_device->super.super.device_index); + parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, device->device_index); +#if defined(PARSEC_HAVE_CUDA) + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {} ); status = (cudaError_t)cudaFree( gpu_copy->device_private ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaFree ", status, {} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_result_t status = zeMemFree(level_zero_device->driver->ze_context, gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, {} ); +#else + free(gpu_copy->device_private); +#endif gpu_copy->device_private = NULL; - parsec_data_copy_detach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_detach(dta, gpu_copy, device->device_index); PARSEC_OBJ_RELEASE(gpu_copy); g++; } @@ -101,18 +117,18 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb parsec_matrix_block_cyclic_t *dcA; parsec_matrix_block_cyclic_t *userM; - /** Find all CUDA devices */ + /** Find all GPU devices */ nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { nb++; } } if(nb == 0) { char hostname[256]; gethostname(hostname, 256); - fprintf(stderr, "This test requires at least one CUDA device per node -- no CUDA device found on rank %d on %s\n", + fprintf(stderr, "This test requires at least one GPU device per node -- no GPU device found on rank %d on %s\n", ctx->my_rank, hostname); return NULL; } @@ -120,7 +136,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { dev_index[nb++] = device->device_index; } } @@ -179,31 +195,81 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb /* Now, we create a GPU version of each tile. As these tiles will be accessed RW * in the JDF, this also pins the task on the GPU that we chose to host the tile */ for(int g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { /* We get the data from the data collection */ parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, ctx->my_rank); /* The corresponding data copy on CPU RAM */ parsec_data_copy_t *cpu_copy = parsec_data_get_copy(dta, 0); /* And we create a new data copy on GPU */ parsec_data_copy_t *gpu_copy = PARSEC_OBJ_NEW(parsec_data_copy_t); +#if defined(PARSEC_HAVE_CUDA) /* We chose the GPU */ + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {return NULL;} ); /* Allocate memory on it, for one tile */ status = (cudaError_t)cudaMalloc( &gpu_copy->device_private, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMalloc ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_device_memory_properties_t devMemProperties; + uint32_t count = 1; + /* Safety: we check that there is one memory segment available on this device (ordinal 0), and + * that this segment is big enough to store what we need to allocate */ + ze_result_t status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { return NULL; }); + assert(count >= 1); + assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= (int)devMemProperties.totalSize); + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = 0 + }; + /* Allocate memory on it, for one tile */ + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + level_zero_device->ze_device, &gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); +#else + gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); +#endif /* Attach this copy to the data, on the corresponding device */ - parsec_data_copy_attach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_attach(dta, gpu_copy, device->device_index); /* We also need to tell PaRSEC that the owner of this data is the GPU, or the * GPU might not be selected to work on that data */ - parsec_data_transfer_ownership_to_copy(dta, cuda_device->super.super.device_index, PARSEC_FLOW_ACCESS_RW); + parsec_data_transfer_ownership_to_copy(dta, device->device_index, PARSEC_FLOW_ACCESS_RW); +#if defined(PARSEC_HAVE_CUDA) /* And copy the tile from CPU to GPU */ status = (cudaError_t)cudaMemcpy( gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, cudaMemcpyHostToDevice ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; + ze_fence_handle_t copySignalFence = level_zero_stream->fences[0]; + status = zeFenceReset(copySignalFence); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", status, { return NULL; } ); + status = zeCommandListReset(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); + status = zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[0], gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, NULL, 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + status = zeCommandListClose(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); + status = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[0], copySignalFence); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); + while(1) { + status = zeFenceQueryStatus(copySignalFence); + if(status == ZE_RESULT_SUCCESS) + break; + if(status != ZE_RESULT_NOT_READY) + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceQueryStatus ", status, { break; } ); + usleep(1000); + } +#else + memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); +#endif g++; } } diff --git a/tests/subsystem/CMakeLists.txt b/tests/subsystem/CMakeLists.txt new file mode 100644 index 000000000..38a7fed60 --- /dev/null +++ b/tests/subsystem/CMakeLists.txt @@ -0,0 +1,5 @@ +if(PARSEC_HAVE_LEVEL_ZERO) + add_subdirectory(level_zero) +else(PARSEC_HAVE_LEVEL_ZERO) + message(STATUS "PaRSEC doesn't have Level Zero support") +endif(PARSEC_HAVE_LEVEL_ZERO) diff --git a/tests/subsystem/level_zero/CMakeLists.txt b/tests/subsystem/level_zero/CMakeLists.txt new file mode 100644 index 000000000..de11e7887 --- /dev/null +++ b/tests/subsystem/level_zero/CMakeLists.txt @@ -0,0 +1,14 @@ +if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) +endif(NOT TARGET MKL::MKL_DPCPP) +if(TARGET MKL::MKL_DPCPP) +message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C lz SOURCES main.c kernel.dpcpp.C interface.dpcpp.C) + target_include_directories(lz PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_link_libraries(lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) +else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, Level Zero test with DPCPP is disabled") +endif(TARGET MKL::MKL_DPCPP) diff --git a/tests/subsystem/level_zero/interface.dpcpp.C b/tests/subsystem/level_zero/interface.dpcpp.C new file mode 100644 index 000000000..8abe7c5e8 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.C @@ -0,0 +1,56 @@ +#include "level_zero/ze_api.h" +#include "interface.dpcpp.h" + +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + sycl_wrapper_driver_t *res = new sycl_wrapper_driver_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device) +{ + std::vectordevices; + + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + swp->context = sycl::make_context(hContextInteropInput); +} + +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + sycl_wrapper_device_t *res = new sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); + + return res; +} + +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) +{ + sycl_wrapper_queue_t *swq = new sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} + +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp) +{ + delete swp; + return 0; +} + +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd) +{ + delete swd; + return 0; +} + +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq) +{ + delete swq; + return 0; +} + diff --git a/tests/subsystem/level_zero/interface.dpcpp.h b/tests/subsystem/level_zero/interface.dpcpp.h new file mode 100644 index 000000000..04aa2f380 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.h @@ -0,0 +1,40 @@ +#ifndef INTERFACE_DPCPP_H +#define INTERFACE_DPCPP_H + +typedef struct sycl_wrapper_driver_s sycl_wrapper_driver_t; +typedef struct sycl_wrapper_device_s sycl_wrapper_device_t; +typedef struct sycl_wrapper_queue_s sycl_wrapper_queue_t; + +#if defined(c_plusplus) || defined(__cplusplus) +#include "sycl/ext/oneapi/backend/level_zero.hpp" + +struct sycl_wrapper_driver_s { + sycl::platform platform; + sycl::context context; +}; + +struct sycl_wrapper_device_s { + sycl::device device; +}; + +struct sycl_wrapper_queue_s { + sycl::queue queue; +}; + +extern "C" { +#endif + +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device); +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device); +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp); +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd); +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //INTERFACE_DPCPP_H diff --git a/tests/subsystem/level_zero/kernel.dpcpp.C b/tests/subsystem/level_zero/kernel.dpcpp.C new file mode 100644 index 000000000..0ab12e76a --- /dev/null +++ b/tests/subsystem/level_zero/kernel.dpcpp.C @@ -0,0 +1,64 @@ +#include +#include "sycl/ext/oneapi/backend/level_zero.hpp" +#include "interface.dpcpp.h" + +extern "C" { + int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); +} + +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb) +{ + double alpha=1.0; + double beta=1.0; +/* sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bA = sycl::make_buffer(hBufferInteropInputA, swp->context); + auto bbA = bA.reinterpret(sycl::range<1>(mb*mb)); + sycl::backend_input_t> hBufferInteropInputC = { (void*)C, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bC = sycl::make_buffer(hBufferInteropInputC, swp->context); + auto bbC = bC.reinterpret(sycl::range<1>(mb*mb));*/ + try { + oneapi::mkl::blas::gemm(swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, mb, mb, + alpha, A, mb, + A, mb, + beta, C, mb); + } catch (const oneapi::mkl::invalid_argument &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception\n"); + } catch (const oneapi::mkl::unsupported_device &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unsuported device exception\n"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws host bad allocation exception\n"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws device bad allocation exception\n"); + } catch (const oneapi::mkl::unimplemented &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unimplemented exception\n"); + } catch (const std::exception& e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception '%s'\n", e.what()); + } catch (...) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + fprintf(stderr, "kernel has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n"); + + return 0; +} +/* +void *sycl_malloc(sycl_wrapper_t *sw, size_t size) +{ + return sycl::malloc_device(size, sw->queue); +} + +void sycl_free(sycl_wrapper_t *sw, void *ptr) +{ + sycl::free(ptr, sw->queue); +} +*/ diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c new file mode 100644 index 000000000..d6cce9635 --- /dev/null +++ b/tests/subsystem/level_zero/main.c @@ -0,0 +1,581 @@ +#include +#include +#include +#include +#include + +#include +#include "interface.dpcpp.h" + +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); + +struct driver_s; +struct device_s; +struct stream_s; + +#define NB_STREAMS 4 +#define MAX_FENCES 2 + +typedef struct stream_s { + int immediate; + sycl_wrapper_queue_t *swq; + ze_command_queue_handle_t cq; + ze_command_list_handle_t cl; + struct device_s *device; + ze_fence_handle_t fences[MAX_FENCES]; +} stream_t; + +typedef struct device_s { + ze_device_handle_t device; + struct driver_s *driver; + sycl_wrapper_device_t *swd; + stream_t streams[NB_STREAMS]; +} device_t; + +typedef struct driver_s { + ze_driver_handle_t driver; + ze_context_handle_t context; + sycl_wrapper_driver_t *swp; + int nb_devices; + device_t *devices; +} driver_t; + +#define LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + fprintf(stderr, "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +static int init_device(device_t *device, ze_device_handle_t gpuDevice) +{ + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + ze_result_t ze_rc; + + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (count) ", ze_rc, { return -1; } ); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, cmdqueueGroupProperties); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (populate) ", ze_rc, { return -1; } ); + + // Find a command queue type that support compute + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); ++i ) { + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as compute queue\n", i); + if(cmdqueueGroupCount == computeQueueGroupOrdinal) + computeQueueGroupOrdinal = i; + } + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as copy queue\n", i); + if(cmdqueueGroupCount == copyQueueGroupOrdinal) + copyQueueGroupOrdinal = i; + } + } + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + fprintf(stderr, "INFO: computeQueueGroupOrdinal = %d, copyQueueGroupOrdinal = %d\n", computeQueueGroupOrdinal, copyQueueGroupOrdinal); + + for(int j = 0; j < NB_STREAMS; j++ ) { + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + device->streams[j].device = device; + if( 0 && j < 2 ) { + device->streams[j].immediate = 1; + commandQueueDesc.ordinal = copyQueueGroupOrdinal; + ze_rc = zeCommandListCreateImmediate(device->driver->context, gpuDevice, + &commandQueueDesc, + &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, { return -1;} ); + } else { + device->streams[j].immediate = 0; + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(device->driver->context, gpuDevice, + &commandQueueDesc, &device->streams[j].cq); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, { return -1;} ); + ze_command_list_desc_t commandListDesc = { + 0 // flags + }; + ze_rc = zeCommandListCreate(device->driver->context, gpuDevice, + &commandListDesc, &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, { return -1;} ); + } + + for(int k = 0; k < MAX_FENCES; k++ ) { + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 + }; + device->streams[j].fences[k] = NULL; + ze_rc = zeFenceCreate(device->streams[j].cq, &fence_desc, &(device->streams[j].fences[k])); + LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, { continue; } ); + } + } + device->device = gpuDevice; + return 0; +} + +static int init_driver(driver_t *driver, int maxDevices) +{ + uint32_t deviceCount = 0; + ze_device_handle_t *allDevices; + ze_device_handle_t *gpuDevices; + ze_result_t ze_rc; + + ze_rc = zeDeviceGet(driver->driver, &deviceCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (count) ", ze_rc, { return -1; } ); + if(deviceCount == 0) + return 0; + + allDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + gpuDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + ze_rc = zeDeviceGet(driver->driver, &deviceCount, allDevices); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (populate) ", ze_rc, { return -1; } ); + + int deviceId = 0; + for(int did = 0; did < (int)deviceCount; did++) { + ze_device_properties_t device_properties; + zeDeviceGetProperties(allDevices[did], &device_properties); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + gpuDevices[deviceId++] = allDevices[did]; + if( deviceId > maxDevices ) { + break; + } + } + free(allDevices); + allDevices = NULL; + + if( deviceId == 0) { + free(gpuDevices); + return 0; + } + deviceCount = deviceId; + + driver->devices = (device_t*)malloc(deviceCount * sizeof(device_t)); + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_rc = zeContextCreate(driver->driver, &ctxtDesc, &driver->context); + LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { continue; } ); + + int dpos = 0; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[dpos].driver = driver; + if( init_device(&driver->devices[dpos], gpuDevices[did]) < 0 ) { + continue; + } + dpos++; + } + deviceCount = dpos; + + driver->nb_devices = deviceCount; + if(deviceCount == 0) { + free(driver->devices); + driver->devices = NULL; + return 0; + } + free(gpuDevices); + + driver->swp = sycl_wrapper_platform_create(driver->driver); + sycl_wrapper_device_t *swd[deviceCount]; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[did].swd = sycl_wrapper_device_create(driver->devices[did].device); + swd[did] = driver->devices[did].swd; + } + sycl_wrapper_platform_add_context(driver->swp, driver->context, swd, deviceCount); + for(int did = 0; did < (int)deviceCount; did++) { + for(int sid = 0; sid < NB_STREAMS; sid++) { + driver->devices[did].streams[sid].swq = sycl_wrapper_queue_create(driver->swp, driver->devices[did].swd, driver->devices[did].streams[sid].cq); + } + } + + return deviceCount; +} + +static void *allocate_workspace(device_t *device, size_t size) +{ + ze_result_t status; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + void *device_ptr; + uint32_t count = 0; + int memIndex = -1; + + status = zeDeviceGetMemoryAccessProperties(device->device, &memAccessProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { return NULL; }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + fprintf(stderr, "Device does not have memory allocation capabilities with RW access\n"); + return NULL; + } + status = zeDeviceGetProperties(device->device, &devProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { return NULL; }); + status = zeDeviceGetMemoryProperties(device->device, &count, NULL); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (count) ", status, { return NULL; }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(device->device, &count, devMemProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (populate) ", status, { free(devMemProperties); return NULL; }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + + if( size > devMemProperties[memIndex].totalSize ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + fprintf(stderr, "Requested %zd bytes on LEVEL_ZERO device, but only %zd bytes are available -- Returning NULL\n", + size, devMemProperties[memIndex].totalSize); + return NULL; + } + free(devMemProperties); devMemProperties = NULL; + + /*device_ptr = sycl_malloc(device->streams[2].sw, size); + assert(NULL != device_ptr); */ + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + + status = zeMemAllocDevice(device->driver->context, &memAllocDesc, size, 128, + device->device, &device_ptr); + LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); + return device_ptr; +} + +typedef struct { + struct timespec start; + struct timespec enter_wait; + struct timespec end; +} run_timing_t; + +typedef struct { + struct timespec start; + struct timespec start_alloc; + struct timespec end_alloc; + struct timespec start_cpu2gpu; + struct timespec enter_wait_cpu2gpu; + struct timespec end_cpu2gpu; + struct timespec start_gpu2cpu; + struct timespec enter_wait_gpu2cpu; + struct timespec end_gpu2cpu; + run_timing_t *runs; +} device_timing_t; + +double diff_timespec(const struct timespec *time1, const struct timespec *time0) { + return (time1->tv_sec - time0->tv_sec) + + (time1->tv_nsec - time0->tv_nsec) / 1000000000.0; +} + +static void usage(const char *name) { + fprintf(stderr, "USAGE: %s [] [Problem size] [Number of iterations]\n", name); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + ze_result_t ze_rc; + uint32_t driverCount = 0; + driver_t *drivers; + ze_driver_handle_t *allDrivers; + int max_devices = 1024*1024, nb_devices = 0; + void **device_workspaceA, **device_workspaceC; + int errors = 0; + int N = 4096; + int NRUNS = 3; + device_timing_t *timings; + + if(argc > 1) { + max_devices = atoi(argv[1]); + if(max_devices <= 0) { + usage(argv[0]); + } + + if(argc > 2) { + N = atoi(argv[2]); + if(N <= 0) { + usage(argv[0]); + } + + if(argc >3) { + NRUNS = atoi(argv[3]); + if(NRUNS <= 0) { + usage(argv[0]); + } + } + } + } + + ze_rc = zeInit( 0 ); + LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, { return -1; }); + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (count) ", ze_rc, { return 1; } ); + fprintf(stderr, "STATUS: found %d drivers\n", driverCount); + drivers = malloc(driverCount * sizeof(driver_t)); + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (populate) ", ze_rc, { return 1; } ); + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + int nb; + drivers[driverId].driver = allDrivers[driverId]; + if( (nb = init_driver(&drivers[driverId], max_devices)) <= 0 ) { + fprintf(stderr, "%d device found in driver %d... Bailing out\n", nb_devices, driverId); + return 1; + } else { + fprintf(stderr, "STATUS: there are %d GPU devices in driver %d\n", nb, driverId); + nb_devices += nb; + nb_devices = nb_devices <= max_devices ? nb_devices : max_devices; + } + } + free(allDrivers); + fprintf(stderr, "%d devices found and initialized\n", nb_devices); + + timings = (device_timing_t*)calloc(sizeof(device_timing_t), nb_devices); + for(int i = 0; i < nb_devices; i++) { + clock_gettime(CLOCK_REALTIME, &timings[i].start); + timings[i].runs = (run_timing_t*)calloc(sizeof(run_timing_t), NRUNS); + } + + //Allocate GPU memory for each device + device_workspaceA = (void**)malloc(sizeof(void*)*nb_devices); + device_workspaceC = (void**)malloc(sizeof(void*)*nb_devices); + int did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + clock_gettime(CLOCK_REALTIME, &timings[did].start_alloc); + device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N*2); + device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + clock_gettime(CLOCK_REALTIME, &timings[did].end_alloc); + did++; + } + } + + //Initialize the GPU memory for each device + double *cpuA = (double*)calloc(sizeof(double), N*N); + for(int i = 0; i < N; i++) { + cpuA[i*N+i] = 1.0; + } + double *cpuC = (double*)calloc(sizeof(double), N*N); + did=0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Initialize A and C on device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)cpuA; + void *copy_out = (void*)device_workspaceA[did]; + size_t count = N*N*sizeof(double); + clock_gettime(CLOCK_REALTIME, &timings[did].start_cpu2gpu); + ze_rc = zeCommandListReset(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(A->GPU) ", ze_rc, { continue; } ); + copy_in = (void*)cpuC; + copy_out = (void*)device_workspaceC[did]; + count = N*N*sizeof(double); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(C->GPU) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[0].cq, 1, &device->streams[0].cl, device->streams[0].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_cpu2gpu); + do { + ze_rc = zeFenceQueryStatus(device->streams[0].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: copies ended on device %d of driver %d\n", deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].end_cpu2gpu); + fprintf(stderr, "STATUS: CPU->GPU copies on device %d of driver %d completed!\n", deviceId, driverId); + } + did++; + } + } + + for(int run = 0; run < NRUNS; run++) { + //Do a GEMM (blocking) on each device, and wait for its completion -- yes, memory is not initialized. + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Ready to submit GEMM[%d] on device %d of driver %d\n", run, deviceId, driverId); + fprintf(stderr, "STATUS: Context of driver %d is %s\n", driverId, zeContextGetStatus(drivers[driverId].context) == ZE_RESULT_SUCCESS ? "Fine" : "Broken"); + dpcpp_kernel_GEMM(device->driver->swp, device->swd, device->streams[2].swq, (double*)device_workspaceA[did], (double*)device_workspaceC[did], N); + fprintf(stderr, "STATUS: GEMM[%d] submitted on device %d of driver %d\n", run, deviceId, driverId); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].start); + ze_rc = zeCommandListReset(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = zeCommandListClose(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + + ze_rc = zeFenceReset(device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[2].cq, 1, &device->streams[2].cl, device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].enter_wait); + do { + ze_rc = zeFenceQueryStatus(device->streams[2].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: GEMM[%d] ended on device %d of driver %d\n", run, deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].end); + fprintf(stderr, "STATUS: GEMM[%d] on device %d of driver %d completed!\n", run, deviceId, driverId); + } + did++; + } + } + } + + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Get C back from device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)device_workspaceC[did]; + void *copy_out = (void*)cpuC; + size_t count = N*N*sizeof(double); + + for(int i = 0; i < N*N; i++) cpuC[i] = -1.0; + + clock_gettime(CLOCK_REALTIME, &timings[did].start_gpu2cpu); + ze_rc = zeCommandListReset(device->streams[1].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[1].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(GPU->A) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[1].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeFenceReset(device->streams[1].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[1].cq, 1, &device->streams[1].cl, device->streams[1].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_gpu2cpu); + do { + ze_rc = zeFenceQueryStatus(device->streams[1].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: copies ended from device %d of driver %d\n", deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].end_gpu2cpu); + fprintf(stderr, "STATUS: GPU->CPU copies on device %d of driver %d completed!\n", deviceId, driverId); + + int error0 = 0; + int error1 = 0; + for(int i = 0; i < N; i++) { + for(int j = 0; j < N; j++) { + if(i != j) { + if(cpuC[i*N+j] != 0.0) { + if(0 == error0) + fprintf(stderr, "ERROR during checks of driver %d: C[%d,%d] = %g expected 0.0\n", did, i, j, cpuC[i*N+j]); + error0++; + errors++; + } + } else { + if(cpuC[i*N+j] != (double)NRUNS) { + if(0 == error1) + fprintf(stderr, "ERROR during checks of driver %d: C[%d, %d] = %g, expected %g\n", did, i, j, cpuC[i*N+j], (double)NRUNS); + error1++; + errors++; + } + } + } + } + if(error0 > 0) fprintf(stderr, " %d errors of type 0 during checks of driver %d\n", error0, did); + if(error1 > 0) fprintf(stderr, " %d errors of type 1 during checks of driver %d\n", error1, did); + } + did++; + } + } + + for(did = 0; did < nb_devices; did++) { + printf("TIME: Device %d\n", did); + printf("TIME: START: %ld.%09ld s\n", timings[did].start.tv_sec, timings[did].start.tv_nsec); + printf("TIME: START_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].start_cpu2gpu, &timings[did].start)); + printf("TIME: START_ENTER_WAIT_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_cpu2gpu, &timings[did].start_cpu2gpu)); + printf("TIME: END_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].end_cpu2gpu, &timings[did].enter_wait_cpu2gpu)); + for(int run = 0; run < NRUNS; run++) { + printf("TIME: START_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].start, + run == 0 ? &timings[did].end_cpu2gpu : &timings[did].runs[run-1].end)); + printf("TIME: ENTER_WAIT_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].enter_wait, &timings[did].runs[run].start)); + printf("TIME: END_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].enter_wait)); + double tot_kernel = diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].start); + double gflops = 2.0*N*N*N/1e9; + printf("PERF: device %d run %d %11.9g s for %d x %d x %d DGEMM (%g . 10^9 flops): %g GFLOP/s\n", did, run, tot_kernel, N, N, N, gflops, gflops/tot_kernel); + } + printf("TIME: START_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].start_gpu2cpu, &timings[did].runs[NRUNS-1].end)); + printf("TIME: ENTER_WAIT_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_gpu2cpu, &timings[did].start_gpu2cpu)); + printf("TIME: END_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].end_gpu2cpu, &timings[did].enter_wait_gpu2cpu)); + printf("\n"); + } + + if(errors == 0) + return EXIT_SUCCESS; + return EXIT_FAILURE; +} diff --git a/tools/profiling/python/setup.py.in b/tools/profiling/python/setup.py.in index 1e250c308..fc2305a78 100644 --- a/tools/profiling/python/setup.py.in +++ b/tools/profiling/python/setup.py.in @@ -44,14 +44,14 @@ if 'Debug' == build_type or 'RelWithDebInfo' == build_type: # Cython does not support VPATH, so the generated files will always reside on the # SOURCE_DIR, except if we explicitly move them to the build directory. -extensions = [Extension('pbt2ptt', ['@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pyx', +extensions = [Extension('pbt2ptt', [os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pyx'), os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/../dbpreader.c')], include_dirs=['@PROJECT_SOURCE_DIR@/parsec/include', '@PROJECT_SOURCE_DIR@', '@PROJECT_BINARY_DIR@/parsec/include', '@PROJECT_BINARY_DIR@', '@CMAKE_CURRENT_SOURCE_DIR@/../'], - depends=['pbt2ptt.pxd', + depends=[os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/pbt2ptt.pxd'), os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/../dbpreader.h'), '@PROJECT_SOURCE_DIR@/parsec/parsec_binary_profile.h', '@PROJECT_SOURCE_DIR@/parsec/include/parsec/os-spec-timing.h'], @@ -130,7 +130,9 @@ setup( description='PaRSEC Binary Trace Interface parses and converts the PaRSEC Binary Trace format into a pandas-based Python tabular format', url='http://icl.cs.utk.edu/parsec/', package_dir={ '': '@PYTHON_TOOLS_BIN_DIR@'}, - py_modules=['ptt_utils', 'parsec_trace_tables', 'common_utils'], + py_modules=[os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/ptt_utils'), + os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/parsec_trace_tables'), + os.path.realpath('@CMAKE_CURRENT_SOURCE_DIR@/common_utils')], cmdclass = {'build_ext': local_compiler_build_ext}, ext_modules = cythonize(extensions, compiler_directives={'language_level' : @Python_VERSION_MAJOR@},